Prepare for custom lists

out/static/ranking.js

@@ -51,9 +51,9 @@ function sort_by(col, asc) {
   _data.sort(function(a, b){ return a[i] < b[i] ? -o : a[i] > b[i] ? o : 0; });
   update(col, asc);
 }
-function rank_js(fname) {
+function rank_js(fname, column, order) {
   loadJSON(fname, function(response) {
     _data = JSON.parse(response);
-    update(12,-1);
+    update(column, order);
   });
 }

src/html_ranking.py

@@ -23,7 +23,7 @@ def html_script_chunk(fname):
 <script type="text/javascript" src="/static/ranking.js"></script>
 <script type="text/javascript" src="/static/lozad.js"></script>
 <script type="text/javascript">
-  rank_js('{}');
+  rank_js('{}', 12, -1);
 </script>'''.format(fname)
 
 
@@ -36,7 +36,7 @@ def write_ranking_category(cid, category_name):
                                 'raw-category-{}.json'.format(cid))
     src += html_script_chunk('data.json')
     HTML.write(base, src, title='Category Ranking: ' + category_name)
-    mylib.symlink(index_rank.fname_ranking_category(cid),
+    mylib.symlink(index_rank.fname_rank_list('category', cid),
                   mylib.path_add(base, 'data.json'))
 
 

src/index_rank.py

@@ -17,15 +17,15 @@ def fname_app_rank():
 
 
 def fname_ranking_all():
-    return mylib.path_data_index('ranking_all.json')
+    return mylib.path_data_index('rank', 'all.json')
 
 
-def fname_ranking_category(cid):
+def fname_rank_list(sublist, cid):
-    return mylib.path_data_index('rank', 'id_{}.json'.format(cid))
+    return mylib.path_data_index('rank', sublist, 'id_{}.json'.format(cid))
 
 
-def make_dir_individuals(reset=False):
+def make_rank_list_dir(sublist, reset=False):
-    pth = mylib.path_data_index('rank')
+    pth = mylib.path_data_index('rank', sublist)
     if reset:
         mylib.rm_dir(pth)
     mylib.mkdir(pth)
@@ -75,47 +75,58 @@ def update_summary_index(index, bundle_ids, deleteOnly=False):
     return did_change
 
 
+def filter_by_list(index, list_ids, updated_ids):
+    if len(list_ids) == 0 or len(updated_ids) == 0:
+        return
+    if updated_ids != ['*'] and not any(x in list_ids for x in updated_ids):
+        return
+    c = 0
+    for x in index:
+        if x[0] in list_ids:
+            yield x
+            c += 1
+            if c >= MAX_RANKING_LIMIT:
+                break
+
+
 def write_ranking_category_list(index, affected_ids):
-    reset = affected_ids == ['*']
+    make_rank_list_dir('category', reset=affected_ids == ['*'])
-
-    def category_affected(category_bundle_ids):
-        if reset or len(affected_ids) > 10:
-            return True
-        for x in affected_ids:
-            if x in category_bundle_ids:
-                return True
-        return False
-
-    make_dir_individuals(reset)
     for _, json in mylib.enum_categories():
-        cid, cname = json['meta']
         ids = [bid for bid, _ in json['apps']]
-        ret = []
+        ret = list(filter_by_list(index, ids, affected_ids))
-        if len(ids) > 0 and category_affected(ids):
+        cid = json['meta'][0]
-            for x in index:
+        mylib.json_write(fname_rank_list('category', cid), ret, pretty=False)
-                if x[0] not in ids:
+
-                    continue
+
-                ret.append(x)
+def write_ranking_custom_lists(index, affected_ids):
-                if len(ids) == 0 or len(ret) >= MAX_RANKING_LIMIT:
+    make_rank_list_dir('custom', reset=affected_ids == ['*'])
-                    break
+    for list_id, json in mylib.enum_custom_lists():
-        mylib.json_write(fname_ranking_category(cid), ret, pretty=False)
+        ret = list(filter_by_list(index, json['apps'], affected_ids))
+        mylib.json_write(fname_rank_list('custom', list_id), ret, pretty=False)
 
 
 def write_ranking_list(index, affected_ids):
     ret = []
     for bid, values in index.items():
         ret.append([bid, index_app_names.get_name(bid)] + values)
-        del(values[8:])
+        del(values[8:])  # prepare for write_rank_index
+
+    print('  write custom lists')
+    write_ranking_custom_lists(ret, affected_ids)
+
     ret.sort(key=lambda x: -x[2 + 10])  # sort by last update
+
+    print('  write category lists')
     write_ranking_category_list(ret, affected_ids)
-    # TODO: doesnt scale well, 100'000 apps ~> 12mb
     if len(ret) > MAX_RANKING_LIMIT:  # limit to most recent X entries
         ret = ret[:MAX_RANKING_LIMIT]
     # mylib.sort_by_name(ret, 1)
+    print('  write overall list')
     mylib.json_write(fname_ranking_all(), ret, pretty=False)
 
 
 def write_rank_index(index):
+    print('  generate bundle ranks')
     mylib.try_del(index, ['_ranks', '_min', '_max'])
     mins = []
     maxs = []
@@ -151,7 +162,7 @@ def get_total_counts():
 
 
 def process(bundle_ids, deleteOnly=False):
-    print('writing index: meta ...')
+    print('writing index: ranking ...')
     fname = fname_app_summary()
     if bundle_ids == ['*']:
         print('  full reset')

src/lib_common.py

@@ -247,6 +247,12 @@ def enum_newly_added():
         yield fname, os.path.basename(fname)[3:]  # del prefix 'in_'
 
 
+def enum_custom_lists():
+    for fname in glob.glob(path_data('_lists', 'id_*.json')):
+        with open(fname, 'r') as fp:
+            yield os.path.basename(fname)[3:-5], json.load(fp)
+
+
 def enum_jsons(bundle_id):
     for fname in glob.glob(path_data_app(bundle_id, 'id_*.json')):
         with open(fname, 'r') as fp:
@@ -262,7 +268,8 @@ def enum_categories():
 def appids_in_out(selection=None):
     if selection and selection != ['*']:
         return selection
-    return [os.path.basename(x) for x in glob.glob(path_out_app('*'))]
+    return [os.path.basename(os.path.dirname(x))
+            for x in glob.glob(path_out_app('*/'))]
 
 
 def appids_in_data(selection=None):