From 56087d5b2109378325faa3d8b58a8870b1467958 Mon Sep 17 00:00:00 2001 From: relikd Date: Sat, 26 Sep 2020 21:32:44 +0200 Subject: [PATCH] Prepare for custom lists --- out/static/ranking.js | 4 +-- src/html_ranking.py | 4 +-- src/index_rank.py | 69 +++++++++++++++++++++++++------------------ src/lib_common.py | 9 +++++- 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/out/static/ranking.js b/out/static/ranking.js index c064978..351a41e 100644 --- a/out/static/ranking.js +++ b/out/static/ranking.js @@ -51,9 +51,9 @@ function sort_by(col, asc) { _data.sort(function(a, b){ return a[i] < b[i] ? -o : a[i] > b[i] ? o : 0; }); update(col, asc); } -function rank_js(fname) { +function rank_js(fname, column, order) { loadJSON(fname, function(response) { _data = JSON.parse(response); - update(12,-1); + update(column, order); }); } \ No newline at end of file diff --git a/src/html_ranking.py b/src/html_ranking.py index 06ecdbe..ac75fd6 100755 --- a/src/html_ranking.py +++ b/src/html_ranking.py @@ -23,7 +23,7 @@ def html_script_chunk(fname): '''.format(fname) @@ -36,7 +36,7 @@ def write_ranking_category(cid, category_name): 'raw-category-{}.json'.format(cid)) src += html_script_chunk('data.json') HTML.write(base, src, title='Category Ranking: ' + category_name) - mylib.symlink(index_rank.fname_ranking_category(cid), + mylib.symlink(index_rank.fname_rank_list('category', cid), mylib.path_add(base, 'data.json')) diff --git a/src/index_rank.py b/src/index_rank.py index b31c0db..ec30653 100755 --- a/src/index_rank.py +++ b/src/index_rank.py @@ -17,15 +17,15 @@ def fname_app_rank(): def fname_ranking_all(): - return mylib.path_data_index('ranking_all.json') + return mylib.path_data_index('rank', 'all.json') -def fname_ranking_category(cid): - return mylib.path_data_index('rank', 'id_{}.json'.format(cid)) +def fname_rank_list(sublist, cid): + return mylib.path_data_index('rank', sublist, 'id_{}.json'.format(cid)) -def make_dir_individuals(reset=False): - pth = mylib.path_data_index('rank') +def make_rank_list_dir(sublist, reset=False): + pth = mylib.path_data_index('rank', sublist) if reset: mylib.rm_dir(pth) mylib.mkdir(pth) @@ -75,47 +75,58 @@ def update_summary_index(index, bundle_ids, deleteOnly=False): return did_change +def filter_by_list(index, list_ids, updated_ids): + if len(list_ids) == 0 or len(updated_ids) == 0: + return + if updated_ids != ['*'] and not any(x in list_ids for x in updated_ids): + return + c = 0 + for x in index: + if x[0] in list_ids: + yield x + c += 1 + if c >= MAX_RANKING_LIMIT: + break + + def write_ranking_category_list(index, affected_ids): - reset = affected_ids == ['*'] - - def category_affected(category_bundle_ids): - if reset or len(affected_ids) > 10: - return True - for x in affected_ids: - if x in category_bundle_ids: - return True - return False - - make_dir_individuals(reset) + make_rank_list_dir('category', reset=affected_ids == ['*']) for _, json in mylib.enum_categories(): - cid, cname = json['meta'] ids = [bid for bid, _ in json['apps']] - ret = [] - if len(ids) > 0 and category_affected(ids): - for x in index: - if x[0] not in ids: - continue - ret.append(x) - if len(ids) == 0 or len(ret) >= MAX_RANKING_LIMIT: - break - mylib.json_write(fname_ranking_category(cid), ret, pretty=False) + ret = list(filter_by_list(index, ids, affected_ids)) + cid = json['meta'][0] + mylib.json_write(fname_rank_list('category', cid), ret, pretty=False) + + +def write_ranking_custom_lists(index, affected_ids): + make_rank_list_dir('custom', reset=affected_ids == ['*']) + for list_id, json in mylib.enum_custom_lists(): + ret = list(filter_by_list(index, json['apps'], affected_ids)) + mylib.json_write(fname_rank_list('custom', list_id), ret, pretty=False) def write_ranking_list(index, affected_ids): ret = [] for bid, values in index.items(): ret.append([bid, index_app_names.get_name(bid)] + values) - del(values[8:]) + del(values[8:]) # prepare for write_rank_index + + print(' write custom lists') + write_ranking_custom_lists(ret, affected_ids) + ret.sort(key=lambda x: -x[2 + 10]) # sort by last update + + print(' write category lists') write_ranking_category_list(ret, affected_ids) - # TODO: doesnt scale well, 100'000 apps ~> 12mb if len(ret) > MAX_RANKING_LIMIT: # limit to most recent X entries ret = ret[:MAX_RANKING_LIMIT] # mylib.sort_by_name(ret, 1) + print(' write overall list') mylib.json_write(fname_ranking_all(), ret, pretty=False) def write_rank_index(index): + print(' generate bundle ranks') mylib.try_del(index, ['_ranks', '_min', '_max']) mins = [] maxs = [] @@ -151,7 +162,7 @@ def get_total_counts(): def process(bundle_ids, deleteOnly=False): - print('writing index: meta ...') + print('writing index: ranking ...') fname = fname_app_summary() if bundle_ids == ['*']: print(' full reset') diff --git a/src/lib_common.py b/src/lib_common.py index 5805c88..5c7000c 100755 --- a/src/lib_common.py +++ b/src/lib_common.py @@ -247,6 +247,12 @@ def enum_newly_added(): yield fname, os.path.basename(fname)[3:] # del prefix 'in_' +def enum_custom_lists(): + for fname in glob.glob(path_data('_lists', 'id_*.json')): + with open(fname, 'r') as fp: + yield os.path.basename(fname)[3:-5], json.load(fp) + + def enum_jsons(bundle_id): for fname in glob.glob(path_data_app(bundle_id, 'id_*.json')): with open(fname, 'r') as fp: @@ -262,7 +268,8 @@ def enum_categories(): def appids_in_out(selection=None): if selection and selection != ['*']: return selection - return [os.path.basename(x) for x in glob.glob(path_out_app('*'))] + return [os.path.basename(os.path.dirname(x)) + for x in glob.glob(path_out_app('*/'))] def appids_in_data(selection=None):