diff --git a/src/index_rank.py b/src/index_rank.py index ba5e8d2..f17d199 100755 --- a/src/index_rank.py +++ b/src/index_rank.py @@ -3,7 +3,9 @@ import sys import lib_common as mylib import bundle_combine # get_evaluated -import index_app_names +import index_app_names # get_name + +MAX_RANKING_LIMIT = 500 def fname_app_summary(): @@ -35,35 +37,73 @@ def json_to_list(json): ] -def write_summary_index(index, bundle_ids, deleteOnly=False): - for bid in bundle_ids: - # delete old value - mylib.try_del(index, [bid]) - if deleteOnly: - continue - # set new value - index[bid] = json_to_list(bundle_combine.get_evaluated(bid)) - - # sum of counts - mylib.try_del(index, ['_sum']) - total = [0, 0] - for val in index.values(): - total[0] += val[0] - total[1] += val[8] - index['_sum'] = total - mylib.json_write(fname_app_summary(), index, pretty=False) - mylib.try_del(index, ['_sum']) +def update_summary_index(index, bundle_ids, deleteOnly=False): + did_change = False + if deleteOnly: + did_change = mylib.try_del(index, bundle_ids) + else: + for bid in bundle_ids: + # set new value + new_value = json_to_list(bundle_combine.get_evaluated(bid)) + try: + if new_value == index[bid]: + continue + except KeyError: + pass + index[bid] = new_value + did_change = True + if did_change: + mylib.try_del(index, ['_sum']) + total = [0, 0] + for val in index.values(): + total[0] += val[0] + total[1] += val[8] + index['_sum'] = total + mylib.json_write(fname_app_summary(), index, pretty=False) + mylib.try_del(index, ['_sum']) + return did_change -def write_ranking_list(index): +def write_ranking_category_list(index, affected_ids): + reset = affected_ids == ['*'] + + def category_affected(category_bundle_ids): + if reset or len(affected_ids) > 10: + return True + for x in affected_ids: + if x in category_bundle_ids: + return True + return False + + pth = mylib.path_data_index('rank') + if reset: + mylib.rm_dir(pth) + mylib.mkdir(pth) + for _, json in mylib.enum_categories(): + cid, cname = json['meta'] + ids = [bid for bid, _ in json['apps']] + ret = [] + if len(ids) > 0 and category_affected(ids): + for x in index: + if x[0] not in ids: + continue + ret.append(x) + if len(ids) == 0 or len(ret) >= MAX_RANKING_LIMIT: + break + mylib.json_write(mylib.path_add(pth, 'id_{}.json'.format(cid)), + ret, pretty=False) + + +def write_ranking_list(index, affected_ids): ret = [] for bid, values in index.items(): ret.append([bid, index_app_names.get_name(bid)] + values) del(values[8:]) ret.sort(key=lambda x: -x[2 + 10]) # sort by last update + write_ranking_category_list(ret, affected_ids) # TODO: doesnt scale well, 100'000 apps ~> 12mb - if len(ret) > 500: # limit to most recent X entries - ret = ret[:500] + if len(ret) > MAX_RANKING_LIMIT: # limit to most recent X entries + ret = ret[:MAX_RANKING_LIMIT] # mylib.sort_by_name(ret, 1) mylib.json_write(fname_ranking_list(), ret, pretty=False) @@ -112,9 +152,9 @@ def process(bundle_ids, deleteOnly=False): index = mylib.json_safe_read(fname, {}) ids = mylib.appids_in_data(bundle_ids) - write_summary_index(index, ids, deleteOnly=deleteOnly) - write_ranking_list(index) - write_rank_index(index) + if update_summary_index(index, ids, deleteOnly=deleteOnly): + write_ranking_list(index, bundle_ids) + write_rank_index(index) print('')