diff --git a/src/bundle_combine.py b/src/bundle_combine.py index 41674fa..005a6a6 100755 --- a/src/bundle_combine.py +++ b/src/bundle_combine.py @@ -113,10 +113,7 @@ def json_evaluate_inplace(obj): def process(bundle_ids): print('writing combined json ...') - if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_data_appids()) - - for bid in bundle_ids: + for bid in mylib.appids_in_data(bundle_ids): print(' ' + bid) obj = json_combine(bid) mylib.json_write(fname_combined(bid), obj, pretty=False) diff --git a/src/common_lib.py b/src/common_lib.py index 6d3ebf2..0e40e3e 100755 --- a/src/common_lib.py +++ b/src/common_lib.py @@ -235,28 +235,38 @@ def download_file(url, path): # Enumerator +_all_data_bundle_ids = None + + def enum_newly_added(): for fname in glob.glob(path_data('_in', 'in_*')): yield fname, os.path.basename(fname)[3:] # del prefix 'in_' -def enum_appids(): - for x in glob.glob(path_out_app('*')): - yield os.path.basename(x) - - def enum_jsons(bundle_id): for fname in glob.glob(path_data_app(bundle_id, 'id_*.json')): with open(fname, 'r') as fp: yield fname, json.load(fp) -def enum_data_appids(): - data_root = path_data() - prfx = path_len(data_root) - for path, dirs, files in os.walk(data_root): - if 'combined.json' in files: - yield path[prfx:].replace(os.sep, '.') +def appids_in_out(selection=['*']): + if selection != ['*']: + return selection + return [os.path.basename(x) for x in glob.glob(path_out_app('*'))] + + +def appids_in_data(selection=['*']): + if selection != ['*']: + return selection + global _all_data_bundle_ids + if not _all_data_bundle_ids: + _all_data_bundle_ids = [] + data_root = path_data() + prfx = path_len(data_root) + for path, dirs, files in os.walk(data_root): + if 'combined.json' in files: + _all_data_bundle_ids.append(path[prfx:].replace(os.sep, '.')) + return _all_data_bundle_ids # JSON diff --git a/src/download_itunes.py b/src/download_itunes.py index fb2698d..9b4a9bf 100755 --- a/src/download_itunes.py +++ b/src/download_itunes.py @@ -77,7 +77,7 @@ def download_icon(bundle_id, force=False, langs=AVAILABLE_LANGS): def download_missing_icons(force=False, langs=AVAILABLE_LANGS): didAny = False - for bid in mylib.enum_appids(): + for bid in mylib.appids_in_out(): exists, _ = needs_icon_path(bid) if not exists: if not didAny: @@ -123,11 +123,8 @@ def download(bundle_id, force=False): def process(bundle_ids, force=False): print('downloading bundle info ...') - if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_data_appids()) - newly_created = set() - for bid in bundle_ids: + for bid in mylib.appids_in_data(bundle_ids): if download(bid, force=force): newly_created.add(bid) print('') diff --git a/src/html_bundle.py b/src/html_bundle.py index 8dd7155..5561224 100755 --- a/src/html_bundle.py +++ b/src/html_bundle.py @@ -157,13 +157,10 @@ def gen_html(bundle_id, obj): def process(bundle_ids): print('generating html: apps ...') - if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_appids()) - - for bid in bundle_ids: + for bid in mylib.appids_in_out(bundle_ids): print(' ' + bid) - json = bundle_combine.get_evaluated(bid) mylib.mkdir_out_app(bid) + json = bundle_combine.get_evaluated(bid) with open(mylib.path_out_app(bid, 'index.html'), 'w') as fp: fp.write(gen_html(bid, json)) mylib.symlink(bundle_combine.fname_evaluated(bid), diff --git a/src/html_index_apps.py b/src/html_index_apps.py index 3835c03..406481c 100755 --- a/src/html_index_apps.py +++ b/src/html_index_apps.py @@ -65,7 +65,7 @@ def process(per_page=60): mylib.rm_dir(index_dir) mylib.mkdir(index_dir) - apps = [gen_obj(x) for x in mylib.enum_appids()] + apps = [gen_obj(x) for x in mylib.appids_in_out()] apps_total = len(apps) pages_total, rest = divmod(apps_total, per_page) if rest > 0: diff --git a/src/index_app_names.py b/src/index_app_names.py index 1854e9c..67ffc35 100755 --- a/src/index_app_names.py +++ b/src/index_app_names.py @@ -42,13 +42,12 @@ def get_name(bundle_id, langs=['us', 'de'], fallback='< App-Name >'): def process(bundle_ids): print('writing index: app names ...') if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_data_appids()) print(' full reset') mylib.rm_file(index_fname()) # rebuild from ground up load_json_if_not_already() did_change = False - for bid in bundle_ids: + for bid in mylib.appids_in_data(bundle_ids): names = download_itunes.app_names(bid) if not names: mylib.err('index-app-names', 'could not load: {}'.format(bid)) diff --git a/src/index_domains.py b/src/index_domains.py index 547c604..755460a 100755 --- a/src/index_domains.py +++ b/src/index_domains.py @@ -112,14 +112,14 @@ def process(bundle_ids, deleteOnly=False): print('writing index: domains ...') fname = fname_all() if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_data_appids()) print(' full reset') mylib.rm_file(fname) # rebuild from ground up index = load_json_from_disk(fname) - did_change = delete_from_index(index, bundle_ids, deleteOnly=deleteOnly) + ids = mylib.appids_in_data(bundle_ids) + did_change = delete_from_index(index, ids, deleteOnly=deleteOnly) if not deleteOnly: - did_change |= insert_in_index(index, bundle_ids) + did_change |= insert_in_index(index, ids) if did_change: mylib.json_write(fname, index, pretty=False) filter_tracker_only(index) diff --git a/src/index_meta.py b/src/index_meta.py index 3de5da6..72aef58 100755 --- a/src/index_meta.py +++ b/src/index_meta.py @@ -119,13 +119,14 @@ def get_rank(bundle_id): def process(bundle_ids, deleteOnly=False): print('writing index: meta ...') + fname = fname_app_summary() if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_data_appids()) print(' full reset') - mylib.rm_file(fname_app_summary()) # rebuild from ground up + mylib.rm_file(fname) # rebuild from ground up - index = load_json_from_disk(fname_app_summary()) - write_summary_index(index, bundle_ids, deleteOnly=deleteOnly) + index = load_json_from_disk(fname) + ids = mylib.appids_in_data(bundle_ids) + write_summary_index(index, ids, deleteOnly=deleteOnly) write_rank_index(index) print('') diff --git a/src/main.py b/src/main.py index c19db99..20c14d6 100755 --- a/src/main.py +++ b/src/main.py @@ -41,11 +41,8 @@ def rebuild_domain_index(bundle_ids, deleteOnly=False): def del_id(bundle_ids): print('removing apps from website:') - if bundle_ids == ['*']: - bundle_ids = list(mylib.enum_appids()) - update_app_index = False - for bid in bundle_ids: + for bid in mylib.appids_in_out(bundle_ids): dest = mylib.path_out_app(bid) if mylib.dir_exists(dest): print(' ' + bid)