Improve asterisk handling

src/bundle_combine.py

@@ -113,10 +113,7 @@ def json_evaluate_inplace(obj):
 
 def process(bundle_ids):
     print('writing combined json ...')
-    if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_data_appids())
-
-    for bid in bundle_ids:
+    for bid in mylib.appids_in_data(bundle_ids):
         print('  ' + bid)
         obj = json_combine(bid)
         mylib.json_write(fname_combined(bid), obj, pretty=False)

src/common_lib.py

@@ -235,28 +235,38 @@ def download_file(url, path):
 
 # Enumerator
 
+_all_data_bundle_ids = None
+
+
 def enum_newly_added():
     for fname in glob.glob(path_data('_in', 'in_*')):
         yield fname, os.path.basename(fname)[3:]  # del prefix 'in_'
 
 
-def enum_appids():
-    for x in glob.glob(path_out_app('*')):
-        yield os.path.basename(x)
-
-
 def enum_jsons(bundle_id):
     for fname in glob.glob(path_data_app(bundle_id, 'id_*.json')):
         with open(fname, 'r') as fp:
             yield fname, json.load(fp)
 
 
-def enum_data_appids():
-    data_root = path_data()
-    prfx = path_len(data_root)
-    for path, dirs, files in os.walk(data_root):
-        if 'combined.json' in files:
-            yield path[prfx:].replace(os.sep, '.')
+def appids_in_out(selection=['*']):
+    if selection != ['*']:
+        return selection
+    return [os.path.basename(x) for x in glob.glob(path_out_app('*'))]
+
+
+def appids_in_data(selection=['*']):
+    if selection != ['*']:
+        return selection
+    global _all_data_bundle_ids
+    if not _all_data_bundle_ids:
+        _all_data_bundle_ids = []
+        data_root = path_data()
+        prfx = path_len(data_root)
+        for path, dirs, files in os.walk(data_root):
+            if 'combined.json' in files:
+                _all_data_bundle_ids.append(path[prfx:].replace(os.sep, '.'))
+    return _all_data_bundle_ids
 
 
 # JSON

src/download_itunes.py

@@ -77,7 +77,7 @@ def download_icon(bundle_id, force=False, langs=AVAILABLE_LANGS):
 
 def download_missing_icons(force=False, langs=AVAILABLE_LANGS):
     didAny = False
-    for bid in mylib.enum_appids():
+    for bid in mylib.appids_in_out():
         exists, _ = needs_icon_path(bid)
         if not exists:
             if not didAny:
@@ -123,11 +123,8 @@ def download(bundle_id, force=False):
 
 def process(bundle_ids, force=False):
     print('downloading bundle info ...')
-    if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_data_appids())
-
     newly_created = set()
-    for bid in bundle_ids:
+    for bid in mylib.appids_in_data(bundle_ids):
         if download(bid, force=force):
             newly_created.add(bid)
     print('')

src/html_bundle.py

@@ -157,13 +157,10 @@ def gen_html(bundle_id, obj):
 
 def process(bundle_ids):
     print('generating html: apps ...')
-    if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_appids())
-
-    for bid in bundle_ids:
+    for bid in mylib.appids_in_out(bundle_ids):
         print('  ' + bid)
-        json = bundle_combine.get_evaluated(bid)
         mylib.mkdir_out_app(bid)
+        json = bundle_combine.get_evaluated(bid)
         with open(mylib.path_out_app(bid, 'index.html'), 'w') as fp:
             fp.write(gen_html(bid, json))
         mylib.symlink(bundle_combine.fname_evaluated(bid),

src/html_index_apps.py

@@ -65,7 +65,7 @@ def process(per_page=60):
     mylib.rm_dir(index_dir)
     mylib.mkdir(index_dir)
 
-    apps = [gen_obj(x) for x in mylib.enum_appids()]
+    apps = [gen_obj(x) for x in mylib.appids_in_out()]
     apps_total = len(apps)
     pages_total, rest = divmod(apps_total, per_page)
     if rest > 0:

src/index_app_names.py

@@ -42,13 +42,12 @@ def get_name(bundle_id, langs=['us', 'de'], fallback='< App-Name >'):
 def process(bundle_ids):
     print('writing index: app names ...')
     if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_data_appids())
         print('  full reset')
         mylib.rm_file(index_fname())  # rebuild from ground up
 
     load_json_if_not_already()
     did_change = False
-    for bid in bundle_ids:
+    for bid in mylib.appids_in_data(bundle_ids):
         names = download_itunes.app_names(bid)
         if not names:
             mylib.err('index-app-names', 'could not load: {}'.format(bid))

src/index_domains.py

@@ -112,14 +112,14 @@ def process(bundle_ids, deleteOnly=False):
     print('writing index: domains ...')
     fname = fname_all()
     if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_data_appids())
         print('  full reset')
         mylib.rm_file(fname)  # rebuild from ground up
 
     index = load_json_from_disk(fname)
-    did_change = delete_from_index(index, bundle_ids, deleteOnly=deleteOnly)
+    ids = mylib.appids_in_data(bundle_ids)
+    did_change = delete_from_index(index, ids, deleteOnly=deleteOnly)
     if not deleteOnly:
-        did_change |= insert_in_index(index, bundle_ids)
+        did_change |= insert_in_index(index, ids)
     if did_change:
         mylib.json_write(fname, index, pretty=False)
         filter_tracker_only(index)

src/index_meta.py

@@ -119,13 +119,14 @@ def get_rank(bundle_id):
 
 def process(bundle_ids, deleteOnly=False):
     print('writing index: meta ...')
+    fname = fname_app_summary()
     if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_data_appids())
         print('  full reset')
-        mylib.rm_file(fname_app_summary())  # rebuild from ground up
+        mylib.rm_file(fname)  # rebuild from ground up
 
-    index = load_json_from_disk(fname_app_summary())
-    write_summary_index(index, bundle_ids, deleteOnly=deleteOnly)
+    index = load_json_from_disk(fname)
+    ids = mylib.appids_in_data(bundle_ids)
+    write_summary_index(index, ids, deleteOnly=deleteOnly)
     write_rank_index(index)
     print('')
 

src/main.py

@@ -41,11 +41,8 @@ def rebuild_domain_index(bundle_ids, deleteOnly=False):
 
 def del_id(bundle_ids):
     print('removing apps from website:')
-    if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_appids())
-
     update_app_index = False
-    for bid in bundle_ids:
+    for bid in mylib.appids_in_out(bundle_ids):
         dest = mylib.path_out_app(bid)
         if mylib.dir_exists(dest):
             print('  ' + bid)