diff --git a/src/README.md b/src/README.md
index c618a10..7e7af21 100644
--- a/src/README.md
+++ b/src/README.md
@@ -43,9 +43,12 @@ digraph Dependency {
"." -> html_root
"." -> download_itunes
download_itunes -> index_app_names
+ download_itunes -> index_categories
+ index_categories -> html_categories
index_app_names -> html_bundle
index_app_names -> html_index_apps
index_app_names -> html_index_domains
+ index_app_names -> html_categories
index_meta -> html_index_domains
"." -> bundle_combine
bundle_combine -> index_meta
diff --git a/src/download_itunes.py b/src/download_itunes.py
index d5ad19e..18416a7 100755
--- a/src/download_itunes.py
+++ b/src/download_itunes.py
@@ -69,18 +69,18 @@ def download_icon(bundle_id, force=False, langs=AVAILABLE_LANGS):
def download_missing_icons(force=False, langs=AVAILABLE_LANGS):
- didAny = False
+ affected = []
for bid in mylib.appids_in_out():
exists, _ = needs_icon_path(bid)
if not exists:
- if not didAny:
+ if not affected:
print('downloading missing icons ...')
- didAny = True
+ affected.append(bid)
print(' ' + bid)
download_icon(bid, force=force, langs=langs)
- if didAny:
+ if affected:
print('')
- return didAny
+ return affected
def download(bundle_id, force=False):
diff --git a/src/html_bundle.py b/src/html_bundle.py
index 4904af2..3c14b81 100755
--- a/src/html_bundle.py
+++ b/src/html_bundle.py
@@ -111,11 +111,16 @@ def gen_page(bundle_id, obj):
def process(bundle_ids):
print('generating html: apps ...')
+ i = 0
for bid in mylib.appids_in_out(bundle_ids):
- # print(' ' + bid)
gen_page(bid, bundle_combine.get_evaluated(bid))
mylib.symlink(bundle_combine.fname_evaluated(bid),
mylib.path_out_app(bid, 'data.json'))
+ mylib.printf(' .' if i == 0 else '.')
+ i = (i + 1) % 50
+ if i == 0:
+ print('') # close printf
+ print('') # close printf
print('')
diff --git a/src/html_categories.py b/src/html_categories.py
index 291ec6e..06b2aa8 100755
--- a/src/html_categories.py
+++ b/src/html_categories.py
@@ -5,7 +5,7 @@ import lib_html as HTML
import index_categories # enum_all_categories
-def process(per_page=60):
+def process(affected=None, per_page=60):
print('generating html: category-index ...')
base = mylib.path_out('category')
parent = 'All Categories'
@@ -13,11 +13,14 @@ def process(per_page=60):
for cid, cat, apps in sorted(index_categories.enum_all_categories(),
key=lambda x: x[1].lower()):
arr.append((cid, cat))
+ if affected and cid not in affected:
+ continue
pre = HTML.h2(HTML.a_path([(parent, '../')], cat))
_, a = HTML.write_app_pages(mylib.path_add(base, cid), apps, cat,
per_page, pre=pre)
print(' {} ({})'.format(cat, a))
+ print(' .. {} categories'.format(len(arr)))
src = ''.join([HTML.a(n, '{}/'.format(cid)) for cid, n in arr])
HTML.write(base, '''
{}
diff --git a/src/index_categories.py b/src/index_categories.py
index 4bc347a..27b266a 100755
--- a/src/index_categories.py
+++ b/src/index_categories.py
@@ -58,15 +58,6 @@ def reset_index():
_dict_apps = None
-def try_persist_changes(flag_apps, flag_names):
- if flag_apps:
- print(' write app-index')
- mylib.json_write(fname_app_categories(), _dict_apps, pretty=False)
- if flag_names:
- print(' write name-index')
- mylib.json_write(fname_category_names(), _dict_names, pretty=False)
-
-
def get_categories(bundle_id):
load_json_if_not_already()
try:
@@ -110,8 +101,8 @@ def process(bundle_ids, force=False):
reset_index()
load_json_if_not_already()
- write_app_index = False
write_name_index = False
+ write_app_index = False
for bid in mylib.appids_in_data(bundle_ids):
genre_ids = []
for lang, gid, gname in download_itunes.enum_genres(bid):
@@ -122,7 +113,12 @@ def process(bundle_ids, force=False):
if try_update_app(bid, genre_ids):
write_app_index = True
- try_persist_changes(write_app_index, write_name_index)
+ if write_name_index:
+ print(' write name-index')
+ mylib.json_write(fname_category_names(), _dict_names, pretty=False)
+ if write_app_index:
+ print(' write app-index')
+ mylib.json_write(fname_app_categories(), _dict_apps, pretty=False)
print('')
@@ -131,5 +127,5 @@ if __name__ == '__main__':
if len(args) > 0:
process(args)
else:
- # process(['*'])
+ # process(['*'], force=True)
mylib.usage(__file__, '[bundle_id] [...]')
diff --git a/src/main.py b/src/main.py
index 20c14d6..8f9cffb 100755
--- a/src/main.py
+++ b/src/main.py
@@ -7,10 +7,12 @@ import bundle_combine
import download_itunes
import download_tracker
import html_bundle
+import html_categories
import html_index_apps
import html_index_domains
import html_root
import index_app_names
+import index_categories
import index_domains
import index_meta
@@ -28,52 +30,53 @@ def print_usage_and_exit():
exit(0)
-def rebuild_app_index_html(inclRoot=False):
- html_index_apps.process()
+def rebuild_html(bundle_ids=None, cat_ids=None, inclIApp=True, inclRoot=False):
+ # all of these must happen after index_app_names
+ if bundle_ids:
+ html_bundle.process(bundle_ids) # after index_meta
+ html_categories.process(affected=cat_ids) # after index_categories
+ html_index_domains.process() # after index_domains
+ if inclIApp:
+ html_index_apps.process()
+ else:
+ print('no new bundle, not rebuilding index')
if inclRoot: # TODO: remove check if root contains dynamic content
html_root.process()
-def rebuild_domain_index(bundle_ids, deleteOnly=False):
- index_domains.process(bundle_ids, deleteOnly=deleteOnly)
- html_index_domains.process()
-
-
def del_id(bundle_ids):
+ def delete_from_all_indices(bundle_ids):
+ index_meta.process(bundle_ids, deleteOnly=True)
+ index_domains.process(bundle_ids, deleteOnly=True)
+ index_app_names.process(bundle_ids, deleteOnly=True)
+
print('removing apps from website:')
- update_app_index = False
for bid in mylib.appids_in_out(bundle_ids):
dest = mylib.path_out_app(bid)
if mylib.dir_exists(dest):
print(' ' + bid)
mylib.rm_dir(dest)
- update_app_index = True
print('')
- index_meta.process(bundle_ids, deleteOnly=True)
- rebuild_domain_index(bundle_ids, deleteOnly=True)
- if update_app_index:
- rebuild_app_index_html(inclRoot=True)
+ delete_from_all_indices(bundle_ids)
+ rebuild_html()
def combine_and_update(bundle_ids):
# 1. download meta data from iTunes store, incl. app icons
new_ids = download_itunes.process(bundle_ids)
- # 2. if new apps, update bundle name index
+ # 2. if new apps, update bundle name index & categories
if bundle_ids == ['*']:
new_ids = ['*'] # special case needed to force rebuilt index
if len(new_ids) > 0:
index_app_names.process(new_ids) # after download_itunes
- # 3. re-calculate combined.json and evaluated.json files
+ index_categories.process(new_ids) # after download_itunes
+ # 3. re-calculate combined.json
bundle_combine.process(bundle_ids)
- # 4. make html and update domain index
+ # 4. re-build indices
index_meta.process(bundle_ids) # after bundle_combine
- html_bundle.process(bundle_ids) # after index_app_names
- rebuild_domain_index(bundle_ids) # after bundle_combine
- # 5. make all apps index
- if len(new_ids) > 0:
- rebuild_app_index_html() # after bundle_combine
- else:
- print('no new bundle, not rebuilding index')
+ index_domains.process(bundle_ids) # after bundle_combine
+ # 5. make all html files
+ rebuild_html(bundle_ids, inclIApp=len(new_ids) > 0)
def import_update():
@@ -123,12 +126,13 @@ try:
tracker_update()
# download_tracker.combine_all()
elif cmd == 'icons':
- if download_itunes.download_missing_icons(force=False):
- rebuild_app_index_html()
+ bundle_ids = download_itunes.download_missing_icons(force=False)
+ if bundle_ids:
+ rebuild_html(bundle_ids)
elif cmd == 'index':
index_meta.process(['*'])
- rebuild_domain_index(['*'])
- rebuild_app_index_html(inclRoot=True)
+ index_domains.process(['*'])
+ rebuild_html(inclRoot=True)
elif cmd == 'run':
if len(params) == 0:
print_usage_and_exit()
diff --git a/src/z_dependency.svg b/src/z_dependency.svg
index ccda6ad..b089fb0 100644
--- a/src/z_dependency.svg
+++ b/src/z_dependency.svg
@@ -1,126 +1,151 @@
-