diff --git a/src/README.md b/src/README.md index ba3f7ff..9ea28b0 100644 --- a/src/README.md +++ b/src/README.md @@ -1,4 +1,4 @@ -![dependency](z_dependency.png) +![dependency](z_dependency.svg) ## Structure @@ -39,17 +39,19 @@ If you are missing some icons run `main.py icons`. This should also download any Given A → B, B depends on A ``` -digraph G { +digraph Dependency { "." -> html_root "." -> bundle_download bundle_download -> index_app_names index_app_names -> html_bundle index_app_names -> html_index_apps index_app_names -> html_index_domains + index_meta -> html_index_domains "." -> bundle_combine + bundle_combine -> index_meta + bundle_combine -> html_bundle bundle_combine -> index_domains index_domains -> html_index_domains - bundle_combine -> html_bundle "." -> tracker_download } ``` diff --git a/src/html_index_domains.py b/src/html_index_domains.py index 477c5e6..a29620c 100755 --- a/src/html_index_domains.py +++ b/src/html_index_domains.py @@ -3,6 +3,7 @@ import common_lib as mylib import index_app_names import index_domains +import index_meta def a_app(bundle_id): @@ -115,19 +116,23 @@ def gen_html_lookup(html_dir, json, key, title): def gen_html_stats(c_apps, c_domains): + [c_recordings, c_logs] = index_meta.get_total_counts() title = 'Statistics' mylib.mkdir(mylib.path_out('stats')) with open(mylib.path_out('stats', 'index.html'), 'w') as fp: fp.write(mylib.template_with_base('''

{}

- The AppCheck database currently contains {} apps with a total of {} unique domains. + The AppCheck database currently contains {:,} apps with a total of {:,} unique domains. +

+

+ Collected through {:,} recordings with {:,} individual requests.

'''.format(title, c_apps, c_domains), title=title)) +'''.format(title, c_apps, c_domains, c_recordings, c_logs), title=title)) def process(): diff --git a/src/index_domains.py b/src/index_domains.py index b317e3c..a080b77 100755 --- a/src/index_domains.py +++ b/src/index_domains.py @@ -128,5 +128,5 @@ if __name__ == '__main__': if len(args) > 0: process(args) else: - process(['*'], deleteOnly=False) + # process(['*'], deleteOnly=False) mylib.usage(__file__, '[bundle_id] [...]') diff --git a/src/index_meta.py b/src/index_meta.py new file mode 100755 index 0000000..63085b6 --- /dev/null +++ b/src/index_meta.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import sys +import common_lib as mylib + + +def index_file(): + return mylib.path_data_index('meta.json') + + +def load_json_from_disk(fname): + return mylib.json_read(fname) if mylib.file_exists(fname) else {} + + +def get_total_counts(): + try: + return load_json_from_disk(index_file())['_'] + except KeyError: + return [0, 0] + + +def process(bundle_ids, deleteOnly=False): + print('writing index: meta ...') + fname = index_file() + if bundle_ids == ['*']: + bundle_ids = list(mylib.enum_data_appids()) + print(' full reset') + mylib.rm_file(fname) # rebuild from ground up + + # json format: `bundle-id : [#recordings, #logs, #domains, #subdomains]` + index = load_json_from_disk(fname) + for bid in bundle_ids: + # delete old value + try: + del(index[bid]) + except KeyError: + pass + if deleteOnly: + continue + # set new value + json = mylib.json_read(mylib.path_data_app(bid, 'evaluated.json')) + index[bid] = [json['sum_rec'], json['sum_logs'], + len(json['pardom']), len(json['subdom'])] + # sum of counts + try: + del(index['_']) + except KeyError: + pass + total = [0, 0] + for val in index.values(): + total[0] += val[0] + total[1] += val[1] + index['_'] = total + + # write json + mylib.json_write(fname, index, pretty=False) + print('') + + +if __name__ == '__main__': + args = sys.argv[1:] + if len(args) > 0: + process(args) + else: + # process(['*'], deleteOnly=False) + mylib.usage(__file__, '[bundle_id] [...]') diff --git a/src/main.py b/src/main.py index 9c9a259..5c544af 100755 --- a/src/main.py +++ b/src/main.py @@ -11,6 +11,7 @@ import html_bundle import html_index_domains import index_app_names import index_domains +import index_meta import tracker_download @@ -27,13 +28,14 @@ def print_usage_and_exit(): exit(0) -def rebuild_app_index(inclRoot=False): +def rebuild_app_index_html(inclRoot=False): html_index_apps.process() if inclRoot: # TODO: remove check if root contains dynamic content html_root.process() def rebuild_domain_index(bundle_ids, deleteOnly=False): + index_meta.process(bundle_ids, deleteOnly=deleteOnly) index_domains.process(bundle_ids, deleteOnly=deleteOnly) html_index_domains.process() @@ -60,7 +62,7 @@ def del_id(bundle_ids): print('') rebuild_domain_index(bundle_ids, deleteOnly=True) if update_app_index: - rebuild_app_index(inclRoot=True) + rebuild_app_index_html(inclRoot=True) def combine_and_update(bundle_ids, where=None): @@ -81,7 +83,7 @@ def combine_and_update(bundle_ids, where=None): print('no bundle affected by tracker, not generating bundle html') # 5. make all apps index if len(new_ids) > 0: - rebuild_app_index() # must be called after bundle_combine + rebuild_app_index_html() # must be called after bundle_combine else: print('no new bundle, not rebuilding index') @@ -131,10 +133,10 @@ try: # tracker_download.combine_all() elif cmd == 'icons': if bundle_download.download_missing_icons(force=False): - rebuild_app_index() + rebuild_app_index_html() elif cmd == 'index': rebuild_domain_index(['*']) - rebuild_app_index(inclRoot=True) + rebuild_app_index_html(inclRoot=True) elif cmd == 'run': if len(params) == 0: print_usage_and_exit() diff --git a/src/z_dependency.png b/src/z_dependency.png deleted file mode 100644 index e35e406..0000000 Binary files a/src/z_dependency.png and /dev/null differ diff --git a/src/z_dependency.svg b/src/z_dependency.svg new file mode 100644 index 0000000..d7bcaab --- /dev/null +++ b/src/z_dependency.svg @@ -0,0 +1,126 @@ + + +Dependency + + +. + +. + + +html_root + +html_root + + +.->html_root + + + + +bundle_download + +bundle_download + + +.->bundle_download + + + + +bundle_combine + +bundle_combine + + +.->bundle_combine + + + + +tracker_download + +tracker_download + + +.->tracker_download + + + + +index_app_names + +index_app_names + + +bundle_download->index_app_names + + + + +html_bundle + +html_bundle + + +index_app_names->html_bundle + + + + +html_index_apps + +html_index_apps + + +index_app_names->html_index_apps + + + + +html_index_domains + +html_index_domains + + +index_app_names->html_index_domains + + + + +index_meta + +index_meta + + +index_meta->html_index_domains + + + + +bundle_combine->html_bundle + + + + +bundle_combine->index_meta + + + + +index_domains + +index_domains + + +bundle_combine->index_domains + + + + +index_domains->html_index_domains + + + + + \ No newline at end of file