diff --git a/out/static/style.css b/out/static/style.css index 5b1e845..2d6621d 100644 --- a/out/static/style.css +++ b/out/static/style.css @@ -230,11 +230,8 @@ p.trckr { font-size: 0.9em; margin-left: 0.5em; } width: 40%; margin-left: 1%; } - #dom-top10 { - margin: 0 auto; - width: max-content; - max-width: 100%; - text-align: right; - } + #dom-top10 { text-align: right; } + #dom-top10 p { margin: 0.4em; } + .div-center { margin: 0 auto; width: max-content; max-width: 100%; } .loadbar { display: inline-block; } } diff --git a/src/README.md b/src/README.md index d589c6b..ba3f7ff 100644 --- a/src/README.md +++ b/src/README.md @@ -42,13 +42,13 @@ Given A → B, B depends on A digraph G { "." -> html_root "." -> bundle_download - bundle_download -> index_bundle_names - index_bundle_names -> html_bundle - index_bundle_names -> html_index - index_bundle_names -> html_reverse_domains + bundle_download -> index_app_names + index_app_names -> html_bundle + index_app_names -> html_index_apps + index_app_names -> html_index_domains "." -> bundle_combine - bundle_combine -> index_reverse_domains - index_reverse_domains -> html_reverse_domains + bundle_combine -> index_domains + index_domains -> html_index_domains bundle_combine -> html_bundle "." -> tracker_download } diff --git a/src/html_bundle.py b/src/html_bundle.py index 00a49a5..d7e8696 100755 --- a/src/html_bundle.py +++ b/src/html_bundle.py @@ -4,7 +4,7 @@ import sys import time import math import common_lib as mylib -import index_bundle_names +import index_app_names def seconds_to_time(seconds): @@ -81,7 +81,7 @@ def gen_dom_tags(sorted_arr, isSub, onlyTrackers=False): def gen_html(bundle_id, obj): - name = index_bundle_names.get_name(bundle_id) + name = index_app_names.get_name(bundle_id) obj['tracker'] = list(filter(lambda x: x[2], obj['subdom'])) return mylib.template_with_base(f'''

{name}

diff --git a/src/html_index.py b/src/html_index_apps.py similarity index 96% rename from src/html_index.py rename to src/html_index_apps.py index 8e04d01..660973b 100755 --- a/src/html_index.py +++ b/src/html_index_apps.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import common_lib as mylib -import index_bundle_names +import index_app_names def gen_obj(bundle_id): @@ -11,7 +11,7 @@ def gen_obj(bundle_id): icon = '/static/app-template.svg' return { 'id': bundle_id, - 'name': index_bundle_names.get_name(bundle_id), + 'name': index_app_names.get_name(bundle_id), 'img': icon } diff --git a/src/html_reverse_domains.py b/src/html_index_domains.py similarity index 76% rename from src/html_reverse_domains.py rename to src/html_index_domains.py index 2272ac4..98aee0d 100755 --- a/src/html_reverse_domains.py +++ b/src/html_index_domains.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import common_lib as mylib -import index_bundle_names -import index_reverse_domains +import index_app_names +import index_domains def a_app(bundle_id): return '{}'.format( - bundle_id, index_bundle_names.get_name(bundle_id)) + bundle_id, index_app_names.get_name(bundle_id)) def a_dom(domain, key): @@ -61,13 +61,15 @@ def gen_html_top_10(subset, fname, total, title): with open(fname, 'w') as fp: txt = f''' -
-

{ title }

''' +
+

{ title }

+
''' for dom, ids in subset: dom_str = div_dom(dom, len(ids), 'domain') pct_bar = div_loadbar(round(len(ids) / total * 100)) txt += f'\n

{dom_str} {pct_bar}

' fp.write(mylib.template_with_base(txt + ''' +

Get full list sorted by Occurrence frequency or in Alphabetical order.

@@ -96,7 +98,7 @@ def gen_html_trinity(json, idx_dir, app_count, title): def gen_html_lookup(html_dir, json, key, title): mylib.mkdir(html_dir) - names = [[x, index_bundle_names.get_name(x)] for x in json['bundle']] + names = [[x, index_app_names.get_name(x)] for x in json['bundle']] mylib.json_write(mylib.path_add(html_dir, 'apps.json'), names) mylib.json_write(mylib.path_add(html_dir, 'doms.json'), json[key]) with open(mylib.path_add(html_dir, 'index.html'), 'w') as fp: @@ -112,37 +114,58 @@ def gen_html_lookup(html_dir, json, key, title): ''', title=title)) +def gen_html_stats(c_apps, c_domains): + title = 'Statistics' + mylib.mkdir(mylib.path_out('stats')) + with open(mylib.path_out('stats', 'index.html'), 'w') as fp: + fp.write(mylib.template_with_base(''' +

{}

+

+ The AppCheck database currently contains {} apps with a total of {} unique domains. +

+'''.format(title, c_apps, c_domains), title=title)) + + def process(): # bundle_combine assures domain name is [a-zA-Z0-9.-] - print('generating reverse-domain-index ...') - + print('generating domain-index ...') # Data export all_dom_dir = mylib.path_out('index', 'domains', 'all') trkr_dir = mylib.path_out('index', 'domains', 'tracker') mylib.mkdir(all_dom_dir) mylib.mkdir(trkr_dir) - mylib.symlink(index_reverse_domains.fname_all(), + mylib.symlink(index_domains.fname_all(), mylib.path_out_app(all_dom_dir, 'data.json')) - mylib.symlink(index_reverse_domains.fname_tracker(), + mylib.symlink(index_domains.fname_tracker(), mylib.path_out_app(trkr_dir, 'data.json')) - # Load - json = index_reverse_domains.load() - app_count = index_reverse_domains.number_of_apps(json) - # Lookup + json = index_domains.load() + app_count = index_domains.number_of_apps(json) + dom_count = len(json['subdom']) + + print(' Lookup') gen_html_lookup(mylib.path_out('domain'), json, 'pardom', title='Domain Lookup') gen_html_lookup(mylib.path_out('subdomain'), json, 'subdom', title='Subdomain Lookup') - # All domains - index_reverse_domains.enrich_with_bundle_ids(json) + + print(' All Domains') + index_domains.enrich_with_bundle_ids(json) gen_html_trinity(json, all_dom_dir, app_count, title='Requested Domains') - # Tacker only - json = index_reverse_domains.load(tracker=True) - index_reverse_domains.enrich_with_bundle_ids(json) + + print(' Trackers Only') + json = index_domains.load(tracker=True) + index_domains.enrich_with_bundle_ids(json) gen_html_trinity(json, trkr_dir, app_count, title='Tracker') + # Stats + print(' Stats') + gen_html_stats(app_count, dom_count) print('') diff --git a/src/index_bundle_names.py b/src/index_app_names.py similarity index 100% rename from src/index_bundle_names.py rename to src/index_app_names.py diff --git a/src/index_reverse_domains.py b/src/index_domains.py similarity index 100% rename from src/index_reverse_domains.py rename to src/index_domains.py diff --git a/src/main.py b/src/main.py index 811b22b..62c1036 100755 --- a/src/main.py +++ b/src/main.py @@ -6,11 +6,11 @@ import common_lib as mylib import bundle_combine import bundle_download import html_root -import html_index +import html_index_apps import html_bundle -import html_reverse_domains -import index_bundle_names -import index_reverse_domains +import html_index_domains +import index_app_names +import index_domains import tracker_download @@ -28,21 +28,21 @@ def print_usage_and_exit(): def rebuild_app_index(inclRoot=False): - html_index.process() + html_index_apps.process() if inclRoot: # TODO: remove check if root contains dynamic content html_root.process() def rebuild_domain_index(bundle_ids, deleteOnly=False): - index_reverse_domains.process(bundle_ids, deleteOnly=deleteOnly) - html_reverse_domains.process() + index_domains.process(bundle_ids, deleteOnly=deleteOnly) + html_index_domains.process() def rebuild_name_index(new_ids): - if index_bundle_names.missing(): - index_bundle_names.process(['*']) + if index_app_names.missing(): + index_app_names.process(['*']) elif len(new_ids) > 0: - index_bundle_names.process(new_ids) # after bundle_download + index_app_names.process(new_ids) # after bundle_download def del_id(bundle_ids): @@ -76,7 +76,7 @@ def combine_and_update(bundle_ids, where=None): # 4. was any json updated? if so, make html and update reverse index if len(affected) > 0: rebuild_domain_index(affected) # after bundle_combine - html_bundle.process(affected) # after index_bundle_names + html_bundle.process(affected) # after index_app_names else: print('no bundle affected by tracker, not generating bundle html') # 5. make all apps index diff --git a/src/z_dependency.png b/src/z_dependency.png index a1ce1b6..e35e406 100644 Binary files a/src/z_dependency.png and b/src/z_dependency.png differ diff --git a/templates/base.html b/templates/base.html index 7dc9b82..b66fc6b 100644 --- a/templates/base.html +++ b/templates/base.html @@ -3,9 +3,9 @@ - + #_TITLE_#AppCheck: Privacy Monitor - + @@ -22,7 +22,7 @@