From 4aceb4a98a1051c069e151984d7bbe8c1f7170c0 Mon Sep 17 00:00:00 2001 From: relikd Date: Sun, 6 Sep 2020 21:57:42 +0200 Subject: [PATCH] Dot graph --- out/static/style.css | 80 ++++++++++++++++++----- src/bundle_combine.py | 4 ++ src/html_bundle.py | 148 +++++++++++++++++++++++++++--------------- src/main.py | 6 +- 4 files changed, 165 insertions(+), 73 deletions(-) diff --git a/out/static/style.css b/out/static/style.css index 8272833..cbb0e5f 100644 --- a/out/static/style.css +++ b/out/static/style.css @@ -29,9 +29,6 @@ header h1 { margin-top: 0; } header h1 span { font-size: 0.7em; color: silver; } -@media(max-width: 622px) { /* 3 columns */ - header h1 span { display: none; } -} main { padding: 0.1em 2em 1.5em; background: #fff; @@ -45,9 +42,6 @@ footer .col3 div { padding: 1%; display: inline-block; } -@media(max-width: 647px) { - footer .col3 div { width: 100%; padding: 0; } -} footer .links { text-align: center; font-size: 0.9em; @@ -60,6 +54,7 @@ footer .links a { color: #ddd; } td { padding: 0.2em 1em 0.2em 0.1em; } .squeeze { max-width: 700px; } +.wrap { word-wrap: anywhere; } #get-appcheck:hover { color: #586472; border-bottom: unset; } #get-appcheck img { width: 3em; height: 3em; margin: 0.3em; } @@ -84,22 +79,22 @@ td { padding: 0.2em 1em 0.2em 0.1em; } margin: 0.5em auto 1em; display: block; } -#app-toc img, #get-appcheck img, #appicon { +#app-toc img, #get-appcheck img, #meta img { border-radius: 21.5%; border: 0.7px solid #ccc; } -#app-toc span { font-size: 0.8em; } -#app-toc span.name { font-weight: bold; } +#app-toc span.name { font-size: 0.8em; font-weight: bold; } +#app-toc span.detail { font-size: 0.7em; } #pagination { text-align: center; margin-top: 2em; } #pagination a { margin: 0.5em; padding: 0.2em } #pagination a.active { border: 1pt solid black; border-radius: 0.2em; } -@media(min-width: 647px) { - #meta #appicon { float: right; } -} #meta td:nth-child(2) { font-weight: bold } -#connections i:not(.empty) { + +/* domain tags */ +.tags { margin: 2em 0; } +.tags i { font-size: 0.9em; font-style: normal; font-weight: normal; @@ -110,6 +105,59 @@ td { padding: 0.2em 1em 0.2em 0.1em; } display: inline-block; margin: 0.12em; } -#connections i.bad { border-color: red; } -#connections td { vertical-align: top; } -#connections figure { display: inline-block; } +.tags i.trckr:before, .tags.trckr i:before, p.trckr:before { + content: '* '; + color: red; + font-weight: bold; +} +.tags i.trckr, .tags.trckr i { background: #DBB; border-color: #C88; } +p.trckr { font-size: 0.8em; } + +/* graphs */ +.pie-chart { width: 100px; height: 100px; } +.dot-graph span { outline: 3px solid transparent; outline-offset: 2px; } +.dot-graph span:hover { outline-color: black; } +.dot-graph span:hover p { display: inline; } +.dot-graph p { + display: none; + position: absolute; + vertical-align: bottom; + background: white; + border: 1px solid #ddd; + padding: 0.2em; + margin: -2em 0; +} +.dot-graph i { + display: inline-block; + vertical-align: bottom; + width: 20px; + height: 20px; + padding: 0; + margin: 1px 1px; +} +/* color-bind friendly color palette */ +.c0{color:#63ACBE} .cb0{background:#63ACBE} +.c1{color:#601A4A} .cb1{background:#601A4A} +.c2{color:#09F4EC} .cb2{background:#09F4EC} +.c3{color:#1F77B4} .cb3{background:#1F77B4} +.c4{color:#EE442F} .cb4{background:#EE442F} +.c5{color:#7F7F7F} .cb5{background:#7F7F7F} +.c6{color:#0F2080} .cb6{background:#0F2080} +.c7{color:#3b9f35} .cb7{background:#3b9f35} +.c8{color:#F5793A} .cb8{background:#F5793A} +.c9{color:#AC66FB} .cb9{background:#AC66FB} + +.cs0{stroke:#3AE48C} +.cs1{stroke:#D11} + +/* responsive */ +@media(max-width: 647px) { + header h1 span { display: none; } /* header subtitle */ + footer .col3 div { width: 100%; padding: 0; } /* 3 columns */ + #meta .icons { margin-bottom: 1em; } /* icons beside each other */ + .pie-chart { float: right; } +} +@media(min-width: 648px) { + #meta .icons { float: right; } /* icons below each other */ + .pie-chart { margin-top: 1em; } +} diff --git a/src/bundle_combine.py b/src/bundle_combine.py index b5f9193..ee45723 100755 --- a/src/bundle_combine.py +++ b/src/bundle_combine.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import os import sys import common_lib as mylib import tracker_download as tracker @@ -39,7 +40,9 @@ def json_combine(bundle_id): domB = dict() # total sub domains domC = dict() # unique parent domains domD = dict() # total parent domains + latest = 0 for fname, jdata in mylib.enum_jsons(bundle_id): + latest = max(latest, os.path.getmtime(fname)) # or getctime res['name'] = jdata['app-name'] res['#rec'] += 1 dict_increment(res, 'rec-total', jdata['duration']) @@ -71,6 +74,7 @@ def json_combine(bundle_id): par_tracker[x] = tracker.is_tracker(x) res['tracker_subdom'] = sub_tracker res['tracker_pardom'] = par_tracker + res['last_date'] = latest return res diff --git a/src/html_bundle.py b/src/html_bundle.py index 196c660..f8abdf6 100755 --- a/src/html_bundle.py +++ b/src/html_bundle.py @@ -1,29 +1,9 @@ #!/usr/bin/env python3 import sys +import time +import math import common_lib as mylib -# import matplotlib -# import matplotlib.pyplot as plt - -# matplotlib.use('Agg') # disable interactive mode - - -def sort_dict(count_dict): - sorted_count = sorted(count_dict.items(), key=lambda x: (-x[1], x[0])) - names = ['{} ({})'.format(*x) for x in sorted_count] - sizes = [x[1] for x in sorted_count] - return names, sizes - - -def gen_graph(count_dict, outfile, overwrite=False): - if mylib.file_exists(outfile) and not overwrite: - return - # names, sizes = sort_dict(count_dict) - # pie1, _ = plt.pie(sizes, labels=names) - # plt.setp(pie1, width=0.5, edgecolor='white') - # plt.subplots_adjust(left=0, right=1, top=0.7, bottom=0.3) - # plt.savefig(outfile, bbox_inches='tight', pad_inches=0) # transparent=True - # plt.close() def seconds_to_time(seconds): @@ -32,14 +12,74 @@ def seconds_to_time(seconds): return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds) -def gen_dom_tags(unsorted_dict, trackers=None): +def gen_dom_tags(unsorted_dict, trackers=None, additionalClasses=None): sorted_arr = sorted(unsorted_dict, key=lambda x: (-x[1], x[0])) - res = [] - for x, y in sorted_arr: - clss = ' class="bad"' if trackers and trackers[x] else '' - title = x # if y == 1 else '{} ({})'.format(x, y) - res.append('{}'.format(clss, title)) - return ' '.join(res) if len(res) > 0 else '– None –' + txt = '' + anyMark = False + for i, (x, y) in enumerate(sorted_arr): + mark = trackers[x] if trackers else True + title = x if y == 1 else '{} ({})'.format(x, y) + txt += '{} '.format(' class="trckr"' if mark else '', title) + anyMark |= mark + if txt: + note = '

known tracker

' + return '
{}{}
'.format( + additionalClasses or '', txt, note if anyMark else '') + else: + return '– None –' + + +def gen_dotgraph(count_dict): + txt = '' + sorted_count = sorted(count_dict.items(), key=lambda x: (-x[1], x[0])) + for i, (name, count) in enumerate(sorted_count): + # TODO: use average not total count + txt += '

{0} ({1})

'.format(name, count) + for x in range(count): + txt += ''.format(i % 10) + txt += '' + return '
{}
'.format(txt) + + +def gen_pie_chart(parts, classes, stroke=0.6): + size = 1000 + stroke *= size * 0.5 + stroke_p = '{:.0f}'.format(stroke) + r = (0.99 * size - stroke) / 2 + r_p = '{:.0f},{:.0f}'.format(r, r) + mid = '{:.0f}'.format(size / 2) + + def arc(deg): + deg -= 90 + x = r * math.cos(math.pi * deg / 180) + y = r * math.sin(math.pi * deg / 180) + return '{:.0f},{:.0f}'.format(size / 2 + x, size / 2 + y) + + txt = '' + total = 0 + for i, x in enumerate(parts): + clss = classes[i % len(classes)] + deg = x * 360 + if x == 0: + continue + elif x == 1: + txt += f'' + else: + txt += f'' + total += deg + return '{1}'.format(size, txt) + + +def gen_radial_graph(obj): + total = 0 + tracker = 0 + for name, count in obj['total_subdom'].items(): + total += count + if obj['tracker_subdom'][name]: + tracker += count + percent = tracker / total + return '
{}
'.format( + gen_pie_chart([1 - percent, percent], ['cs0', 'cs1'])) def gen_html(bundle_id, obj): @@ -48,9 +88,12 @@ def gen_html(bundle_id, obj): return mylib.template_with_base(f'''

{obj['name']}

- +
+ + { gen_radial_graph(obj) } +
- +
Bundle-id:{ +
Bundle-id:{ bundle_id }
Number of recordings:{ @@ -65,45 +108,42 @@ def gen_html(bundle_id, obj):
Average recording time:{ round(obj['rec-total'] / obj['#rec'], 1) } s
Last updated:

Connections

-
- - - - -
Known Trackers ({ len(track_dom) }):{ - gen_dom_tags(track_dom) - }
Domains:{ - gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom']) - }
Subdomains:{ - gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom']) - }
-
-
+
+

Known Trackers ({ len(track_dom) }):

+ { gen_dom_tags(track_dom, additionalClasses=' trckr') } +

+ +

Domains:

+ { gen_dotgraph(obj['total_pardom']) } + { gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom']) } + +

Subdomains:

+ { gen_dotgraph(obj['total_subdom']) } + { gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom']) }
''', title=obj['name']) -def make_bundle_out(bundle_id, forceGraphs=False): +def make_bundle_out(bundle_id): json = mylib.json_read_combined(bundle_id) out_dir = mylib.path_out_app(bundle_id) needs_update_index = False if not mylib.dir_exists(out_dir): needs_update_index = True mylib.mkdir(out_dir) - - gen_graph(json['total_subdom'], mylib.path_add(out_dir, 'sub.svg'), - overwrite=forceGraphs) - gen_graph(json['total_pardom'], mylib.path_add(out_dir, 'par.svg'), - overwrite=forceGraphs) - with open(mylib.path_add(out_dir, 'index.html'), 'w') as fp: fp.write(gen_html(bundle_id, json)) return needs_update_index -def process(bundle_ids, forceGraphs=False): +def process(bundle_ids): print('generating html pages ...') if bundle_ids == ['*']: bundle_ids = list(mylib.enum_appids()) @@ -111,7 +151,7 @@ def process(bundle_ids, forceGraphs=False): ids_new_in_index = set() for bid in bundle_ids: print(' ' + bid) - if make_bundle_out(bid, forceGraphs=forceGraphs): + if make_bundle_out(bid): ids_new_in_index.add(bid) print('') return ids_new_in_index diff --git a/src/main.py b/src/main.py index dad00d1..baa73c8 100755 --- a/src/main.py +++ b/src/main.py @@ -40,12 +40,12 @@ def del_id(bundle_ids): html_index.process() -def combine_and_update(bundle_ids, where=None, forceGraphs=False): +def combine_and_update(bundle_ids, where=None): affected = bundle_combine.process(bundle_ids, where=where) if len(affected) == 0: print('no bundle affected by tracker, not generating bundle html') return - new_ids = html_bundle.process(affected, forceGraphs=forceGraphs) + new_ids = html_bundle.process(affected) if len(new_ids) == 0: print('no new bundle, not rebuilding index') return @@ -69,7 +69,7 @@ def import_update(): os.remove(fname) print('') if len(needs_update) > 0: - combine_and_update(needs_update, forceGraphs=True) + combine_and_update(needs_update) def tracker_update():