From d3820b678524ccaa89a6dc13517803efefc52bd5 Mon Sep 17 00:00:00 2001 From: relikd Date: Tue, 8 Sep 2020 16:22:54 +0200 Subject: [PATCH] Average counts and threshold processing --- out/static/style.css | 8 +++-- src/html_bundle.py | 73 ++++++++++++++++++++++++++------------------ templates/base.html | 12 +++++--- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/out/static/style.css b/out/static/style.css index d748268..63967b7 100644 --- a/out/static/style.css +++ b/out/static/style.css @@ -54,9 +54,7 @@ footer .links a { color: #ddd; } #main-nav img { height: 1.2em; margin: 0 -0.4em; } #main-nav img:hover { transform: scale(1.2); } -td { padding: 0.2em 1em 0.2em 0.1em; } .squeeze { max-width: 700px; } -.wrap { word-break: break-all; } #get-appcheck:hover { color: #586472; } #get-appcheck img { width: 3em; height: 3em; margin: 0.3em; } @@ -92,6 +90,12 @@ td { padding: 0.2em 1em 0.2em 0.1em; } #pagination a { margin: 0.5em; padding: 0.2em } #pagination a.active { border: 1pt solid black; border-radius: 0.2em; } +/* app bundle */ +h2.title { margin-bottom: 0; } +p.subtitle { margin-top: 0.2em; } +.mg_lr { margin: 0 0.4em; } +.snd { color: #586472; font-size: 0.85em; } +td { padding: 0.2em 1em 0.2em 0.1em; } #meta td:nth-child(2) { font-weight: bold } /* domain tags */ diff --git a/src/html_bundle.py b/src/html_bundle.py index bbbfb97..ccac874 100755 --- a/src/html_bundle.py +++ b/src/html_bundle.py @@ -5,6 +5,9 @@ import time import math import common_lib as mylib +THRESHOLD_PERCENT_OF_LOGS = 0.7 # domain appears in % recordings +THRESHOLD_MIN_AVG_LOGS = 1.0 # at least x times in total (after %-thresh) + def seconds_to_time(seconds): minutes, seconds = divmod(seconds, 60) @@ -54,8 +57,7 @@ def gen_pie_chart(parts, classes, stroke=0.6): return '{1}'.format(size, txt) -def gen_radial_graph(obj): - percent = obj['#logs_tracker'] / (obj['#logs_total'] or 1) +def gen_radial_graph(percent): return '
{}
'.format( gen_pie_chart([1 - percent, percent], ['cs0', 'cs1'])) @@ -77,59 +79,70 @@ def gen_dom_tags(sorted_arr, onlyTrackers=False): def prepare_json(obj): - def calc_sum(arr): - # TODO: use average or median, not total count - return sum(arr) + if not obj['name']: + obj['name'] = '< App-Name >' + rec_count = len(obj['rec_len']) + time_total = sum(obj['rec_len']) + obj['sum_rec'] = rec_count + obj['sum_logs'] = sum([sum(x[1]) for x in obj['pardom'].values()]) + obj['sum_logs_pm'] = obj['sum_logs'] / (time_total or 1) * 60 + obj['sum_time'] = time_total + obj['avg_time'] = time_total / rec_count def transform(ddic): res = list() for name, (is_tracker, counts) in ddic.items(): - res.append([name, calc_sum(counts), is_tracker]) + rec_percent = len(counts) / rec_count + if rec_percent < THRESHOLD_PERCENT_OF_LOGS: + continue + avg = sum(counts) / rec_count # len(counts) + if avg < THRESHOLD_MIN_AVG_LOGS: + continue + res.append([name, round(avg + 0.001), is_tracker]) res.sort(key=lambda x: (-x[1], x[0])) # sort by count desc, then name return res - if not obj['name']: - obj['name'] = '< App-Name >' - obj['#rec'] = len(obj['rec_len']) - obj['rec_len'] = sum(obj['rec_len']) obj['pardom'] = transform(obj['pardom']) obj['subdom'] = transform(obj['subdom']) # do this after the transformation: + c_tracker = 0 + c_total = 0 + for _, c, flag in obj['subdom']: + c_tracker += c if flag else 0 + c_total += c + obj['tracker_percent'] = c_tracker / (c_total or 1) obj['tracker'] = list(filter(lambda x: x[2], obj['subdom'])) - obj['#logs_total'] = sum(map(lambda x: x[1], obj['pardom'])) - obj['#logs_tracker'] = sum(map(lambda x: x[1], obj['tracker'])) + obj['avg_logs'] = c_total + obj['avg_logs_pm'] = c_total / (obj['avg_time'] or 1) * 60 def gen_html(bundle_id, obj): prepare_json(obj) return mylib.template_with_base(f''' -

{obj['name']}

+

{obj['name']}

+

Bundle-id:{ bundle_id }

- { gen_radial_graph(obj) } + { gen_radial_graph(obj['tracker_percent']) }
- - - - - - + + + + +
Bundle-id:{ - bundle_id - }
Number of recordings:{ - obj['#rec'] - }
Total number of logs:{ - obj['#logs_total'] - }
Cumulative recording time:{ - seconds_to_time(obj['rec_len']) - }
Average recording time:{ - round(obj['rec_len'] / obj['#rec'], 1) - } s
Last updated:
Number of recordings:{ obj['sum_rec'] }
Total number of logs:{ + obj['sum_logs'] }({ + round(obj['sum_logs_pm'], 1)} / min)
Average number of logs:{ + obj['avg_logs'] }({ + round(obj['avg_logs_pm'], 1)} / min)
Average recording time:{ + round(obj['avg_time'], 1) } sec
Cumulative recording time:{ + seconds_to_time(obj['sum_time']) }

Connections

diff --git a/templates/base.html b/templates/base.html index 74831e0..26ac4f9 100644 --- a/templates/base.html +++ b/templates/base.html @@ -19,11 +19,13 @@
- +

logo AppCheck – Privacy Monitor