#!/usr/bin/env python3 import sys import time import math import common_lib as mylib THRESHOLD_PERCENT_OF_LOGS = 0.7 # domain appears in % recordings THRESHOLD_MIN_AVG_LOGS = 1.0 # at least x times in total (after %-thresh) def seconds_to_time(seconds): seconds = int(seconds) minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds) def gen_dotgraph(sorted_arr): txt = '' for i, (name, count, mark) in enumerate(sorted_arr): title = '{} ({})'.format(name, count) if count > 1 else name clss = 'cb{}'.format(i % 10) if mark: clss += ' trckr' txt += '

{1}

'.format(clss, title) txt += '' * count txt += '
' return '
{}
'.format(txt) def gen_pie_chart(parts, classes, stroke=0.6): size = 1000 stroke *= size * 0.5 stroke_p = '{:.0f}'.format(stroke) r = (0.99 * size - stroke) / 2 r_p = '{:.0f},{:.0f}'.format(r, r) mid = '{:.0f}'.format(size / 2) def arc(deg): deg -= 90 x = r * math.cos(math.pi * deg / 180) y = r * math.sin(math.pi * deg / 180) return '{:.0f},{:.0f}'.format(size / 2 + x, size / 2 + y) txt = '' total = 0 for i, x in enumerate(parts): clss = classes[i % len(classes)] deg = x * 360 if x == 0: continue elif x == 1: txt += f'' else: txt += f'' total += deg return '{1}'.format(size, txt) def gen_radial_graph(percent): return '
{}
'.format( gen_pie_chart([1 - percent, percent], ['cs0', 'cs1'])) def gen_dom_tags(sorted_arr, onlyTrackers=False): txt = '' anyMark = False for i, (name, count, mark) in enumerate(sorted_arr): title = '{} ({})'.format(name, count) if count > 1 else name clss = ' class="trckr"' if mark and not onlyTrackers else '' txt += '{} '.format(clss, title) anyMark |= mark if txt: note = '

* Potential trackers are highlighted

' return '
{}{}
'.format( 'trckr ' if onlyTrackers else '', txt, note if anyMark else '') else: return '– None –' def prepare_json(obj): if not obj['name']: obj['name'] = '< App-Name >' rec_count = len(obj['rec_len']) time_total = sum(obj['rec_len']) obj['sum_rec'] = rec_count obj['sum_logs'] = sum([sum(x[1]) for x in obj['pardom'].values()]) obj['sum_logs_pm'] = obj['sum_logs'] / (time_total or 1) * 60 obj['sum_time'] = time_total obj['avg_time'] = time_total / rec_count def transform(ddic): res = list() for name, (is_tracker, counts) in ddic.items(): rec_percent = len(counts) / rec_count if rec_percent < THRESHOLD_PERCENT_OF_LOGS: continue avg = sum(counts) / rec_count # len(counts) if avg < THRESHOLD_MIN_AVG_LOGS: continue res.append([name, round(avg + 0.001), is_tracker]) res.sort(key=lambda x: (-x[1], x[0])) # sort by count desc, then name return res obj['pardom'] = transform(obj['pardom']) obj['subdom'] = transform(obj['subdom']) # do this after the transformation: c_tracker = 0 c_total = 0 for _, c, flag in obj['subdom']: c_tracker += c if flag else 0 c_total += c obj['tracker_percent'] = c_tracker / (c_total or 1) obj['tracker'] = list(filter(lambda x: x[2], obj['subdom'])) obj['avg_logs'] = c_total obj['avg_logs_pm'] = c_total / (obj['avg_time'] or 1) * 60 def gen_html(bundle_id, obj): prepare_json(obj) return mylib.template_with_base(f'''

{obj['name']}

Bundle-id:{ bundle_id }

{ gen_radial_graph(obj['tracker_percent']) }
Last update:
Number of recordings:{ obj['sum_rec'] }
Total number of requests:{ obj['sum_logs'] }({ round(obj['sum_logs_pm'], 1)} / min)
Average number of requests:{ obj['avg_logs'] }({ round(obj['avg_logs_pm'], 1)} / min)
Average recording time:{ seconds_to_time(obj['avg_time']) }
Cumulative recording time:{ seconds_to_time(obj['sum_time']) }

Connections

Potential Trackers ({ len(obj['tracker']) }):

{ gen_dom_tags(obj['tracker'], onlyTrackers=True) }

Domains ({ len(obj['pardom']) }):

{ gen_dotgraph(obj['pardom']) } { gen_dom_tags(obj['pardom']) }

Subdomains ({ len(obj['subdom']) }):

{ gen_dotgraph(obj['subdom']) } { gen_dom_tags(obj['subdom']) }
''', title=obj['name']) def process(bundle_ids): print('generating html pages ...') if bundle_ids == ['*']: bundle_ids = list(mylib.enum_appids()) for bid in bundle_ids: print(' ' + bid) json = mylib.json_read_combined(bid) mylib.mkdir_out_app(bid) with open(mylib.path_out_app(bid, 'index.html'), 'w') as fp: fp.write(gen_html(bid, json)) print('') if __name__ == '__main__': args = sys.argv[1:] if len(args) > 0: process(args) else: # process(['*']) mylib.usage(__file__, '[bundle_id] [...]')