Dot graph
This commit is contained in:
relikd
@@ -29,9 +29,6 @@ header h1 {
|
||||
margin-top: 0;
|
||||
}
|
||||
header h1 span { font-size: 0.7em; color: silver; }
|
||||
@media(max-width: 622px) { /* 3 columns */
|
||||
header h1 span { display: none; }
|
||||
}
|
||||
main {
|
||||
padding: 0.1em 2em 1.5em;
|
||||
background: #fff;
|
||||
@@ -45,9 +42,6 @@ footer .col3 div {
|
||||
padding: 1%;
|
||||
display: inline-block;
|
||||
}
|
||||
@media(max-width: 647px) {
|
||||
footer .col3 div { width: 100%; padding: 0; }
|
||||
}
|
||||
footer .links {
|
||||
text-align: center;
|
||||
font-size: 0.9em;
|
||||
@@ -60,6 +54,7 @@ footer .links a { color: #ddd; }
|
||||
|
||||
td { padding: 0.2em 1em 0.2em 0.1em; }
|
||||
.squeeze { max-width: 700px; }
|
||||
.wrap { word-wrap: anywhere; }
|
||||
|
||||
#get-appcheck:hover { color: #586472; border-bottom: unset; }
|
||||
#get-appcheck img { width: 3em; height: 3em; margin: 0.3em; }
|
||||
@@ -84,22 +79,22 @@ td { padding: 0.2em 1em 0.2em 0.1em; }
|
||||
margin: 0.5em auto 1em;
|
||||
display: block;
|
||||
}
|
||||
#app-toc img, #get-appcheck img, #appicon {
|
||||
#app-toc img, #get-appcheck img, #meta img {
|
||||
border-radius: 21.5%;
|
||||
border: 0.7px solid #ccc;
|
||||
}
|
||||
#app-toc span { font-size: 0.8em; }
|
||||
#app-toc span.name { font-weight: bold; }
|
||||
#app-toc span.name { font-size: 0.8em; font-weight: bold; }
|
||||
#app-toc span.detail { font-size: 0.7em; }
|
||||
|
||||
#pagination { text-align: center; margin-top: 2em; }
|
||||
#pagination a { margin: 0.5em; padding: 0.2em }
|
||||
#pagination a.active { border: 1pt solid black; border-radius: 0.2em; }
|
||||
|
||||
@media(min-width: 647px) {
|
||||
#meta #appicon { float: right; }
|
||||
}
|
||||
#meta td:nth-child(2) { font-weight: bold }
|
||||
#connections i:not(.empty) {
|
||||
|
||||
/* domain tags */
|
||||
.tags { margin: 2em 0; }
|
||||
.tags i {
|
||||
font-size: 0.9em;
|
||||
font-style: normal;
|
||||
font-weight: normal;
|
||||
@@ -110,6 +105,59 @@ td { padding: 0.2em 1em 0.2em 0.1em; }
|
||||
display: inline-block;
|
||||
margin: 0.12em;
|
||||
}
|
||||
#connections i.bad { border-color: red; }
|
||||
#connections td { vertical-align: top; }
|
||||
#connections figure { display: inline-block; }
|
||||
.tags i.trckr:before, .tags.trckr i:before, p.trckr:before {
|
||||
content: '* ';
|
||||
color: red;
|
||||
font-weight: bold;
|
||||
}
|
||||
.tags i.trckr, .tags.trckr i { background: #DBB; border-color: #C88; }
|
||||
p.trckr { font-size: 0.8em; }
|
||||
|
||||
/* graphs */
|
||||
.pie-chart { width: 100px; height: 100px; }
|
||||
.dot-graph span { outline: 3px solid transparent; outline-offset: 2px; }
|
||||
.dot-graph span:hover { outline-color: black; }
|
||||
.dot-graph span:hover p { display: inline; }
|
||||
.dot-graph p {
|
||||
display: none;
|
||||
position: absolute;
|
||||
vertical-align: bottom;
|
||||
background: white;
|
||||
border: 1px solid #ddd;
|
||||
padding: 0.2em;
|
||||
margin: -2em 0;
|
||||
}
|
||||
.dot-graph i {
|
||||
display: inline-block;
|
||||
vertical-align: bottom;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
padding: 0;
|
||||
margin: 1px 1px;
|
||||
}
|
||||
/* color-bind friendly color palette */
|
||||
.c0{color:#63ACBE} .cb0{background:#63ACBE}
|
||||
.c1{color:#601A4A} .cb1{background:#601A4A}
|
||||
.c2{color:#09F4EC} .cb2{background:#09F4EC}
|
||||
.c3{color:#1F77B4} .cb3{background:#1F77B4}
|
||||
.c4{color:#EE442F} .cb4{background:#EE442F}
|
||||
.c5{color:#7F7F7F} .cb5{background:#7F7F7F}
|
||||
.c6{color:#0F2080} .cb6{background:#0F2080}
|
||||
.c7{color:#3b9f35} .cb7{background:#3b9f35}
|
||||
.c8{color:#F5793A} .cb8{background:#F5793A}
|
||||
.c9{color:#AC66FB} .cb9{background:#AC66FB}
|
||||
|
||||
.cs0{stroke:#3AE48C}
|
||||
.cs1{stroke:#D11}
|
||||
|
||||
/* responsive */
|
||||
@media(max-width: 647px) {
|
||||
header h1 span { display: none; } /* header subtitle */
|
||||
footer .col3 div { width: 100%; padding: 0; } /* 3 columns */
|
||||
#meta .icons { margin-bottom: 1em; } /* icons beside each other */
|
||||
.pie-chart { float: right; }
|
||||
}
|
||||
@media(min-width: 648px) {
|
||||
#meta .icons { float: right; } /* icons below each other */
|
||||
.pie-chart { margin-top: 1em; }
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import common_lib as mylib
|
||||
import tracker_download as tracker
|
||||
@@ -39,7 +40,9 @@ def json_combine(bundle_id):
|
||||
domB = dict() # total sub domains
|
||||
domC = dict() # unique parent domains
|
||||
domD = dict() # total parent domains
|
||||
latest = 0
|
||||
for fname, jdata in mylib.enum_jsons(bundle_id):
|
||||
latest = max(latest, os.path.getmtime(fname)) # or getctime
|
||||
res['name'] = jdata['app-name']
|
||||
res['#rec'] += 1
|
||||
dict_increment(res, 'rec-total', jdata['duration'])
|
||||
@@ -71,6 +74,7 @@ def json_combine(bundle_id):
|
||||
par_tracker[x] = tracker.is_tracker(x)
|
||||
res['tracker_subdom'] = sub_tracker
|
||||
res['tracker_pardom'] = par_tracker
|
||||
res['last_date'] = latest
|
||||
return res
|
||||
|
||||
|
||||
|
||||
@@ -1,29 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import time
|
||||
import math
|
||||
import common_lib as mylib
|
||||
# import matplotlib
|
||||
# import matplotlib.pyplot as plt
|
||||
|
||||
# matplotlib.use('Agg') # disable interactive mode
|
||||
|
||||
|
||||
def sort_dict(count_dict):
|
||||
sorted_count = sorted(count_dict.items(), key=lambda x: (-x[1], x[0]))
|
||||
names = ['{} ({})'.format(*x) for x in sorted_count]
|
||||
sizes = [x[1] for x in sorted_count]
|
||||
return names, sizes
|
||||
|
||||
|
||||
def gen_graph(count_dict, outfile, overwrite=False):
|
||||
if mylib.file_exists(outfile) and not overwrite:
|
||||
return
|
||||
# names, sizes = sort_dict(count_dict)
|
||||
# pie1, _ = plt.pie(sizes, labels=names)
|
||||
# plt.setp(pie1, width=0.5, edgecolor='white')
|
||||
# plt.subplots_adjust(left=0, right=1, top=0.7, bottom=0.3)
|
||||
# plt.savefig(outfile, bbox_inches='tight', pad_inches=0) # transparent=True
|
||||
# plt.close()
|
||||
|
||||
|
||||
def seconds_to_time(seconds):
|
||||
@@ -32,14 +12,74 @@ def seconds_to_time(seconds):
|
||||
return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds)
|
||||
|
||||
|
||||
def gen_dom_tags(unsorted_dict, trackers=None):
|
||||
def gen_dom_tags(unsorted_dict, trackers=None, additionalClasses=None):
|
||||
sorted_arr = sorted(unsorted_dict, key=lambda x: (-x[1], x[0]))
|
||||
res = []
|
||||
for x, y in sorted_arr:
|
||||
clss = ' class="bad"' if trackers and trackers[x] else ''
|
||||
title = x # if y == 1 else '{} ({})'.format(x, y)
|
||||
res.append('<i{}>{}</i>'.format(clss, title))
|
||||
return ' '.join(res) if len(res) > 0 else '<i class="empty">– None –</i>'
|
||||
txt = ''
|
||||
anyMark = False
|
||||
for i, (x, y) in enumerate(sorted_arr):
|
||||
mark = trackers[x] if trackers else True
|
||||
title = x if y == 1 else '{} ({})'.format(x, y)
|
||||
txt += '<i{}>{}</i> '.format(' class="trckr"' if mark else '', title)
|
||||
anyMark |= mark
|
||||
if txt:
|
||||
note = '<p class="trckr">known tracker</p>'
|
||||
return '<div class="tags{}">{}{}</div>'.format(
|
||||
additionalClasses or '', txt, note if anyMark else '')
|
||||
else:
|
||||
return '<i>– None –</i>'
|
||||
|
||||
|
||||
def gen_dotgraph(count_dict):
|
||||
txt = ''
|
||||
sorted_count = sorted(count_dict.items(), key=lambda x: (-x[1], x[0]))
|
||||
for i, (name, count) in enumerate(sorted_count):
|
||||
# TODO: use average not total count
|
||||
txt += '<span title="{0} ({1})"><p>{0} ({1})</p>'.format(name, count)
|
||||
for x in range(count):
|
||||
txt += '<i class="cb{}"></i>'.format(i % 10)
|
||||
txt += '</span>'
|
||||
return '<div class="dot-graph">{}</div>'.format(txt)
|
||||
|
||||
|
||||
def gen_pie_chart(parts, classes, stroke=0.6):
|
||||
size = 1000
|
||||
stroke *= size * 0.5
|
||||
stroke_p = '{:.0f}'.format(stroke)
|
||||
r = (0.99 * size - stroke) / 2
|
||||
r_p = '{:.0f},{:.0f}'.format(r, r)
|
||||
mid = '{:.0f}'.format(size / 2)
|
||||
|
||||
def arc(deg):
|
||||
deg -= 90
|
||||
x = r * math.cos(math.pi * deg / 180)
|
||||
y = r * math.sin(math.pi * deg / 180)
|
||||
return '{:.0f},{:.0f}'.format(size / 2 + x, size / 2 + y)
|
||||
|
||||
txt = ''
|
||||
total = 0
|
||||
for i, x in enumerate(parts):
|
||||
clss = classes[i % len(classes)]
|
||||
deg = x * 360
|
||||
if x == 0:
|
||||
continue
|
||||
elif x == 1:
|
||||
txt += f'<circle fill="transparent" class="{clss}" stroke-width="{stroke_p}" cx="{mid}" cy="{mid}" r="{r}"/>'
|
||||
else:
|
||||
txt += f'<path fill="transparent" class="{clss}" stroke-width="{stroke_p}" d="M{arc(total)}A{r_p},0,{1 if deg > 180 else 0},1,{arc(total + deg)}" />'
|
||||
total += deg
|
||||
return '<svg viewBox="0 0 {0} {0}">{1}</svg>'.format(size, txt)
|
||||
|
||||
|
||||
def gen_radial_graph(obj):
|
||||
total = 0
|
||||
tracker = 0
|
||||
for name, count in obj['total_subdom'].items():
|
||||
total += count
|
||||
if obj['tracker_subdom'][name]:
|
||||
tracker += count
|
||||
percent = tracker / total
|
||||
return '<div class="pie-chart">{}</div>'.format(
|
||||
gen_pie_chart([1 - percent, percent], ['cs0', 'cs1']))
|
||||
|
||||
|
||||
def gen_html(bundle_id, obj):
|
||||
@@ -48,9 +88,12 @@ def gen_html(bundle_id, obj):
|
||||
return mylib.template_with_base(f'''
|
||||
<h2>{obj['name']}</h2>
|
||||
<div id="meta">
|
||||
<img id="appicon" src="icon.png" width="100" height="100">
|
||||
<div class="icons">
|
||||
<img src="icon.png" width="100" height="100">
|
||||
{ gen_radial_graph(obj) }
|
||||
</div>
|
||||
<table>
|
||||
<tr><td>Bundle-id:</td><td>{
|
||||
<tr><td>Bundle-id:</td><td class="wrap">{
|
||||
bundle_id
|
||||
}</td></tr>
|
||||
<tr><td>Number of recordings:</td><td>{
|
||||
@@ -65,45 +108,42 @@ def gen_html(bundle_id, obj):
|
||||
<tr><td>Average recording time:</td><td>{
|
||||
round(obj['rec-total'] / obj['#rec'], 1)
|
||||
} s</td></tr>
|
||||
<tr><td>Last updated:</td><td><time datetime="{
|
||||
time.strftime('%Y-%m-%d %H:%M', time.gmtime(obj['last_date']))
|
||||
}">{
|
||||
time.strftime('%Y-%m-%d, %H:%M', time.gmtime(obj['last_date']))
|
||||
}</time></td></tr>
|
||||
</table>
|
||||
</div>
|
||||
<h3>Connections</h3>
|
||||
<div id="connections">
|
||||
<table>
|
||||
<tr><td>Known Trackers ({ len(track_dom) }):</td><td>{
|
||||
gen_dom_tags(track_dom)
|
||||
}</td></tr>
|
||||
<tr><td>Domains:</td><td>{
|
||||
gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom'])
|
||||
}</td></tr>
|
||||
<tr><td>Subdomains:</td><td>{
|
||||
gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom'])
|
||||
}</td></tr>
|
||||
</table>
|
||||
<figure><img src="par.svg"></figure>
|
||||
<figure><img src="sub.svg"></figure>
|
||||
<div>
|
||||
<h4>Known Trackers ({ len(track_dom) }):</h4>
|
||||
{ gen_dom_tags(track_dom, additionalClasses=' trckr') }
|
||||
<p></p>
|
||||
|
||||
<h4>Domains:</h4>
|
||||
{ gen_dotgraph(obj['total_pardom']) }
|
||||
{ gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom']) }
|
||||
|
||||
<h4>Subdomains:</h4>
|
||||
{ gen_dotgraph(obj['total_subdom']) }
|
||||
{ gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom']) }
|
||||
</div>''', title=obj['name'])
|
||||
|
||||
|
||||
def make_bundle_out(bundle_id, forceGraphs=False):
|
||||
def make_bundle_out(bundle_id):
|
||||
json = mylib.json_read_combined(bundle_id)
|
||||
out_dir = mylib.path_out_app(bundle_id)
|
||||
needs_update_index = False
|
||||
if not mylib.dir_exists(out_dir):
|
||||
needs_update_index = True
|
||||
mylib.mkdir(out_dir)
|
||||
|
||||
gen_graph(json['total_subdom'], mylib.path_add(out_dir, 'sub.svg'),
|
||||
overwrite=forceGraphs)
|
||||
gen_graph(json['total_pardom'], mylib.path_add(out_dir, 'par.svg'),
|
||||
overwrite=forceGraphs)
|
||||
|
||||
with open(mylib.path_add(out_dir, 'index.html'), 'w') as fp:
|
||||
fp.write(gen_html(bundle_id, json))
|
||||
return needs_update_index
|
||||
|
||||
|
||||
def process(bundle_ids, forceGraphs=False):
|
||||
def process(bundle_ids):
|
||||
print('generating html pages ...')
|
||||
if bundle_ids == ['*']:
|
||||
bundle_ids = list(mylib.enum_appids())
|
||||
@@ -111,7 +151,7 @@ def process(bundle_ids, forceGraphs=False):
|
||||
ids_new_in_index = set()
|
||||
for bid in bundle_ids:
|
||||
print(' ' + bid)
|
||||
if make_bundle_out(bid, forceGraphs=forceGraphs):
|
||||
if make_bundle_out(bid):
|
||||
ids_new_in_index.add(bid)
|
||||
print('')
|
||||
return ids_new_in_index
|
||||
|
||||
@@ -40,12 +40,12 @@ def del_id(bundle_ids):
|
||||
html_index.process()
|
||||
|
||||
|
||||
def combine_and_update(bundle_ids, where=None, forceGraphs=False):
|
||||
def combine_and_update(bundle_ids, where=None):
|
||||
affected = bundle_combine.process(bundle_ids, where=where)
|
||||
if len(affected) == 0:
|
||||
print('no bundle affected by tracker, not generating bundle html')
|
||||
return
|
||||
new_ids = html_bundle.process(affected, forceGraphs=forceGraphs)
|
||||
new_ids = html_bundle.process(affected)
|
||||
if len(new_ids) == 0:
|
||||
print('no new bundle, not rebuilding index')
|
||||
return
|
||||
@@ -69,7 +69,7 @@ def import_update():
|
||||
os.remove(fname)
|
||||
print('')
|
||||
if len(needs_update) > 0:
|
||||
combine_and_update(needs_update, forceGraphs=True)
|
||||
combine_and_update(needs_update)
|
||||
|
||||
|
||||
def tracker_update():
|
||||
|
||||
Reference in New Issue
Block a user