Redirect + trackers + php upload api + binary tree search + uuids

src/html_bundle.py

@@ -15,7 +15,9 @@ def sort_dict(count_dict):
     return names, sizes
 
 
-def gen_graph(count_dict, outfile):
+def gen_graph(count_dict, outfile, overwrite=False):
+    if mylib.file_exists(outfile) and not overwrite:
+        return
     names, sizes = sort_dict(count_dict)
     pie1, _ = plt.pie(sizes, labels=names)
     plt.setp(pie1, width=0.5, edgecolor='white')
@@ -30,15 +32,23 @@ def seconds_to_time(seconds):
     return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds)
 
 
-def gen_domain_tags(unsorted_dict):
-    for x in sorted(unsorted_dict):
-        yield '<i>{}</i>'.format(x)
+def gen_dom_tags(unsorted_dict, trackers=None):
+    sorted_arr = sorted(unsorted_dict, key=lambda x: (-x[1], x[0]))
+    res = []
+    for x, y in sorted_arr:
+        clss = ' class="bad"' if trackers and trackers[x] else ''
+        title = x  # if y == 1 else '{} ({})'.format(x, y)
+        res.append('<i{}>{}</i>'.format(clss, title))
+    return ' '.join(res)
 
 
 def gen_html(bundle_id, obj):
+    track_dom = [(dom, obj['total_subdom'][dom])
+                 for dom, known in obj['tracker_subdom'].items() if known]
     return mylib.template_with_base(f'''
 <h2>{obj['name']}</h2>
 <div id="meta">
+  <img id="appicon" src="icon.png" width="100" height="100">
   <table>
     <tr><td>Bundle-id:</td><td>{
         bundle_id
@@ -60,14 +70,14 @@ def gen_html(bundle_id, obj):
 <h3>Connections</h3>
 <div id="connections">
   <table>
+    <tr><td>Known Trackers ({ len(track_dom) }):</td><td>{
+        gen_dom_tags(track_dom)
+    }</td></tr>
     <tr><td>Domains:</td><td>{
-        ''.join(gen_domain_tags(obj['uniq_pardom']))
+        gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom'])
     }</td></tr>
     <tr><td>Subdomains:</td><td>{
-        ''.join(gen_domain_tags(obj['uniq_subdom']))
-    }</td></tr>
-    <tr><td>Known Trackers:</td><td>{
-        '...'
+        gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom'])
     }</td></tr>
   </table>
   <figure><img src="par.svg"></figure>
@@ -75,25 +85,27 @@ def gen_html(bundle_id, obj):
 </div>''', title=obj['name'])
 
 
-def make_bundle_out(bundle_id):
-    jdata = mylib.json_read_combined(bundle_id)
+def make_bundle_out(bundle_id, forceGraphs=False):
+    json = mylib.json_read_combined(bundle_id)
     out_dir = mylib.path_out_app(bundle_id)
     needs_update_index = False
     if not mylib.dir_exists(out_dir):
         needs_update_index = True
         mylib.mkdir(out_dir)
     try:
-        gen_graph(jdata['total_subdom'], mylib.path_add(out_dir, 'sub.svg'))
-        gen_graph(jdata['total_pardom'], mylib.path_add(out_dir, 'par.svg'))
+        gen_graph(json['total_subdom'], mylib.path_add(out_dir, 'sub.svg'),
+                  overwrite=forceGraphs)
+        gen_graph(json['total_pardom'], mylib.path_add(out_dir, 'par.svg'),
+                  overwrite=forceGraphs)
     except KeyError:
         mylib.err('bundle-generate-page', 'skip: ' + bundle_id)
 
     with open(mylib.path_add(out_dir, 'index.html'), 'w') as fp:
-        fp.write(gen_html(bundle_id, jdata))
+        fp.write(gen_html(bundle_id, json))
     return needs_update_index
 
 
-def process(bundle_ids):
+def process(bundle_ids, forceGraphs=False):
     print('generating html pages ...')
     if bundle_ids == ['*']:
         bundle_ids = list(mylib.enum_appids())
@@ -101,7 +113,7 @@ def process(bundle_ids):
     ids_new_in_index = set()
     for bid in bundle_ids:
         print('  ' + bid)
-        if make_bundle_out(bid):
+        if make_bundle_out(bid, forceGraphs=forceGraphs):
             ids_new_in_index.add(bid)
     print('')
     return ids_new_in_index