Average counts and threshold processing

out/static/style.css

@@ -54,9 +54,7 @@ footer .links a { color: #ddd; }
 #main-nav img { height: 1.2em; margin: 0 -0.4em; }
 #main-nav img:hover { transform: scale(1.2); }
 
-td { padding: 0.2em 1em 0.2em 0.1em; }
 .squeeze { max-width: 700px; }
-.wrap { word-break: break-all; }
 
 #get-appcheck:hover { color: #586472; }
 #get-appcheck img { width: 3em; height: 3em; margin: 0.3em; }
@@ -92,6 +90,12 @@ td { padding: 0.2em 1em 0.2em 0.1em; }
 #pagination a { margin: 0.5em; padding: 0.2em }
 #pagination a.active { border: 1pt solid black; border-radius: 0.2em; }
 
+/* app bundle */
+h2.title { margin-bottom: 0; }
+p.subtitle { margin-top: 0.2em; }
+.mg_lr { margin: 0 0.4em; }
+.snd { color: #586472; font-size: 0.85em; }
+td { padding: 0.2em 1em 0.2em 0.1em; }
 #meta td:nth-child(2) { font-weight: bold }
 
 /* domain tags */

src/html_bundle.py

@@ -5,6 +5,9 @@ import time
 import math
 import common_lib as mylib
 
+THRESHOLD_PERCENT_OF_LOGS = 0.7  # domain appears in % recordings
+THRESHOLD_MIN_AVG_LOGS = 1.0  # at least x times in total (after %-thresh)
+
 
 def seconds_to_time(seconds):
     minutes, seconds = divmod(seconds, 60)
@@ -54,8 +57,7 @@ def gen_pie_chart(parts, classes, stroke=0.6):
     return '<svg viewBox="0 0 {0} {0}">{1}</svg>'.format(size, txt)
 
 
-def gen_radial_graph(obj):
+def gen_radial_graph(percent):
-    percent = obj['#logs_tracker'] / (obj['#logs_total'] or 1)
     return '<div class="pie-chart">{}</div>'.format(
         gen_pie_chart([1 - percent, percent], ['cs0', 'cs1']))
 
@@ -77,59 +79,70 @@ def gen_dom_tags(sorted_arr, onlyTrackers=False):
 
 
 def prepare_json(obj):
-    def calc_sum(arr):
+    if not obj['name']:
-        # TODO: use average or median, not total count
+        obj['name'] = '&lt; App-Name &gt;'
-        return sum(arr)
+    rec_count = len(obj['rec_len'])
+    time_total = sum(obj['rec_len'])
+    obj['sum_rec'] = rec_count
+    obj['sum_logs'] = sum([sum(x[1]) for x in obj['pardom'].values()])
+    obj['sum_logs_pm'] = obj['sum_logs'] / (time_total or 1) * 60
+    obj['sum_time'] = time_total
+    obj['avg_time'] = time_total / rec_count
 
     def transform(ddic):
         res = list()
         for name, (is_tracker, counts) in ddic.items():
-            res.append([name, calc_sum(counts), is_tracker])
+            rec_percent = len(counts) / rec_count
+            if rec_percent < THRESHOLD_PERCENT_OF_LOGS:
+                continue
+            avg = sum(counts) / rec_count  # len(counts)
+            if avg < THRESHOLD_MIN_AVG_LOGS:
+                continue
+            res.append([name, round(avg + 0.001), is_tracker])
         res.sort(key=lambda x: (-x[1], x[0]))  # sort by count desc, then name
         return res
 
-    if not obj['name']:
-        obj['name'] = '&lt; App-Name &gt;'
-    obj['#rec'] = len(obj['rec_len'])
-    obj['rec_len'] = sum(obj['rec_len'])
     obj['pardom'] = transform(obj['pardom'])
     obj['subdom'] = transform(obj['subdom'])
     # do this after the transformation:
+    c_tracker = 0
+    c_total = 0
+    for _, c, flag in obj['subdom']:
+        c_tracker += c if flag else 0
+        c_total += c
+    obj['tracker_percent'] = c_tracker / (c_total or 1)
     obj['tracker'] = list(filter(lambda x: x[2], obj['subdom']))
-    obj['#logs_total'] = sum(map(lambda x: x[1], obj['pardom']))
+    obj['avg_logs'] = c_total
-    obj['#logs_tracker'] = sum(map(lambda x: x[1], obj['tracker']))
+    obj['avg_logs_pm'] = c_total / (obj['avg_time'] or 1) * 60
 
 
 def gen_html(bundle_id, obj):
     prepare_json(obj)
     return mylib.template_with_base(f'''
-<h2>{obj['name']}</h2>
+<h2 class="title">{obj['name']}</h2>
+<p class="subtitle snd"><i class="mg_lr">Bundle-id:</i>{ bundle_id }</p>
 <div id="meta">
   <div class="icons">
     <img src="icon.png" width="100" height="100">
-    { gen_radial_graph(obj) }
+    { gen_radial_graph(obj['tracker_percent']) }
   </div>
   <table>
-    <tr><td>Bundle-id:</td><td class="wrap">{
+    <tr><td>Last update:</td><td><time datetime="{
-        bundle_id
-    }</td></tr>
-    <tr><td>Number of recordings:</td><td>{
-        obj['#rec']
-    }</td></tr>
-    <tr><td>Total number of logs:</td><td>{
-        obj['#logs_total']
-    }</td></tr>
-    <tr><td>Cumulative recording time:</td><td>{
-        seconds_to_time(obj['rec_len'])
-    }</td></tr>
-    <tr><td>Average recording time:</td><td>{
-        round(obj['rec_len'] / obj['#rec'], 1)
-    } s</td></tr>
-    <tr><td>Last updated:</td><td><time datetime="{
         time.strftime('%Y-%m-%d %H:%M', time.gmtime(obj['last_date']))
     }">{
         time.strftime('%Y-%m-%d, %H:%M', time.gmtime(obj['last_date']))
     }</time></td></tr>
+    <tr><td>Number of recordings:</td><td>{ obj['sum_rec'] }</td></tr>
+    <tr><td>Total number of logs:</td><td>{
+        obj['sum_logs'] }<i class="snd mg_lr">({
+            round(obj['sum_logs_pm'], 1)} / min)</i></td></tr>
+    <tr><td>Average number of logs:</td><td>{
+        obj['avg_logs'] }<i class="snd mg_lr">({
+            round(obj['avg_logs_pm'], 1)} / min)</i></td></tr>
+    <tr><td>Average recording time:</td><td>{
+        round(obj['avg_time'], 1) } sec</td></tr>
+    <tr><td>Cumulative recording time:</td><td>{
+        seconds_to_time(obj['sum_time']) }</td></tr>
   </table>
 </div>
 <h3>Connections</h3>

templates/base.html

@@ -19,11 +19,13 @@
 </head>
 <body>
   <header>
-    <ul id="main-nav">
+    <nav>
-      <li><a href="/index/page/1">All Apps</a></li>
+      <ul id="main-nav">
-      <li><a href="#">About</a></li>
+        <li><a href="/index/page/1">All Apps</a></li>
-      <li><a class="no-ul" href="https://github.com/relikd/appcheck" target="_blank"><img src="/static/github.svg" alt="GitHub"></a></li>
+        <li><a href="#">About</a></li>
-    </ul>
+        <li><a class="no-ul" href="https://github.com/relikd/appcheck" target="_blank"><img src="/static/github.svg" alt="GitHub"></a></li>
+      </ul>
+    </nav>
     <h1><a class="no-ul" href="/"><img src="/static/logo.svg" alt="logo" width="50" height="50"> AppCheck <span>– Privacy Monitor</span></a></h1>
   </header>
   <main>