More stats

src/README.md

@@ -1,4 +1,4 @@
-![dependency](z_dependency.png)
+![dependency](z_dependency.svg)
 
 ## Structure
 
@@ -39,17 +39,19 @@ If you are missing some icons run `main.py icons`. This should also download any
 Given A → B, B depends on A
 
 ```
-digraph G {
+digraph Dependency {
   "." -> html_root
   "." -> bundle_download
   bundle_download -> index_app_names
   index_app_names -> html_bundle
   index_app_names -> html_index_apps
   index_app_names -> html_index_domains
+  index_meta -> html_index_domains
   "." -> bundle_combine
+  bundle_combine -> index_meta
+  bundle_combine -> html_bundle
   bundle_combine -> index_domains
   index_domains -> html_index_domains
-  bundle_combine -> html_bundle
   "." -> tracker_download
 }
 ```

src/html_index_domains.py

@@ -3,6 +3,7 @@
 import common_lib as mylib
 import index_app_names
 import index_domains
+import index_meta
 
 
 def a_app(bundle_id):
@@ -115,19 +116,23 @@ def gen_html_lookup(html_dir, json, key, title):
 
 
 def gen_html_stats(c_apps, c_domains):
+    [c_recordings, c_logs] = index_meta.get_total_counts()
     title = 'Statistics'
     mylib.mkdir(mylib.path_out('stats'))
     with open(mylib.path_out('stats', 'index.html'), 'w') as fp:
         fp.write(mylib.template_with_base('''
 <h2>{}</h2>
 <p>
-  The AppCheck database currently contains <b>{} apps</b> with a total of <b>{} unique domains</b>.
+  The AppCheck database currently contains <b>{:,} apps</b> with a total of <b>{:,} unique domains</b>.
+</p>
+<p>
+  Collected through <b>{:,} recordings</b> with <b>{:,} individual requests</b>.
 </p>
 <ul>
+  <li>List of <a href="/index/apps/1/">Apps</a></li>
   <li>List of <a href="/index/domains/all/">Requested Domains</a></li>
   <li>List of <a href="/index/domains/tracker/">Trackers</a></li>
-  <li>List of <a href="/index/apps/1/">Apps</a></li>
-</ul>'''.format(title, c_apps, c_domains), title=title))
+</ul>'''.format(title, c_apps, c_domains, c_recordings, c_logs), title=title))
 
 
 def process():

src/index_domains.py

@@ -128,5 +128,5 @@ if __name__ == '__main__':
     if len(args) > 0:
         process(args)
     else:
-        process(['*'], deleteOnly=False)
+        # process(['*'], deleteOnly=False)
         mylib.usage(__file__, '[bundle_id] [...]')

src/index_meta.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import sys
+import common_lib as mylib
+
+
+def index_file():
+    return mylib.path_data_index('meta.json')
+
+
+def load_json_from_disk(fname):
+    return mylib.json_read(fname) if mylib.file_exists(fname) else {}
+
+
+def get_total_counts():
+    try:
+        return load_json_from_disk(index_file())['_']
+    except KeyError:
+        return [0, 0]
+
+
+def process(bundle_ids, deleteOnly=False):
+    print('writing index: meta ...')
+    fname = index_file()
+    if bundle_ids == ['*']:
+        bundle_ids = list(mylib.enum_data_appids())
+        print('  full reset')
+        mylib.rm_file(fname)  # rebuild from ground up
+
+    # json format: `bundle-id : [#recordings, #logs, #domains, #subdomains]`
+    index = load_json_from_disk(fname)
+    for bid in bundle_ids:
+        # delete old value
+        try:
+            del(index[bid])
+        except KeyError:
+            pass
+        if deleteOnly:
+            continue
+        # set new value
+        json = mylib.json_read(mylib.path_data_app(bid, 'evaluated.json'))
+        index[bid] = [json['sum_rec'], json['sum_logs'],
+                      len(json['pardom']), len(json['subdom'])]
+    # sum of counts
+    try:
+        del(index['_'])
+    except KeyError:
+        pass
+    total = [0, 0]
+    for val in index.values():
+        total[0] += val[0]
+        total[1] += val[1]
+    index['_'] = total
+
+    # write json
+    mylib.json_write(fname, index, pretty=False)
+    print('')
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    if len(args) > 0:
+        process(args)
+    else:
+        # process(['*'], deleteOnly=False)
+        mylib.usage(__file__, '[bundle_id] [...]')

src/main.py

@@ -11,6 +11,7 @@ import html_bundle
 import html_index_domains
 import index_app_names
 import index_domains
+import index_meta
 import tracker_download
 
 
@@ -27,13 +28,14 @@ def print_usage_and_exit():
     exit(0)
 
 
-def rebuild_app_index(inclRoot=False):
+def rebuild_app_index_html(inclRoot=False):
     html_index_apps.process()
     if inclRoot:  # TODO: remove check if root contains dynamic content
         html_root.process()
 
 
 def rebuild_domain_index(bundle_ids, deleteOnly=False):
+    index_meta.process(bundle_ids, deleteOnly=deleteOnly)
     index_domains.process(bundle_ids, deleteOnly=deleteOnly)
     html_index_domains.process()
 
@@ -60,7 +62,7 @@ def del_id(bundle_ids):
     print('')
     rebuild_domain_index(bundle_ids, deleteOnly=True)
     if update_app_index:
-        rebuild_app_index(inclRoot=True)
+        rebuild_app_index_html(inclRoot=True)
 
 
 def combine_and_update(bundle_ids, where=None):
@@ -81,7 +83,7 @@ def combine_and_update(bundle_ids, where=None):
         print('no bundle affected by tracker, not generating bundle html')
     # 5. make all apps index
     if len(new_ids) > 0:
-        rebuild_app_index()  # must be called after bundle_combine
+        rebuild_app_index_html()  # must be called after bundle_combine
     else:
         print('no new bundle, not rebuilding index')
 
@@ -131,10 +133,10 @@ try:
             # tracker_download.combine_all()
         elif cmd == 'icons':
             if bundle_download.download_missing_icons(force=False):
-                rebuild_app_index()
+                rebuild_app_index_html()
         elif cmd == 'index':
             rebuild_domain_index(['*'])
-            rebuild_app_index(inclRoot=True)
+            rebuild_app_index_html(inclRoot=True)
         elif cmd == 'run':
             if len(params) == 0:
                 print_usage_and_exit()

src/z_dependency.svg

@@ -0,0 +1,126 @@
+<svg width="657pt" height="260pt" viewBox="0.00 0.00 657.00 260.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph1" class="graph" transform="scale(1 1) rotate(0) translate(4 256)">
+<title>Dependency</title>
+<polygon fill="white" stroke="white" points="-4,5 -4,-256 654,-256 654,5 -4,5"></polygon>
+<!-- . -->
+<g id="node1" class="node"><title>.</title>
+<ellipse fill="none" stroke="black" cx="303" cy="-234" rx="27" ry="18"></ellipse>
+<text text-anchor="middle" x="303" y="-229.8" font-family="Times,serif" font-size="14.00">.</text>
+</g>
+<!-- html_root -->
+<g id="node3" class="node"><title>html_root</title>
+<ellipse fill="none" stroke="black" cx="73" cy="-162" rx="49.1927" ry="18"></ellipse>
+<text text-anchor="middle" x="73" y="-157.8" font-family="Times,serif" font-size="14.00">html_root</text>
+</g>
+<!-- .&#45;&gt;html_root -->
+<g id="edge2" class="edge"><title>.-&gt;html_root</title>
+<path fill="none" stroke="black" d="M278.821,-225.641C241.055,-214.147 167.413,-191.734 119.229,-177.07"></path>
+<polygon fill="black" stroke="black" points="120.239,-173.719 109.653,-174.155 118.2,-180.415 120.239,-173.719"></polygon>
+</g>
+<!-- bundle_download -->
+<g id="node5" class="node"><title>bundle_download</title>
+<ellipse fill="none" stroke="black" cx="390" cy="-162" rx="80.3504" ry="18"></ellipse>
+<text text-anchor="middle" x="390" y="-157.8" font-family="Times,serif" font-size="14.00">bundle_download</text>
+</g>
+<!-- .&#45;&gt;bundle_download -->
+<g id="edge4" class="edge"><title>.-&gt;bundle_download</title>
+<path fill="none" stroke="black" d="M319.774,-219.503C331.662,-209.939 347.897,-196.876 361.763,-185.719"></path>
+<polygon fill="black" stroke="black" points="364.005,-188.408 369.602,-179.412 359.617,-182.954 364.005,-188.408"></polygon>
+</g>
+<!-- bundle_combine -->
+<g id="node17" class="node"><title>bundle_combine</title>
+<ellipse fill="none" stroke="black" cx="216" cy="-162" rx="75.1062" ry="18"></ellipse>
+<text text-anchor="middle" x="216" y="-157.8" font-family="Times,serif" font-size="14.00">bundle_combine</text>
+</g>
+<!-- .&#45;&gt;bundle_combine -->
+<g id="edge16" class="edge"><title>.-&gt;bundle_combine</title>
+<path fill="none" stroke="black" d="M286.226,-219.503C274.338,-209.939 258.103,-196.876 244.237,-185.719"></path>
+<polygon fill="black" stroke="black" points="246.383,-182.954 236.398,-179.412 241.995,-188.408 246.383,-182.954"></polygon>
+</g>
+<!-- tracker_download -->
+<g id="node24" class="node"><title>tracker_download</title>
+<ellipse fill="none" stroke="black" cx="569" cy="-162" rx="80.1284" ry="18"></ellipse>
+<text text-anchor="middle" x="569" y="-157.8" font-family="Times,serif" font-size="14.00">tracker_download</text>
+</g>
+<!-- .&#45;&gt;tracker_download -->
+<g id="edge26" class="edge"><title>.-&gt;tracker_download</title>
+<path fill="none" stroke="black" d="M328.025,-226.415C369.14,-215.595 452.103,-193.762 509.094,-178.765"></path>
+<polygon fill="black" stroke="black" points="510.284,-182.071 519.064,-176.141 508.502,-175.301 510.284,-182.071"></polygon>
+</g>
+<!-- index_app_names -->
+<g id="node7" class="node"><title>index_app_names</title>
+<ellipse fill="none" stroke="black" cx="407" cy="-90" rx="80.1456" ry="18"></ellipse>
+<text text-anchor="middle" x="407" y="-85.8" font-family="Times,serif" font-size="14.00">index_app_names</text>
+</g>
+<!-- bundle_download&#45;&gt;index_app_names -->
+<g id="edge6" class="edge"><title>bundle_download-&gt;index_app_names</title>
+<path fill="none" stroke="black" d="M394.202,-143.697C396.096,-135.898 398.377,-126.509 400.484,-117.829"></path>
+<polygon fill="black" stroke="black" points="403.887,-118.648 402.846,-108.104 397.085,-116.996 403.887,-118.648"></polygon>
+</g>
+<!-- html_bundle -->
+<g id="node9" class="node"><title>html_bundle</title>
+<ellipse fill="none" stroke="black" cx="334" cy="-18" rx="59.2871" ry="18"></ellipse>
+<text text-anchor="middle" x="334" y="-13.8" font-family="Times,serif" font-size="14.00">html_bundle</text>
+</g>
+<!-- index_app_names&#45;&gt;html_bundle -->
+<g id="edge8" class="edge"><title>index_app_names-&gt;html_bundle</title>
+<path fill="none" stroke="black" d="M389.7,-72.411C380.427,-63.519 368.843,-52.4107 358.644,-42.6309"></path>
+<polygon fill="black" stroke="black" points="360.816,-39.8649 351.176,-35.4699 355.971,-44.9174 360.816,-39.8649"></polygon>
+</g>
+<!-- html_index_apps -->
+<g id="node11" class="node"><title>html_index_apps</title>
+<ellipse fill="none" stroke="black" cx="489" cy="-18" rx="77.3345" ry="18"></ellipse>
+<text text-anchor="middle" x="489" y="-13.8" font-family="Times,serif" font-size="14.00">html_index_apps</text>
+</g>
+<!-- index_app_names&#45;&gt;html_index_apps -->
+<g id="edge10" class="edge"><title>index_app_names-&gt;html_index_apps</title>
+<path fill="none" stroke="black" d="M426.433,-72.411C437.053,-63.3446 450.373,-51.9742 461.99,-42.0572"></path>
+<polygon fill="black" stroke="black" points="464.373,-44.6246 469.707,-35.4699 459.829,-39.3006 464.373,-44.6246"></polygon>
+</g>
+<!-- html_index_domains -->
+<g id="node13" class="node"><title>html_index_domains</title>
+<ellipse fill="none" stroke="black" cx="164" cy="-18" rx="92.3709" ry="18"></ellipse>
+<text text-anchor="middle" x="164" y="-13.8" font-family="Times,serif" font-size="14.00">html_index_domains</text>
+</g>
+<!-- index_app_names&#45;&gt;html_index_domains -->
+<g id="edge12" class="edge"><title>index_app_names-&gt;html_index_domains</title>
+<path fill="none" stroke="black" d="M359.859,-75.4203C320.888,-64.1939 265.332,-48.1903 223.337,-36.093"></path>
+<polygon fill="black" stroke="black" points="224.092,-32.6682 213.514,-33.2633 222.154,-39.3947 224.092,-32.6682"></polygon>
+</g>
+<!-- index_meta -->
+<g id="node14" class="node"><title>index_meta</title>
+<ellipse fill="none" stroke="black" cx="56" cy="-90" rx="56.1351" ry="18"></ellipse>
+<text text-anchor="middle" x="56" y="-85.8" font-family="Times,serif" font-size="14.00">index_meta</text>
+</g>
+<!-- index_meta&#45;&gt;html_index_domains -->
+<g id="edge14" class="edge"><title>index_meta-&gt;html_index_domains</title>
+<path fill="none" stroke="black" d="M79.7011,-73.6382C94.3705,-64.1302 113.451,-51.7635 129.748,-41.2006"></path>
+<polygon fill="black" stroke="black" points="131.982,-43.9235 138.47,-35.5475 128.174,-38.0494 131.982,-43.9235"></polygon>
+</g>
+<!-- bundle_combine&#45;&gt;html_bundle -->
+<g id="edge20" class="edge"><title>bundle_combine-&gt;html_bundle</title>
+<path fill="none" stroke="black" d="M238.83,-144.654C251.536,-134.888 267.134,-121.737 279,-108 295.683,-88.6865 310.622,-63.7629 320.705,-45.2056"></path>
+<polygon fill="black" stroke="black" points="323.917,-46.6183 325.518,-36.1451 317.736,-43.334 323.917,-46.6183"></polygon>
+</g>
+<!-- bundle_combine&#45;&gt;index_meta -->
+<g id="edge18" class="edge"><title>bundle_combine-&gt;index_meta</title>
+<path fill="none" stroke="black" d="M181.676,-145.983C156.897,-135.142 123.325,-120.455 97.1134,-108.987"></path>
+<polygon fill="black" stroke="black" points="98.3838,-105.723 87.8194,-104.921 95.5781,-112.136 98.3838,-105.723"></polygon>
+</g>
+<!-- index_domains -->
+<g id="node21" class="node"><title>index_domains</title>
+<ellipse fill="none" stroke="black" cx="200" cy="-90" rx="70.0665" ry="18"></ellipse>
+<text text-anchor="middle" x="200" y="-85.8" font-family="Times,serif" font-size="14.00">index_domains</text>
+</g>
+<!-- bundle_combine&#45;&gt;index_domains -->
+<g id="edge22" class="edge"><title>bundle_combine-&gt;index_domains</title>
+<path fill="none" stroke="black" d="M212.045,-143.697C210.282,-135.983 208.163,-126.712 206.197,-118.112"></path>
+<polygon fill="black" stroke="black" points="209.55,-117.073 203.91,-108.104 202.726,-118.633 209.55,-117.073"></polygon>
+</g>
+<!-- index_domains&#45;&gt;html_index_domains -->
+<g id="edge24" class="edge"><title>index_domains-&gt;html_index_domains</title>
+<path fill="none" stroke="black" d="M191.285,-72.055C187.146,-64.0067 182.105,-54.2046 177.488,-45.2259"></path>
+<polygon fill="black" stroke="black" points="180.57,-43.5675 172.884,-36.2753 174.345,-46.7689 180.57,-43.5675"></polygon>
+</g>
+</g>
+</svg>