Parent domain incl subdomains

src/html_index_domains.py

@@ -89,24 +89,23 @@ def gen_html_trinity(idx_dir, app_count, json, title, symlink):
     mylib.symlink(symlink, mylib.path_out(idx_dir, 'data.json'))
 
 
-def gen_lookup(html_dir, doms_dict, names_dict, title):
-    header = HTML.a_path([('All Domains', '/index/domains/all/')],
-                         '<span id="name"></span>')
-    HTML.write(html_dir, '''
-<h2>{}</h2>
+def gen_lookup(html_dir, doms_dict, flag, title):
+    HTML.write(html_dir, f'''
+<h2>{ HTML.a_path([('All Domains', '/index/domains/all/')],
+                  '<span id="name"></span>') }</h2>
+<p>Known Tracker: <b id="known">?</b></p>
 <p>Present in: <b id="num-apps">… applications</b></p>
+{ '<h3>Subdomains:</h3><div id="subdoms" class="tags"></div>' if flag else '' }
 <h3>Apps containing this domain:</h3>
 <div id="app-toc" class="no-ul-all">
-  {}
+  { HTML.app_tile_template() }
 </div>
-<script type="text/javascript" src="/static/lookup-domain.js"></script>
+<script type="text/javascript" src="/static/lookup-domain.js?2"></script>
 <script type="text/javascript" src="/static/lozad.js"></script>
 <script type="text/javascript">
-  lookup_domain_js('doms.json', 'apps.json', 'name', 'num-apps', 'app-toc');
+  lookup_domain_js('doms.json', '/results/lookup-apps.json', '/results/subdoms.json');
 </script>
-'''.format(header, HTML.app_tile_template()), title=title)
-    # after html write which will create the dir
-    mylib.json_write(mylib.path_add(html_dir, 'apps.json'), names_dict)
+''', title=title)
     mylib.json_write(mylib.path_add(html_dir, 'doms.json'), doms_dict)
 
 
@@ -117,14 +116,21 @@ def process():
     app_count = index_domains.number_of_apps(json)
     dom_count = len(json['subdom'])
 
-    print('  Lookup')
+    # Prepare for lookup
     names = [[x, index_app_names.get_name(x)] for x in json['bundle']]
-    gen_lookup(mylib.path_out('domain'), json['pardom'], names,
-               title='Domain Lookup')
-    gen_lookup(mylib.path_out('subdomain'), json['subdom'], names,
-               title='Subdomain Lookup')
+    dest_dir = mylib.path_out('results')
+    mylib.mkdir(dest_dir)
+    mylib.json_write(mylib.path_add(dest_dir, 'lookup-apps.json'), names)
+    mylib.symlink(index_domains.fname_dom_subdoms(),
+                  mylib.path_add(dest_dir, 'subdoms.json'))
     names = None
 
+    print('  Lookup')
+    gen_lookup(mylib.path_out('domain'), json['pardom'], True,
+               title='Domain Lookup')
+    gen_lookup(mylib.path_out('subdomain'), json['subdom'], False,
+               title='Subdomain Lookup')
+
     print('  All Domains')
     gen_html_trinity(mylib.path_out('index', 'domains', 'all'), app_count,
                      json=json, title='Requested Domains',

src/index_domains.py

@@ -18,6 +18,10 @@ def fname_no_tracker():
     return mylib.path_data_index('domains_no_tracker.json')
 
 
+def fname_dom_subdoms():
+    return mylib.path_data_index('domains_subdomains.json')
+
+
 def load_json_from_disk(index_file):
     return mylib.json_safe_read(
         index_file, fallback={'bundle': [], 'pardom': {}, 'subdom': {}})
@@ -110,6 +114,18 @@ def filter_list_at_least(index, min_count):
     index['pardom'] = par
 
 
+def dict_dom_subdomains(index):
+    ret = {}
+    for subdomain in index['subdom'].keys():
+        pardom = mylib.parent_domain(subdomain)
+        host = subdomain[:-len(pardom) - 1]  # - '.'
+        try:
+            ret[pardom].append(host)
+        except KeyError:
+            ret[pardom] = [host]
+    return ret
+
+
 def number_of_apps(index):
     return sum(1 for x in index['bundle'] if x != '_')
 
@@ -149,6 +165,9 @@ def process(bundle_ids, deleteOnly=False):
         mylib.json_write(fname_tracker(), dict_trkr, pretty=False)
         filter_list_at_least(dict_no_trkr, 5)  # or 0.1 * len(ids)
         mylib.json_write(fname_no_tracker(), dict_no_trkr, pretty=False)
+        mylib.json_write(fname_dom_subdoms(), dict_dom_subdomains(index),
+                         pretty=False)
+
     else:
         print('  no change')
     print('')