Parent domain incl subdomains

out/static/lookup-domain.js

@@ -1,22 +1,30 @@
-function lookup_domain_js(fname_a, fname_b, id1, id2, id3) {
+function lookup_domain_js(fname_doms, fname_apps, fname_subs) {
-  let dom = window.location.hash.substr(1);
+  let dom = window.location.hash.substr(1); // domain name
-  document.getElementById(id1).innerHTML = dom; // domain name
+  document.getElementById('name').innerHTML = dom;
-  let dom_app_list = document.getElementById(id3); // apps list
+  let dom_num_apps = document.getElementById('num-apps');
+  let dom_app_list = document.getElementById('app-toc');
+  let dom_sub_doms = document.getElementById('subdoms');
+  let dom_known_trkr = document.getElementById('known');
   let template = dom_app_list.firstElementChild;
   dom_app_list.innerHTML = 'loading…';
 
   // load reverse domains json
-  loadJSON(fname_a, function(response) {
+  loadJSON(fname_doms, function(response) {
     let elem = JSON.parse(response)[dom];
-    if (!elem || elem.length == 0) {
+    let count = elem.length - 1;
-      document.getElementById(id2).innerHTML = '0 applications';
+    if (!elem || count < 1) {
+      dom_num_apps.innerHTML = '0 applications';
       dom_app_list.innerHTML = '– None –';
       return;
+    } else if (count == 1) {
+      dom_num_apps.innerHTML = '1 application';
+    } else {
+      dom_num_apps.innerHTML = elem.length - 1 + ' applications';
     }
-    document.getElementById(id2).innerHTML = elem.length + ' applications';
+    dom_known_trkr.innerHTML = elem[0] ? 'Yes' : 'No';
 
     // load app name json
-    loadJSON(fname_b, function(response) {
+    loadJSON(fname_apps, function(response) {
       let name_list = JSON.parse(response);
       var apps = [];
       for (var i = elem.length - 1; i >= 0; i--) {
@@ -39,6 +47,23 @@ function lookup_domain_js(fname_a, fname_b, id1, id2, id3) {
         dom_app_list.appendChild(item);
       }
       const observer = lozad(); observer.observe();
+
+      if (!dom_sub_doms) { return }
+      loadJSON(fname_subs, function(response) {
+        let subdomains_list = JSON.parse(response)[dom];
+        if (subdomains_list) {
+          var src = '';
+          for (var i = 0; i < subdomains_list.length; i++) {
+            let sub = subdomains_list[i];
+            let full = sub ? sub + '.' + dom : dom;
+            let lnk = '<a href="/subdomain/#' + full + '">' + sub + '.</a> ';
+            src += lnk;
+          }
+          dom_sub_doms.innerHTML = src;
+        } else {
+          dom_sub_doms.innerHTML = '– None –';
+        }
+      });
     });
   });
 }

src/html_index_domains.py

@@ -89,24 +89,23 @@ def gen_html_trinity(idx_dir, app_count, json, title, symlink):
     mylib.symlink(symlink, mylib.path_out(idx_dir, 'data.json'))
 
 
-def gen_lookup(html_dir, doms_dict, names_dict, title):
+def gen_lookup(html_dir, doms_dict, flag, title):
-    header = HTML.a_path([('All Domains', '/index/domains/all/')],
+    HTML.write(html_dir, f'''
-                         '<span id="name"></span>')
+<h2>{ HTML.a_path([('All Domains', '/index/domains/all/')],
-    HTML.write(html_dir, '''
+                  '<span id="name"></span>') }</h2>
-<h2>{}</h2>
+<p>Known Tracker: <b id="known">?</b></p>
 <p>Present in: <b id="num-apps">… applications</b></p>
+{ '<h3>Subdomains:</h3><div id="subdoms" class="tags"></div>' if flag else '' }
 <h3>Apps containing this domain:</h3>
 <div id="app-toc" class="no-ul-all">
-  {}
+  { HTML.app_tile_template() }
 </div>
-<script type="text/javascript" src="/static/lookup-domain.js"></script>
+<script type="text/javascript" src="/static/lookup-domain.js?2"></script>
 <script type="text/javascript" src="/static/lozad.js"></script>
 <script type="text/javascript">
-  lookup_domain_js('doms.json', 'apps.json', 'name', 'num-apps', 'app-toc');
+  lookup_domain_js('doms.json', '/results/lookup-apps.json', '/results/subdoms.json');
 </script>
-'''.format(header, HTML.app_tile_template()), title=title)
+''', title=title)
-    # after html write which will create the dir
-    mylib.json_write(mylib.path_add(html_dir, 'apps.json'), names_dict)
     mylib.json_write(mylib.path_add(html_dir, 'doms.json'), doms_dict)
 
 
@@ -117,14 +116,21 @@ def process():
     app_count = index_domains.number_of_apps(json)
     dom_count = len(json['subdom'])
 
-    print('  Lookup')
+    # Prepare for lookup
     names = [[x, index_app_names.get_name(x)] for x in json['bundle']]
-    gen_lookup(mylib.path_out('domain'), json['pardom'], names,
+    dest_dir = mylib.path_out('results')
-               title='Domain Lookup')
+    mylib.mkdir(dest_dir)
-    gen_lookup(mylib.path_out('subdomain'), json['subdom'], names,
+    mylib.json_write(mylib.path_add(dest_dir, 'lookup-apps.json'), names)
-               title='Subdomain Lookup')
+    mylib.symlink(index_domains.fname_dom_subdoms(),
+                  mylib.path_add(dest_dir, 'subdoms.json'))
     names = None
 
+    print('  Lookup')
+    gen_lookup(mylib.path_out('domain'), json['pardom'], True,
+               title='Domain Lookup')
+    gen_lookup(mylib.path_out('subdomain'), json['subdom'], False,
+               title='Subdomain Lookup')
+
     print('  All Domains')
     gen_html_trinity(mylib.path_out('index', 'domains', 'all'), app_count,
                      json=json, title='Requested Domains',

src/index_domains.py

@@ -18,6 +18,10 @@ def fname_no_tracker():
     return mylib.path_data_index('domains_no_tracker.json')
 
 
+def fname_dom_subdoms():
+    return mylib.path_data_index('domains_subdomains.json')
+
+
 def load_json_from_disk(index_file):
     return mylib.json_safe_read(
         index_file, fallback={'bundle': [], 'pardom': {}, 'subdom': {}})
@@ -110,6 +114,18 @@ def filter_list_at_least(index, min_count):
     index['pardom'] = par
 
 
+def dict_dom_subdomains(index):
+    ret = {}
+    for subdomain in index['subdom'].keys():
+        pardom = mylib.parent_domain(subdomain)
+        host = subdomain[:-len(pardom) - 1]  # - '.'
+        try:
+            ret[pardom].append(host)
+        except KeyError:
+            ret[pardom] = [host]
+    return ret
+
+
 def number_of_apps(index):
     return sum(1 for x in index['bundle'] if x != '_')
 
@@ -149,6 +165,9 @@ def process(bundle_ids, deleteOnly=False):
         mylib.json_write(fname_tracker(), dict_trkr, pretty=False)
         filter_list_at_least(dict_no_trkr, 5)  # or 0.1 * len(ids)
         mylib.json_write(fname_no_tracker(), dict_no_trkr, pretty=False)
+        mylib.json_write(fname_dom_subdoms(), dict_dom_subdomains(index),
+                         pretty=False)
+
     else:
         print('  no change')
     print('')