Redirect + trackers + php upload api + binary tree search + uuids

src/bundle_combine.py

@@ -2,22 +2,24 @@
 
 import sys
 import common_lib as mylib
+import tracker_download as tracker
 
 
-with open(mylib.path_root('src', '3rd-domains.txt'), 'r') as fp:
-    level3_doms = set([x.strip() for x in fp.readlines()])
+level3_doms = None
 
 
 def dom_in_3rd_domain(needle):
-    # TODO: binary tree lookup
-    return needle in level3_doms
+    global level3_doms
+    if not level3_doms:
+        level3_doms = mylib.read_list('3rd-domains.txt')
+    return mylib.bintree_lookup(level3_doms, needle)
 
 
 def get_parent_domain(subdomain):
     parts = subdomain.split('.')
     if len(parts) < 3:
-        return x
-    elif dom_in_3rd_domain('.'.join(parts[-2:])):
+        return subdomain
+    elif dom_in_3rd_domain(parts[-1] + '.' + parts[-2]):
         return '.'.join(parts[-3:])
     else:
         return '.'.join(parts[-2:])
@@ -46,6 +48,7 @@ def json_combine(bundle_id):
             uniq_par = set()
             for subdomain in logs:
                 occurs = len(logs[subdomain])
+                sub_tracker = tracker.is_tracker(subdomain)
                 dict_increment(res, '#logs', occurs)
                 dict_increment(domA, subdomain, 1)
                 dict_increment(domB, subdomain, occurs)
@@ -60,17 +63,40 @@ def json_combine(bundle_id):
     res['uniq_pardom'] = domC
     res['total_subdom'] = domB
     res['total_pardom'] = domD
+    sub_tracker = dict()
+    par_tracker = dict()
+    for x in domA:
+        sub_tracker[x] = tracker.is_tracker(x)
+    for x in domC:
+        par_tracker[x] = tracker.is_tracker(x)
+    res['tracker_subdom'] = sub_tracker
+    res['tracker_pardom'] = par_tracker
     return res
 
 
-def process(bundle_ids):
+def process(bundle_ids, where=None):
     print('writing combined json ...')
     if bundle_ids == ['*']:
-        bundle_ids = list(mylib.enum_appids())
+        bundle_ids = list(mylib.enum_data_appids())
+
+    affected_ids = []
+    haystack = sorted([x[::-1] for x in where]) if where else None
     for bid in bundle_ids:
-        print('  ' + bid)
-        mylib.json_write_combined(bid, json_combine(bid))
+        obj = json_combine(bid)
+        should_update = False
+        if not haystack:
+            should_update = True
+        else:
+            for x in obj['uniq_subdom']:
+                if mylib.bintree_lookup(haystack, x[::-1]):
+                    should_update = True
+                    break
+        if should_update:
+            print('  ' + bid)
+            mylib.json_write_combined(bid, obj)
+            affected_ids.append(bid)
     print('')
+    return affected_ids
 
 
 if __name__ == '__main__':

src/bundle_download.py

@@ -27,10 +27,27 @@ def download_icon(bundle_id, force=False, langs=['us', 'de']):
         json = None
         for lang in langs:
             if not json:
-                json = mylib.json_read_meta(bundle_id, lang)
+                try:
+                    json = mylib.json_read_meta(bundle_id, lang)
+                except Exception:
+                    continue
         mylib.download_file(json['artworkUrl100'], icon_file)
 
 
+def download_missing_icons(force=False, langs=['us', 'de']):
+    didAny = False
+    for bid in mylib.enum_appids():
+        if not mylib.file_exists(mylib.path_out_app(bid, 'icon.png')):
+            if not didAny:
+                print('downloading missing icons ...')
+                didAny = True
+            print('  ' + bid)
+            download_icon(bid, force=force, langs=langs)
+    if didAny:
+        print('')
+    return didAny
+
+
 def download(bundle_id, force=False):
     if not mylib.valid_bundle_id(bundle_id):
         mylib.err('apple-download', 'invalid id: ' + bundle_id)

src/bundle_import.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import common_lib as mylib
-
-
-def next_path(path_pattern):
-    i = 1
-    while os.path.exists(path_pattern % i):
-        i = i * 2
-    a, b = (i // 2, i)
-    while a + 1 < b:
-        c = (a + b) // 2  # interval midpoint
-        a, b = (c, b) if os.path.exists(path_pattern % c) else (a, c)
-    return path_pattern % b
-
-
-def process():
-    print('checking incoming files ...')
-    prefix = mylib.path_len(mylib.path_data())
-    needs_update = set()
-    for fname, jdata in mylib.enum_newly_added():
-        try:
-            bundle_id = jdata['app-bundle'].strip()
-            if mylib.valid_bundle_id(bundle_id):
-                dest = mylib.path_data_app(bundle_id)
-                needs_update.add(bundle_id)
-            else:
-                dest = mylib.path_data('_manually')
-                # needs_update.add('_manually')
-
-            mylib.mkdir(dest)
-            dest_file = next_path(mylib.path_add(dest, 'id_%s.json'))
-            mylib.mv(fname, dest_file, printOmitPrefix=prefix)
-        except KeyError:
-            mylib.err('json-import', 'malformed json: ' + bundle_id)
-    print('done.')
-    print('')
-    return needs_update

src/common_lib.py

@@ -60,6 +60,7 @@ def template_with_base(content, title=None):
 
 # Other
 
+# same regex as in `api/v1/contribute/index.php`
 regex_bundle_id = re.compile(r'^[A-Za-z0-9\.\-]{1,155}$')
 logging.basicConfig(filename=os.path.join(os.pardir, "error.log"),
                     format='%(asctime)s %(message)s',
@@ -85,6 +86,32 @@ def printf(msg):
     print(msg, end='', flush=True)
 
 
+# Binary Tree Search
+
+def read_list(list_name):
+    path = path_root('src', 'lists', list_name)
+    if not file_exists(path):
+        return []
+    with open(path, 'r') as fp:
+        return [x.strip() for x in fp.readlines()]
+
+
+def bintree_lookup(tree, needle):
+    lo = 0
+    hi = len(tree) - 1
+    while lo <= hi:
+        mid = (lo + hi) // 2
+        if tree[mid] < needle:
+            lo = mid + 1
+        elif needle < tree[mid]:
+            hi = mid - 1
+        else:
+            return True  # mid
+    if lo > 0 and needle.startswith(tree[lo - 1] + '.'):
+        return True  # lo - 1
+    return False  # -1
+
+
 # Filesystem
 
 def mkdir(path):
@@ -113,6 +140,46 @@ def meta_json_exists(bundle_id, lang):
     return file_exists(path_data_app(bundle_id, 'info_{}.json'.format(lang)))
 
 
+def next_path(path_pattern):
+    i = 1
+    while os.path.exists(path_pattern % i):
+        i = i * 2
+    a, b = (i // 2, i)
+    while a + 1 < b:
+        c = (a + b) // 2  # interval midpoint
+        a, b = (c, b) if os.path.exists(path_pattern % c) else (a, c)
+    return path_pattern % b
+
+
+def diff_files(fileA, fileB):
+    with open(fileA, 'r') as fpA:
+        with open(fileB, 'r') as fpB:
+            a = '_'
+            b = '_'
+            diff = []
+            while a != '' and b != '':
+                a = fpA.readline()
+                b = fpB.readline()
+                if a == b:
+                    continue
+                while a != b:
+                    if a == '' or b == '':
+                        break
+                    if a < b:
+                        diff.append(a.strip())
+                        a = fpA.readline()
+                    elif b < a:
+                        diff.append(b.strip())
+                        b = fpB.readline()
+            while a != '':
+                a = fpA.readline()
+                diff.append(a.strip())
+            while b != '':
+                b = fpB.readline()
+                diff.append(b.strip())
+            return diff
+
+
 # Download
 
 def download(url, isJSON=False):
@@ -129,9 +196,8 @@ def download_file(url, path):
 # Enumerator
 
 def enum_newly_added():
-    for fname in glob.glob(path_data('_in', '*.json')):
-        with open(fname, 'r') as fp:
-            yield fname, json.load(fp)
+    for fname in glob.glob(path_data('_in', 'in_*')):
+        yield fname, os.path.basename(fname)[3:]  # del prefix 'in_'
 
 
 def enum_appids():

src/html_bundle.py

@@ -15,7 +15,9 @@ def sort_dict(count_dict):
     return names, sizes
 
 
-def gen_graph(count_dict, outfile):
+def gen_graph(count_dict, outfile, overwrite=False):
+    if mylib.file_exists(outfile) and not overwrite:
+        return
     names, sizes = sort_dict(count_dict)
     pie1, _ = plt.pie(sizes, labels=names)
     plt.setp(pie1, width=0.5, edgecolor='white')
@@ -30,15 +32,23 @@ def seconds_to_time(seconds):
     return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds)
 
 
-def gen_domain_tags(unsorted_dict):
-    for x in sorted(unsorted_dict):
-        yield '<i>{}</i>'.format(x)
+def gen_dom_tags(unsorted_dict, trackers=None):
+    sorted_arr = sorted(unsorted_dict, key=lambda x: (-x[1], x[0]))
+    res = []
+    for x, y in sorted_arr:
+        clss = ' class="bad"' if trackers and trackers[x] else ''
+        title = x  # if y == 1 else '{} ({})'.format(x, y)
+        res.append('<i{}>{}</i>'.format(clss, title))
+    return ' '.join(res)
 
 
 def gen_html(bundle_id, obj):
+    track_dom = [(dom, obj['total_subdom'][dom])
+                 for dom, known in obj['tracker_subdom'].items() if known]
     return mylib.template_with_base(f'''
 <h2>{obj['name']}</h2>
 <div id="meta">
+  <img id="appicon" src="icon.png" width="100" height="100">
   <table>
     <tr><td>Bundle-id:</td><td>{
         bundle_id
@@ -60,14 +70,14 @@ def gen_html(bundle_id, obj):
 <h3>Connections</h3>
 <div id="connections">
   <table>
+    <tr><td>Known Trackers ({ len(track_dom) }):</td><td>{
+        gen_dom_tags(track_dom)
+    }</td></tr>
     <tr><td>Domains:</td><td>{
-        ''.join(gen_domain_tags(obj['uniq_pardom']))
+        gen_dom_tags(obj['total_pardom'].items(), obj['tracker_pardom'])
     }</td></tr>
     <tr><td>Subdomains:</td><td>{
-        ''.join(gen_domain_tags(obj['uniq_subdom']))
-    }</td></tr>
-    <tr><td>Known Trackers:</td><td>{
-        '...'
+        gen_dom_tags(obj['total_subdom'].items(), obj['tracker_subdom'])
     }</td></tr>
   </table>
   <figure><img src="par.svg"></figure>
@@ -75,25 +85,27 @@ def gen_html(bundle_id, obj):
 </div>''', title=obj['name'])
 
 
-def make_bundle_out(bundle_id):
-    jdata = mylib.json_read_combined(bundle_id)
+def make_bundle_out(bundle_id, forceGraphs=False):
+    json = mylib.json_read_combined(bundle_id)
     out_dir = mylib.path_out_app(bundle_id)
     needs_update_index = False
     if not mylib.dir_exists(out_dir):
         needs_update_index = True
         mylib.mkdir(out_dir)
     try:
-        gen_graph(jdata['total_subdom'], mylib.path_add(out_dir, 'sub.svg'))
-        gen_graph(jdata['total_pardom'], mylib.path_add(out_dir, 'par.svg'))
+        gen_graph(json['total_subdom'], mylib.path_add(out_dir, 'sub.svg'),
+                  overwrite=forceGraphs)
+        gen_graph(json['total_pardom'], mylib.path_add(out_dir, 'par.svg'),
+                  overwrite=forceGraphs)
     except KeyError:
         mylib.err('bundle-generate-page', 'skip: ' + bundle_id)
 
     with open(mylib.path_add(out_dir, 'index.html'), 'w') as fp:
-        fp.write(gen_html(bundle_id, jdata))
+        fp.write(gen_html(bundle_id, json))
     return needs_update_index
 
 
-def process(bundle_ids):
+def process(bundle_ids, forceGraphs=False):
     print('generating html pages ...')
     if bundle_ids == ['*']:
         bundle_ids = list(mylib.enum_appids())
@@ -101,7 +113,7 @@ def process(bundle_ids):
     ids_new_in_index = set()
     for bid in bundle_ids:
         print('  ' + bid)
-        if make_bundle_out(bid):
+        if make_bundle_out(bid, forceGraphs=forceGraphs):
             ids_new_in_index.add(bid)
     print('')
     return ids_new_in_index

src/html_root.py

@@ -24,9 +24,21 @@ def gen_root():
   </a>{}'''. format('')))
 
 
+def gen_search():
+    with open(mylib.path_out('redirect.html'), 'w') as fp:
+        fp.write(mylib.template_with_base('''
+<h2>Redirecting …</h2>
+<script type="text/javascript">
+  var GET={};
+  window.location.search.substr(1).split("&").forEach(function(x){GET[x.split("=")[0]]=x.split("=")[1]});
+  if (GET["id"]) { window.location = "/app/" + GET["id"] + "/index.html"; }
+</script>'''))
+
+
 def process():
     print('generating root html ...')
     gen_root()  # root index.thml
+    gen_search()  # root redirect.html?id=my.bundle.id
 
 
 if __name__ == '__main__':

src/lists/3rd-domains.txt

@@ -84,6 +84,31 @@ br.vet
 br.vlog
 br.wiki
 br.zlg
+co.a
+co.b
+co.com
+co.edu
+co.g
+co.gov
+co.inf
+co.m
+co.mil
+co.net
+co.ngo
+co.nom
+co.o
+co.org
+co.s
+co.t
+co.x
+co.y
+er.com
+er.edu
+er.gov
+er.mil
+er.net
+er.org
+er.ind
 es.com
 es.edu
 es.gob

src/lists/tracker_custom.txt

@@ -0,0 +1,7 @@
+adjust.com
+app-measurement.com
+criteo.com
+firebasedynamiclinks-ipv4.googleapis.com
+firebasedynamiclinks-ipv6.googleapis.com
+ioam.de
+userreport.com

src/lists/tracker_easylist_int.txt

@@ -0,0 +1,920 @@
+0stats.com
+109.169.66.161
+123-counter.de
+123compteur.com
+12mnkys.com
+180.76.2.18
+193.197.158.209
+1dmp.io
+212.95.32.75
+24log.de
+24log.ru
+24smi.info
+24smi.net
+2ip.ua
+2iui01.com
+2parale.ro
+2performant.com
+3ng6p6m0.de
+3wnp9.ru
+4stats.de
+50bang.org
+51yes.com
+7x4.fr
+7x5.fr
+99click.com
+a-counter.com.ua
+a-counter.kiev.ua
+abcompteur.com
+accessi.it
+accesstrade.net
+acs86.com
+actionallocator.com
+active-tracking.de
+active24stats.nl
+ad4m.at
+ad5track.com
+ad7.com
+adblockmetrics.ru
+adc-serv.net
+adclear.net
+adcrowd.com
+admatrix.jp
+admeira.ch
+admile.ru
+admo.tv
+adnz.co
+adop.cc
+adpick.co.kr
+adquality.ch
+adrank24.de
+adschoom.com
+adsettings.com
+adsixmedia.fr
+adskom.com
+adstat.4u.pl
+adten.eu
+adthletic.com
+adtimaserver.vn
+adtive.com
+adtlgc.com
+adtr.io
+adtraxx.de
+adtriba.com
+adx.com.ru
+adxvip.com
+af-z.jp
+affclkr.com
+affilbox.cz
+affilizr.com
+agillic.eu
+aicontents.net
+aid-ad.jp
+air360tracker.net
+algopine.com
+alkemics.com
+allo-media.net
+amarfa.ir
+amcdn.vn
+amoad.com
+amunx.de
+analights.com
+analysis.fi
+analyti.io
+analytic.rocks
+analytics-cdiscount.com
+analytics.rechtslupe.org
+analyticsip.net
+andersenit.dk
+andyhoppe.com
+announcement.ru
+anormal-tracker.de
+antvoice.com
+apkonline.ru
+appier.net
+asadcdn.com
+atraxio.com
+atsfi.de
+audiencemanager.de
+audiencesquare.com
+audienzz.ch
+audsp.com
+audtd.com
+avencio.de
+avstat.it
+backlink-test.de
+backlink-umsonst.de
+backlinkdino.de
+baifendian.com
+bekannt-im-web.de
+belboon.de
+beliebtestewebseite.de
+best-top.ro
+besucherstats.de
+besucherzaehler-counter.de
+besucherzaehler-homepage.de
+besucherzaehler-zugriffszaehler.de
+besucherzaehler.org
+besucherzahlen.com
+betarget.de
+bf-tools.net
+bid.run
+bidderrtb.com
+bigmining.com
+blacktri.com
+blog-o-rama.de
+blog-webkatalog.de
+blog104.com
+blogcounter.com
+blogcounter.de
+bloggeramt.de
+bloggerei.de
+blogranking.net
+blogtraffic.de
+blogtw.net
+bluecounter.de
+bonitrust.de
+bonnieradnetwork.se
+bonuscounter.de
+botdetector.ru
+botize.com
+botscanner.com
+brandmetrics.com
+browsiprod.com
+bumlam.com
+businessclick.ch
+buzzoole.com
+caanalytics.com
+call-tracking.by
+canalstat.com
+carts.guru
+casualstat.com
+ccrtvi.com
+cctvgb.com.cn
+cdnmaster.cn
+certifica.com
+cetlog.jp
+chart.dk
+checkeffect.at
+checkru.net
+cheqzone.com
+cifraclub.com.br
+circle.am
+citypaketet.se
+cityua.net
+clearsale.com.br
+clicki.cn
+clickmap.ch
+clickmatic.pl
+clubcollector.com
+cms.lv
+cnstats.ru
+cnzz.net
+coll1onf.com
+compteur-fr.com
+compteur-gratuit.org
+compteur-visite.com
+compteur.com
+compteur.org
+contadordevisitas.es
+contadorgratis.com
+contadorgratis.es
+contadorvisitasgratis.com
+contadorweb.com
+contatoreaccessi.com
+contentsfeed.com
+contineljs.com
+conversion.pl
+conversionlabs.net.pl
+cosmi.io
+count.fr
+count.im
+count24.de
+countar.de
+counter-city.de
+counter-go.de
+counter-gratis.com
+counter-kostenlos.info
+counter-kostenlos.net
+counter-treff.de
+counter.de
+counter.ok.ee
+counter27.ch
+countercity.net
+counterlevel.de
+counteronline.de
+counterseite.de
+counterserver.de
+counterstation.de
+counterstatistik.de
+counthis.com
+counti.de
+countino.de
+countit.ch
+countnow.de
+counto.de
+countok.de
+countus.fr
+countyou.de
+cpaevent.ru
+cpmktg.com
+cptrack.de
+cr-nielsen.com
+cssrvsync.com
+cszz.ru
+ctags.cn
+cuntador.com
+cya1t.net
+d-bi.fr
+d-markets.net
+d-stats.com
+d2-apps.net
+data.caroda.io
+datado.me
+datadome.co
+datamaster.com.cn
+dawin.tv
+dcmn.io
+de17a.com
+delidatax.net
+deliv.lexpress.fr
+denakop.com
+dep-x.com
+deteql.net
+df-srv.de
+die-rankliste.com
+digidip.net
+digital-metric.com
+digitaladvisor.dk
+digitiminimi.com
+dircont3.com
+directcounter.de
+directcrm.ru
+distribeo.com
+divolution.com
+dl8.me
+dmdi.pl
+dnab.info
+do09.net
+docodoco.jp
+dotmetrics.net
+dreamcounter.de
+durocount.com
+e-kaiseki.com
+e-kuzbass.ru
+eanalyzer.de
+early-birds.fr
+early-birds.io
+easy.lv
+easysol.net
+easytracking.de
+ebis.ne.jp
+ec-concier.com
+ec-optimizer.com
+eco-tag.jp
+econda-monitor.de
+edococounter.de
+edt02.net
+edtp.de
+efatik.me
+eltex.co.jp
+emailretargeting.com
+emarbox.com
+emetriq.de
+engageya.com
+enter-system.com
+erate.co.il
+eresmas.net
+erotikcounter.org
+estadisticasgratis.com
+estadisticasgratis.es
+etracker.de
+etracking24.de
+etrust.eu
+euro-pr.eu
+euroads.dk
+eurocounter.com
+ew3.io
+exapxl.de
+exe.bid
+exmarkt.de
+eyeota.net
+ezakus.net
+f-counter.jp
+f-counter.net
+facil-iti.com
+faibl.org
+fastcounter.de
+faststart.ru
+ferank.fr
+fixcounter.com
+flags.es
+fogl1onf.com
+fortvision.com
+fraudmetrix.cn
+free-counters.net
+freecounter.it
+freestat.ws
+freestats.biz
+freestats.net
+freestats.org
+freestats.tk
+freestats.tv
+freestats.ws
+freihit.de
+fremaks.net
+frosmo.com
+ftrack.ru
+fun-hits.com
+gacela.eu
+galaxiemedia.fr
+gallupnet.fi
+gdeslon.ru
+generaltracking.de
+genieedmp.com
+genieessp.jp
+geocompteur.com
+geocontatore.com
+geovisite.ovh
+getaim.info
+gezaehlt.de
+gft2.de
+ggxt.net
+giga-abs.de
+giraff.io
+gixmo.dk
+gm-it.consulting
+gm99.com
+gmodmp.jp
+gnezdo.ru
+google-rank.org
+goprediction.com
+gostats.cn
+gostats.de
+gostats.pl
+gostats.ru
+gostats.vn
+goutee.top
+gpr.hu
+graphinsider.com
+gratis-besucherzaehler.de
+gratis-counter-gratis.de
+greatviews.de
+grfz.de
+gridsum.com
+gridsumdissector.com
+growingio.com
+gsspcln.jp
+gtags.net
+gtop.ro
+he2d.com
+healte.de
+hetchi.com
+hiddencounter.de
+hiperstat.com
+hirmatrix.hu
+hit.copesa.cl
+hit100.ro
+hitcount.dk
+hitcountersonline.com
+hitgraph.jp
+hitmaster.de
+hitmir.ru
+hits.e.cl
+holder.com.ua
+hot-count.com
+hotcounter.de
+hotrank.com.tw
+hsdn.org
+hstrck.com
+hub.com.pl
+hubpd.com
+hung.ch
+hunkal.com
+i-mobile.co.jp
+i22lo.com
+i2ad.jp
+i2i.jp
+idntfy.ru
+idot.cz
+iid-network.jp
+iivt.com
+im-apps.net
+imcht.net
+imetrix.it
+imrk.net
+inaudium.com
+infocollect.dk
+infostroy.nnov.ru
+infox.sg
+ingenioustech.biz
+inistrack.net
+inpref.com
+inrd.ru
+instreamatic.com
+intelliad.de
+interactive-circle.jp
+interakt.ru
+interaktiv-net.de
+intergid.ru
+interhits.de
+intrastats.com
+iogous.com
+iolam.it
+ipcount.net
+ipfrom.com
+ipinyou.com
+irs09.com
+iryazan.ru
+istats.nl
+italianadirectory.com
+itop.cz
+ivitrack.com
+iyi.net
+jiankongbao.com
+jubiitag.dk
+kaizenplatform.net
+kavijaseuranta.fi
+kctag.net
+kdata.fr
+keytrack.de
+keyword-match.com
+keyxel.com
+kir.jp
+klamm-counter.de
+kmindex.ru
+kono-research.de
+kostenlose-counter.com
+kupona.de
+laserstat.com
+lddt.de
+lead-analytics.biz
+lead.im
+leadium.com
+leadslabpixels.net
+legenhit.com
+leiki.com
+lentainform.com
+leserservice-tracking.de
+letro.jp
+libstat.com
+link-empfehlen24.de
+linkwi.se
+listtop.ru
+livecounter.dk
+livestats.fr
+livetex.ru
+livewebstats.dk
+lndata.com
+logger.co.kr
+logly.co.jp
+logua.com
+logxp.ru
+logz.ru
+lookit.cz
+lookmy.info
+losecounter.de
+lugansk-info.ru
+lumitos.com
+luxup2.ru
+luxupadva.com
+luxupcdna.com
+luxupcdnc.com
+lwadm.com
+m-brain.fi
+mabaya.com
+macromill.com
+madnet.ru
+mairdumont.com
+marketing-page.de
+marktest.pt
+mastertag.effiliation.com
+mateti.net
+maxtraffic.com
+mb-srv.com
+mcloudglobal.com
+md-nx.com
+mediaplan.ru
+mediatoday.ru
+mediatraffic.com.ua
+mediav.com
+meetrics.net
+megast.at
+megavisites.com
+mengis-linden.org
+metalyzer.com
+metrigo.com
+metriweb.be
+mgid.com
+miaozhen.com
+micodigo.com
+microcounter.de
+midas-network.com
+mimgoal.com
+mindtake.com
+mmtro.com
+mobylog.jp
+modernus.is
+mokuz.ru
+monkeytracker.cz
+motorpresse-statistik.de
+mps-gba.de
+mpwe.net
+mr-rank.de
+msgs.jp
+mtrack.nl
+musiccounter.ru
+mwstats.net
+mxapis.com
+my-ranking.de
+my-stats.info
+mycounter.com.ua
+mycounter.ua
+mystat-in.net
+mystat.hu
+mystat.it
+mystats.nl
+mytopf.com
+myvisitors.se
+navrcholu.cz
+ncom.dk
+nepohita.com
+netagent.cz
+netcounter.de
+netdebit-counter.de
+netminers.dk
+netmonitor.fi
+netquattro.com
+netstats.dk
+netupdater.info
+netvigie.com
+netzaehler.de
+netzstat.ch
+nex8.net
+ngacm.com
+ngastatic.com
+ninestats.com
+nokaut.link
+nsaudience.pl
+ntlab.org
+oadz.com
+observare.de
+odoscope.cloud
+oevery.com
+oewabox.at
+oghub.io
+omiki.com
+on-line.lv
+onlinewebstat.com
+opentracking.ru
+optimierung-der-website.de
+organicfruitapps.com
+orquideassp.com
+osxau.de
+otclick-adv.ru
+ourstats.de
+owldata.com
+p0y.cn
+p24.hu
+page-hit.de
+pagerank-backlink.eu
+pagerank-linkverzeichnis.de
+pagerank-online.eu
+pagerank-suchmaschine.de
+pagerankfree.com
+pageranking-counter.de
+pagoda56.com
+parameter.dk
+pc-agency24.de
+pdmp.jp
+peakcounter.dk
+percycle.com
+performax.cz
+persianstat.com
+persianstat.ir
+phpstat.com
+phywi.org
+pimpmypr.de
+pingclock.net
+pixanalytics.com
+pladform.ru
+plexworks.de
+pmbox.biz
+pocitadlo.cz
+pocitadlo.sk
+polymorphicads.jp
+popin.cc
+potterpetrey.com
+powercount.com
+pp8.com
+ppdb.pl
+ppro.de
+pr-chart.com
+pr-chart.de
+pr-linktausch.de
+pr-sunshine.de
+prnetwork.de
+productsup.com
+proext.com
+profitshare.ro
+programmatic.cz
+props.id
+prospecteye.com
+prudsys-rde.de
+publish-int.se
+puls.lv
+pushdom.co
+pvmax.net
+qchannel03.cn
+qhupdate.com
+quant.jp
+quick-counter.net
+r-ad.ne.jp
+r.movad.de
+r24-tech.com
+rank-power.com
+rank4all.eu
+rankchamp.de
+ranking-charts.de
+ranking-counter.de
+ranking-hits.de
+ranking-links.de
+rankings24.de
+rawr.at
+rays-counter.com
+rdstation.com.br
+reachmax.cn
+realist.gen.tr
+redretarget.com
+refericon.pl
+refinedads.com
+reitingas.lt
+reitingi.lv
+rejestr.org
+relap.io
+rentracks.jp
+research-artisan.com
+research-int.se
+research.de.com
+reseau-pub.com
+reshin.de
+retag.xyz
+retargeter.com.br
+retargeting.biz
+rightstats.com
+ripost.services
+ritogaga.com
+rnet.plus
+rossel.tech
+royalcount.de
+rtoaster.jp
+ru.net
+rutarget.ru
+rvkcpqz.com
+sagetrc.com
+sambaads.com
+sare25.com
+sarimsol.com
+sarov.ws
+sas.com
+sayyac.com
+sayyac.net
+sblftg.com
+scriptil.com
+scupio.com
+scw.systems
+sedotracker.de
+seedtag.com
+seesaa.jp
+seitwert.de
+selfcampaign.com
+semantiqo.com
+semiocast.com
+semnicneposilejte.cz
+semtracker.de
+sensic.net
+sensor.org.ua
+seo-master.net
+serating.ru
+shinystat.it
+sibautomation.com
+sibulla.com
+sifomedia.se
+site-submit.com.ua
+sitebot.cn
+sitebro.de
+sitetistik.com
+sitizy.network
+sjv.io
+sk1n.fr
+sk8t.fr
+skylog.kz
+slogantrend.de
+smart-counter.net
+smartnews-ads.com
+snrbox.com
+socdm.com
+space-link.de
+spacehits.net
+specialstat.com
+speedcount.de
+speedcounter.net
+speedtracker.de
+speee-ad.jp
+spelar.org
+sphostserver.com
+spider-mich.com
+spolecznosci.net
+sponsorcounter.de
+spring-tns.net
+srvtrck.com
+stat-well.com
+stat.4u.pl
+stat.media
+stat.pl
+stat.www.fi
+stat24.ru
+static-fra.de
+statistiche-free.com
+statistiche.it
+statistiche.ws
+statistichegratis.net
+statistics.ro
+statistik-gallup.net
+statistika.lv
+statistiq.com
+stats.de
+stats.fr
+stats.lt
+stats4free.de
+statsforever.com
+stattds.club
+stealth.nl
+stetic.com
+suchmaschinen-ranking-hits.de
+sunios.de
+suntcontent.se
+superstat.info
+svtrd.com
+sync.tv
+synovite-scripts.com
+t4ft.de
+tagcdn.com
+tagmanager.cn
+tagtool.de
+tailtarget.com
+tamedia.ch
+tanx.com
+targetix.net
+taxel.jp
+tbex.ru
+tda.io
+tds.io
+team-rec.jp
+technical-service.net
+telemetric.dk
+tenmax.io
+tenping.kr
+tetigi.com
+tget.me
+tgknt.com
+thestat.net
+tidningsnatet.se
+tinyclues.com
+tinystat.ir
+tisoomi-services.com
+titag.com
+tnative.ru
+tns-gallup.dk
+tns-sifo.se
+toc.io
+tomonline-inc.com
+top-bloggers.com
+top-ro.ro
+top.lv
+tophits4u.de
+toplist.sk
+toplist100.org
+topsem.com
+topsite.lv
+topstat.com
+toptracker.ru
+tovery.net
+tpm.pw
+tracdelight.com
+tracdelight.io
+tracker.stats.in.th
+trackfeed.com
+tracking*.euroads.fi
+tracking.wlscripts.net
+trackset.it
+traffic4u.nl
+trafficmaxx.de
+trafikkfondet.no
+trafit.com
+trafix.ro
+trbo.com
+trendcounter.de
+trick17.it
+triver.jp
+trkme.net
+truehits.net
+truehits3.gits.net.th
+trugaze.io
+twcouponcenter.com
+txt.eu
+tyxo.bg
+uapoisk.net
+uarating.com
+ukw.jp
+uniconsent.com
+universaltrackingcontainer.com
+up-rank.com
+upravel.com
+uptolike.com
+urstats.de
+usage.seibert-media.io
+usemaxserver.de
+uzerly.net
+uzrating.com
+vamaker.com
+variti.net
+vdoing.com
+verypopularwebsite.com
+vidigital.ru
+viewar.org
+vihtori-analytics.fi
+vinsight.de
+vira.ru
+visilabs.net
+visitor-stats.de
+visits.lt
+vivistats.com
+vm5apis.com
+volgograd-info.ru
+vologda-info.ru
+volvelle.tech
+vtracy.de
+warlog.ru
+way2traffic.com
+wcfbc.net
+web-visor.com
+webads.eu
+webcompteur.com
+webdissector.com
+webest.info
+webgozar.com
+webgozar.ir
+webhits.de
+weblist.de
+weblog.com.ua
+webmeter.ws
+webmobile.ws
+webprospector.de
+webserviceaward.com
+webservis.gen.tr
+websitesampling.com
+webstat.no
+webstatistika.lv
+webtalking.ru
+webtrekk-us.net
+webtrekk.de
+webtrekk.net
+webttracking.de
+webturn.ru
+webvisor.com
+webvisor.ru
+wecount4u.com
+welt-der-links.de
+whoseesyou.com
+winitout.com
+wipe.de
+wonder-ma.com
+wos.lv
+wstatslive.com
+wwgate.ru
+www.hey.lt
+wysistat.com
+x-traceur.com
+xclaimwords.net
+xcounter.ch
+xhit.com
+xplosion.de
+xtgreat.com
+xtractor.no
+yektanet.com
+yhjinm.cn
+yieldbird.com
+yigao.com
+yoochoose.net
+youle55.com
+ywywjrlbsuv.com
+zaehler.tv
+zampda.net
+zero.kz
+zipstat.dk
+zirve100.com
+ziyu.net
+zmctrack.net
+zontera.com
+ztcadx.com

src/lists/tracker_exodus.txt

@@ -0,0 +1,580 @@
+247realmedia.com
+2mdn.net
+3lift.com
+4seeresults.com
+a.fiksu.com
+a.tiles.mapbox.com
+a4.tl
+aatkit.com
+abtasty.com
+accengage.com
+account.talkingdata.com
+acrcloud.com
+actv8technologies.com
+ad-brix.com
+ad.cauly.co.kr
+ad.crwdcntrl.net
+ad4game.com
+ad4push.com
+ad4screen.com
+adc3-launch.adcolony.com
+adcolony.com
+adform.com
+adformdsp.net
+adfurikun.jp
+adj.st
+adjust.com
+adlibr.com
+adm.dynamicyield.com
+admixer.co.kr
+adnxs.com
+adotsolution.com
+adproxy.fyber.com
+ads.adadapted.com
+ads.aerserv.com
+ads.api.vungle.com
+ads.glispa.com
+ads.heyzap.com
+ads.mobclix.com
+ads.pubmatic.com
+ads.rubiconproject.com
+ads.tapdaq.com
+ads30.adcolony.com
+adsafeprotected.com
+adserver.unityads.unity3d.com
+adsrvr.org
+adswizz.com
+adtech.de
+adtheorent.com
+adx.adform.net
+adzerk.com
+adzerk.net
+akamai.smartadserver.com
+akamai.vungle-cdn.vungle.com
+aktrack.pubmatic.com
+alog.umeng.com
+alogs.umeng.com
+amobee.com
+amplitude.com
+analytics-server.gimbal.com
+analytics.ad.daum.net
+analytics.foresee.com
+analytics.localytics.com
+analytics.mobile.yandex.net
+analytics.rayjump.com
+analytics.social.unity.com
+analytics.tapad.com
+analytics.yinzcam.com
+android-quinoa-config-prod.sense360eng.com
+androidads20.adcolony.com
+androidads21.adcolony.com
+androidads23.adcolony.com
+aos.wall.youmi.net
+aotter.net
+api-device.mocaplatform.com
+api.adflake.com
+api.airpush.com
+api.alphonso.tv
+api.altamob.com
+api.amplitude.com
+api.appsee.com
+api.apptentive.com
+api.beaconinside.com
+api.branch.io
+api.cloudmobi.net
+api.crittercism.com
+api.crowdtangle.com
+api.dynamicyield.com
+api.findgravy.com
+api.gimbal.com
+api.giphy.com
+api.hypertrack.com
+api.keen.io
+api.mixpanel.com
+api.mobpowertech.com
+api.otherlevels.com
+api.pingstart.com
+api.proximi.fi
+api.pushspring.com
+api.safegraph.com
+api.segment.io
+api.sentiance.com
+api.swrve.com
+api.taplytics.com
+api.tiles.mapbox.com
+api.trillbit.com
+api.uca.cloud.unity3d.com
+api.vungle.akadns.net
+api.vungle.com
+api.wonderpush.com
+apidm.airpush.com
+apistaging.airpush.com
+apiv2.moengage.com
+app-config.enhance.co
+app.opentracker.net
+app.roximity.com
+appboy.com
+appcelerator.com
+appcelerator.net
+appengage-video.fyber.com
+appengage-video.sponsorpay.com
+appload.ingest.crittercism.com
+applovin.com
+applvn.com
+appmetrica.yandex.com
+appnext.com
+appnexus.com
+appnexus.net
+appodeal.com
+appodealx.com
+apportal.airpush.com
+appsflyer.com
+appwall.api.airpush.com
+apx.moatads.com
+ar.umeng.com
+areametrics.com
+assets.tapad.com
+ati-host.net
+au.youmi.net
+auction.unityads.unity3d.com
+auditude.com
+av1.xdrig.com
+avazutracking.net
+avocarrot.com
+axonix.com
+backelite.com
+backtrace.io
+banner.fyber.com
+banners-slb.mobile.yandex.net
+banners.mobile.yandex.net
+batch.com
+bd.vungle.com
+beta.airpush.com
+billboard.vungle.com
+bm.adentifi.com
+brahe.apptimize.com
+bugly.qq.com
+c.betrad.com
+c.w.inmobi.com
+c2i.startappnetwork.com
+c2s.startappnetwork.com
+cdn-adn.rayjump.com
+cdn-api.admost.com
+cdn-api.swirl.com
+cdn-highwinds.unityads.unity3d.com
+cdn-lb.vungle.com
+cdn.adswizz.com.edgesuite.net
+cdn.dynamicyield.com
+cdn.gigya.com
+cdn.unityads.unity3d.com
+cdn1.gigya.com
+cdn1.smartadserver.com
+cdn1.sponsorpay.com
+cdn2.gigya.com
+cdn2.sponsorpay.com
+cdn3.gigya.com
+cdn3.sponsorpay.com
+cdn4.sponsorpay.com
+cdnap.airpush.com
+cdns.us1.gigya.com
+cedexis-radar.net
+cedexis.com
+chartbeat.com
+chartbeat.net
+chartboost.com
+china.inmobi.com
+ci.vungle.com
+click-haproxy.supersonicads.com
+click.dawin.tv
+click.startappservice.com
+cloud.xdrig.com
+cloudmobi.net
+cmcm.com
+colocator.net
+commander1.com
+comscore.com
+config-ltvp.inmobi.com
+config.inmobi.com
+config.uca.cloud.unity3d.com
+config.unityads.unity3d.com
+content.swrve.com
+contextual.media.net
+control.kochava.com
+conversantmedia.com
+crashlytics.com
+criteo.com
+ct.pinterest.com
+cuebiq.com
+cws.conviva.com
+cx.ssacdn.com
+data-location.enhance.co
+data.de.coremetrics.com
+data.mobclix.com
+data.vungle.com
+databerries.com
+dc.webtrends.com
+de.ioam.de
+de.rayjump.com
+de01.rayjump.com
+debug.aerserv.com
+decide.mixpanel.com
+deliver.oztam.com.au
+deltadna.net
+demdex.net
+detect.rayjump.com
+dev-api.persona.ly
+dev.dsp.persona.ly
+dev.persona.ly
+device-api.urbanairship.com
+devices.carnivalmobile.com
+diff.smartadserver.com
+diff2.smartadserver.com
+diff3.smartadserver.com
+dispatcher.mng-ads.com
+display.io
+dmtry.com
+doubleclick.com
+doubleclick.net
+dov-e.com
+dsp.persona.ly
+dts.startappservice.com
+duapps.com
+dynatrace.com
+e-ssl.apsalar.com
+e.apsalar.com
+ehawk.com
+engine.adzerk.net
+engine.fyber.com
+engine.sponsorpay.com
+eqx.smartadserver.com
+et.w.inmobi.com
+etl.tindersparks.com
+eulerian.com
+event.split.io
+events.startappservice.com
+events.uber.com
+events3alt.adcolony.com
+evt.tamoco.com
+exp.glispa.com
+facebook.com
+fastlane.rubiconproject.com
+fidzup
+files.adform.net
+firebase.com
+fk-mtrack.rayjump.com
+flurry.com
+foresee.com
+foreseeresults.com
+frm.acuant.net
+fwmrm.net
+fyc.heyzap.com
+g.jwpsrv.com
+gads.pubmatic.com
+gallery.smartadserver.com
+gemius.pl
+geo.moatads.com
+geodata.otherlevels.com
+ginf.adfurikun.jp
+glympse.com
+go.admost.com
+google-analytics.com
+google.com
+graph.accountkit.com
+hb-minify-juc1ugur1qwqqqo4.stackpathdns.com
+heapanalytics.com
+helpshift.com
+hockeyapp.net
+houndify.com
+hybird.rayjump.com
+hypertrack.amazonaws.com
+hyprmx.com
+i.4see.mobi
+i.l.inmobicdn.net
+i.w.inmobi.com
+iasds01.com
+im2.smartadserver.com
+image2.pubmatic.com
+images.startappservice.com
+img-cloudflare-2.haizap.com
+img-cloudflare.haizap.com
+imp.startappservice.com
+incoming-data-sense360.s3.amazonaws.com
+info.static.startappservice.com
+ingest.vungle.com
+init.startappservice.com
+init.supersonicads.com
+inlocomedia.com
+inmobi.cn
+inmobi.com
+inmobi.info
+inmobi.net
+inmobi.us
+inmobicdn.com
+inmobicdn.net
+inmobisdk-a.akamaihd.net
+inrix.com
+inrix.io
+insight.adsrvr.org
+instreamatic.com
+integralads.com
+interstitial.fyber.com
+ipsws.indooratlas.com
+itx5-publicidad.smartadserver.com
+itx5.smartadserver.com
+jaeger.vungle.com
+japan.inmobi.com
+js-agent.newrelic.com
+js.adsrvr.org
+js.moatads.com
+jssdk.rayjump.com
+jwpltx.com
+kiip.me
+kontakt.io
+krxd.net
+kvinit-prod.api.kochava.com
+leanplum.com
+ligatus.com
+lisnr.com
+loadus.exelator.com
+locuslabs.com
+log.mobpowertech.com
+log.pinterest.com
+logger.cloudmobi.net
+loggly.com
+login.microsoftonline.com
+logs.supersonic.com
+ltv-data-api.kube-prod.vungle.com
+m.airpush.com
+m.talkingdata.com
+m2m-api.inmarket.com
+management.azure.com
+manifest.localytics.com
+map.baidu.com
+map.qq.com
+marketo.com
+marketo.net
+mars.telequid.com
+match.adsrvr.org
+matomo.org
+mb.moatads.com
+mbe-cdn.fyber.com
+mc.yandex.ru
+md-a-c.apptimize.com
+md-a-s.apptimize.com
+mdn.otherlevels.com
+med-api.admost.com
+med.heyzap.com
+media.net
+medicscan.acuant.net
+metrics.adflake.com
+metrics.brightcove.com
+millennialmedia.com
+mixpanel.com
+mktoedge.com
+mktossl.com
+mng-ads.com
+moat.com
+mobclix.com
+mobile-collector.newrelic.com
+mobile.mng-ads.com
+mobileanalytics.us-east-1.amazonaws.com
+mobileapptracking.com
+mobpowertech.com
+mobvista.com
+monitoring.vungle.com
+moodpresence.com
+mopub.com
+mparticle.com
+mqtt.evrythng.com
+nativex.com
+net.rayjump.com
+newrelic.com
+nexage.com
+nexus.ensighten.com
+nr-data.net
+oc.umeng.com
+offer.fyber.com
+omniture.com
+omtrdc.net
+onesignal.com
+online.rayjump.com
+ooyala.com
+openx.com
+openx.net
+openx.org
+optimized-by.rubiconproject.com
+optimizely.com
+outbrain.com
+outcome.supersonicads.com
+ow-gateway.supersonicads.com
+p.jwpcdn.com
+partner-service.link
+persona.ly
+ping.tapylitics.com
+pingback.giphy.com
+pixel-tracking.sonic-us.supersonicads.com
+pixel.mathtag.com
+pixel.moatads.com
+pixel.rubiconproject.com
+pixel.tapad.com
+presage.io
+preview.smartadserver.com
+profile.localytics.com
+prov.alphonso.tv
+proximitykit.radiusnetworks.com
+pubnative.net
+push.xdrig.com
+pushwoosh.com
+px.dynamicyield.com
+px.moatads.com
+pyze.com
+qualtrics.com
+quantcast.com
+quantcast.net
+quinoa-personal-identify-prod.sense360eng.com
+r.w.inmobi.com
+radar.cedexis.com
+rayjump.com
+rec.replay.answerscloud.com
+recommender.scarabresearch.com
+registration.gimbal.com
+report.appmetrica.yandex.net
+req.startappservice.com
+rich.otherlevels.com
+rtb-csync.smartadserver.com
+rtb.adentifi.com
+rtb.persona.ly
+rtb.platform.glispa.com
+rv-gateway.supersonicads.com
+s.mobclix.com
+s4m.io
+sam4m.com
+saspreview.com
+scandit.com
+scheme.mobpowertech.com
+schibsted.com
+schibsted.io
+scorecardresearch.com
+sdk-assets.localytics.com
+sdk-info.gimbal.com
+sdk.adbuddiz.com
+sdk.adincube.com
+sdk.adotmob.com
+sdk.appbrain.com
+sdk.fiksu.com
+sdk.follow-apps.com
+sdk.foursquare.com
+sdk.gpshopper.com
+sdk.persona.ly
+sdk.shopkick.com
+sdk.split.io
+sdk6.ibm.xtify.com
+sdkm.w.inmobi.com
+sdktm.w.inmobi.com
+segment.com
+sejs.moatads.com
+service.fyber.com
+services.assureid.net
+serving-sys.com
+setting.rayjump.com
+sg-mtrack.rayjump.com
+sg.rayjump.com
+sg01.rayjump.com
+shopkick.com
+shopkick.de
+signal360.com
+silverpush.co
+silverpush.com
+simage2.pubmatic.com
+singlespot.com
+smaato.net
+smartadserver.com
+smartadserver.ru
+smartlook.com
+soda.startappservice.com
+soma.smaato.net
+sonicnotify.com
+soom.la
+ssl.vungle.com
+st.dynamicyield.com
+stags.bluekai.com
+startappservice.com
+startup.mobile.yandex.net
+static.adzerk.net
+static.ssacdn.com
+statistics.videofarm.daum.net
+stats.aws.rubiconproject.com
+stats.unity3d.com
+supersonic.com
+supersonicads-a.akamaihd.net
+switchboard.mixpanel.com
+sync.adotmob.com
+sync2ad.com
+synerise.com
+sypi.gpshopper.com
+taboola.com
+tag-mediation.supersonic.com
+tags.bluekai.com
+tags.otherlevels.com
+tap2-cdn.rubiconproject.com
+tapestry.tapad.com
+tapjoy.com
+tapjoyads.com
+target.my.com
+teads.tv
+tealiumiq.com
+tele.fm
+templates.glispaconnect.com
+tiqcdn.com
+tknet.rayjump.com
+tmk.smartadserver.com
+track.adform.net
+tracker-api.my.com
+tracker.adotmob.com
+tracker.fyber.com
+trck.at
+trk.glispa.com
+trk.pinterest.com
+tusdk.com
+txn.ingest.crittercism.com
+ua.supersonicads.com
+ultimedia.com
+umeng.com
+uop.umeng.com
+urbanairship.com
+us-ads.openx.net
+us01.rayjump.com
+usw-lax.adsrvr.org
+v.ssacdn.com
+v.vungle.com
+va.origin.startappservice.com
+vast.cloudmobi.net
+vectaury.io
+video-ads.rubiconproject.com
+video-interstitial-assets-cdn.fyber.com
+video.fyber.com
+w.inmobi.com
+wd.adcolony.com
+weborama.fr
+weborama.net
+webtrends.com
+webview.unityads.unity3d.com
+widespace.com
+wootric.com
+wootric.com.herokudns.com
+ws.findgravy.com
+ws.otherlevels.com
+www.5rocks.io
+www.googletagmanager.com
+www.googletagservices.com
+www.supersonicads.com
+wzrkt.com
+x.heyzap.com
+yoc-performance.com
+yoc.com
+youmi.net
+yt.moatads.com
+yts.moatads.com
+z.moatads.com
+zapr.in
+zucks.net

src/main.py

@@ -1,23 +1,13 @@
 #!/usr/bin/env python3
 
+import os
 import common_lib as mylib
-import bundle_import
 import bundle_combine
 import bundle_download
 import html_root
 import html_index
 import html_bundle
-
-
-def reset_new():
-    print("RESET json files ...")
-    prefix = mylib.path_len(mylib.path_data())
-    for bid in mylib.enum_appids():
-        for src, _ in mylib.enum_jsons(bid):
-            frmt = mylib.path_add(mylib.path_data('_in', '%s.json'))
-            dest = bundle_import.next_path(frmt)
-            mylib.mv(src, dest, printOmitPrefix=prefix)
-    print('')
+import tracker_download
 
 
 def del_id(bundle_ids):
@@ -31,25 +21,51 @@ def del_id(bundle_ids):
             html_index.process()
 
 
-def full_chain(force=False):
-    bundle_ids = bundle_import.process()
-    if force:
-        bundle_ids = list(mylib.enum_data_appids())
-    if len(bundle_ids) > 0:
-        bundle_combine.process(bundle_ids)
-        new_ids = html_bundle.process(bundle_ids)
-        if len(new_ids) > 0:
-            bundle_download.process(new_ids)
-            html_index.process()
-            html_root.process()
+def combine_and_update(bundle_ids, where=None, forceGraphs=False):
+    affected = bundle_combine.process(bundle_ids, where=where)
+    if len(affected) == 0:
+        print('no bundle affected by tracker, not generating bundle html')
+        return
+    new_ids = html_bundle.process(affected, forceGraphs=forceGraphs)
+    if len(new_ids) == 0:
+        print('no new bundle, not rebuilding index')
+        return
+    bundle_download.process(new_ids)
+    html_index.process()
+    html_root.process()
+
+
+def import_update():
+    print('checking incoming data ...')
+    needs_update = set()
+    for fname, bid in mylib.enum_newly_added():
+        if bid == '_manually':
+            # TODO: notify admin that manual action is required
+            mylib.err('import', 'manual action required!')
         else:
-            print('none to import, not rebuilding index')
+            print('  ' + bid)
+            needs_update.add(bid)
+        os.remove(fname)
+    print('')
+    if len(needs_update) > 0:
+        combine_and_update(needs_update, forceGraphs=True)
+
+
+def tracker_update():
+    new_trackers = tracker_download.process()
+    if new_trackers:
+        combine_and_update(['*'], where=new_trackers)
 
 
 def process():
-    # reset_new()
-    # del_id(['*'])
-    full_chain(force=False)
+    # del_id(['_manually'])
+    import_update()
+    # tracker_update()
+    # tracker_download.combine_all('x')
+    # combine_and_update(['*'])  # where=['test.com']
+    # if bundle_download.download_missing_icons(force=False):
+    #     html_index.process()
+    # html_index.process()
 
 
 process()

src/tracker_download.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+import common_lib as mylib
+# import hashlib
+
+known_trackers = None
+
+
+# def md5(fname):
+#     hash_md5 = hashlib.md5()
+#     with open(fname, 'rb') as f:
+#         for chunk in iter(lambda: f.read(4096), b''):
+#             hash_md5.update(chunk)
+#     return hash_md5.hexdigest()
+
+
+def save_list(result_set, fname, binary=False):
+    if not result_set:
+        return False
+    out = mylib.path_root('src', 'lists', fname)
+    with open(out + '_tmp', 'wb' if binary else 'w') as fp:
+        end = b'\n' if binary else '\n'
+        for domain in sorted(result_set):
+            fp.write(domain + end)
+    try:
+        changes = mylib.diff_files(out, out + '_tmp')
+    except Exception:
+        changes = list(result_set)
+    mylib.mv(out + '_tmp', out)
+    # md5_old = md5(out) if mylib.file_exists(out) else None
+    # md5_new = md5(out)
+    if changes:
+        print('  updating: ' + fname)
+    else:
+        print('  no-change: ' + fname)
+    return changes
+
+
+def enum_lines(url, ignore=None):
+    whole = mylib.download(url)
+    for line in whole.split(b'\n'):
+        if not line or ignore and line.startswith(ignore):
+            continue
+        yield line
+
+
+def github(path):
+    return 'https://raw.githubusercontent.com/' + path
+
+
+def lockdown(fname, urlname):
+    url = github('confirmedcode/lockdown-ios/master/LockdowniOS/') + urlname
+    return save_list(set(enum_lines(url)), fname, binary=True)
+
+
+def easylist(fname, urlname):
+    url = github('easylist/easylist/master/easyprivacy/') + urlname
+    res = set()
+    for x in enum_lines(url, b'!'):
+        if not x.startswith(b'||'):
+            continue
+        x = x[2:]
+        parts = x.split(b'^')
+        if len(parts) == 1:
+            parts = x.split(b'$')
+        res.add(parts[0].split(b'/')[0])
+    return save_list(res, fname, binary=True)
+
+
+def lowe(fname):
+    res = set()
+    for x in enum_lines('https://pgl.yoyo.org/adservers/serverlist.php'
+                        '?hostformat=hosts&mimetype=plaintext', b'#'):
+        p = x.split()
+        if len(p) != 2:
+            mylib.err('tracker-list', 'Lowe: parsing error')
+            continue
+        res.add(p[1])
+    return save_list(res, fname, binary=True)
+
+
+def exodus(fname):
+    res = set()
+    url = 'https://etip.exodus-privacy.eu.org/trackers/export'
+    json = mylib.download(url, isJSON=True)
+    try:
+        for entry in json['trackers']:
+            net = entry['network_signature']
+            if not net:
+                continue
+            net = net.replace('\\.', '.').replace('\\-', '-')
+            for dom in net.split('|'):
+                if dom[-1] in '/.':
+                    continue
+                if dom[0] in '\\.':
+                    dom = dom[1:]
+                res.add(dom)
+    except KeyError:
+        pass
+    return save_list(res, fname, binary=False)
+
+
+def is_tracker(domain):
+    global known_trackers
+    if not known_trackers:
+        known_trackers = mylib.read_list('tracker_all.txt')
+    return mylib.bintree_lookup(known_trackers, domain[::-1])
+
+
+def combine_all(changes):
+    final = mylib.path_root('src', 'lists', 'tracker_all.txt')
+    if changes or not mylib.file_exists(final):
+        print('  updating: tracker_all.txt')
+    else:
+        print('  no-change: tracker_all.txt')
+        return
+    res = set()
+    for fname in ['custom.txt', 'lowe.txt', 'easylist.txt', 'easylist_int.txt',
+                  'exodus.txt', 'lockdown_clickbait.txt',
+                  'lockdown_marketing.txt', 'lockdown_game_ads.txt']:
+        for dom in mylib.read_list('tracker_' + fname):
+            if dom == 'google.com':
+                continue  # added by exodus, not a tracker per se
+            res.add(dom[::-1])  # reverse for bintree lookup
+    with open(final, 'w') as fp:
+        for domain in sorted(res):
+            fp.write(domain + '\n')
+
+
+def process():
+    print('downloading tracker domains ...')
+    changes = []
+    changes += lowe('tracker_lowe.txt')
+    changes += easylist('tracker_easylist.txt',
+                        'easyprivacy_trackingservers.txt')
+    changes += easylist('tracker_easylist_int.txt',
+                        'easyprivacy_trackingservers_international.txt')
+    changes += exodus('tracker_exodus.txt')
+    # changes += lockdown('tracker_lockdown_clickbait.txt', 'clickbait.txt')
+    # changes += lockdown('tracker_lockdown_marketing.txt', 'marketing.txt')
+    # changes += lockdown('tracker_lockdown_game_ads.txt', 'game_ads.txt')
+    combine_all(changes)
+    print('')
+    return changes
+
+
+if __name__ == "__main__":
+    # combine_all()
+    process()