diff --git a/out/static/lookup-rank.js b/out/static/lookup-rank.js
index c462b74..f92b45c 100644
--- a/out/static/lookup-rank.js
+++ b/out/static/lookup-rank.js
@@ -12,7 +12,7 @@ function lookup_rank_js(bundle_id) {
function update(i, id, fmt=String) {
let r = (rank[i] - 1) / (rank_max - 1);
let target = document.getElementById(id);
- let bar = target.querySelector('.percentile');
+ let bar = target.querySelector('.pcbar');
bar.classList.add(r < 0.5 ? 'g' : 'b');
bar.firstChild.style.left = r * 100 + '%';
let meta = target.lastElementChild.children;
diff --git a/out/static/style.css b/out/static/style.css
index add9759..f25bf64 100644
--- a/out/static/style.css
+++ b/out/static/style.css
@@ -15,7 +15,7 @@ a.no-ul, a.no-ul:hover, .no-ul-all a, .no-ul-all a:hover {
main, footer { padding: 0 1em; }
header, main, footer > div {
margin: 0 auto;
- max-width: 1120px; /*1307px*/
+ max-width: 1118px;
}
header { height: 50px; }
header img { vertical-align: top; padding: 0 7px; }
@@ -75,21 +75,21 @@ footer .links {
}
.dropdown:hover nav { display: block; }
.dropdown a { display: block; padding: .5em 1em; }
-.dropdown a:hover { background-color: #eee; }
#app-toc div:hover, .dropdown:hover button, .dropdown a:hover {
- background: #BBC6CA;
+ background: #DDD;
}
/* app index */
+#app-toc {
+ display: grid;
+ grid-gap: 10px;
+ grid-template-columns: repeat(auto-fill, minmax(178px, 1fr));
+}
#app-toc a { text-align: center; }
#app-toc div {
- display: inline-block;
- width: 140px;
height: 12em;
- margin: 5px;
padding: 16px;
- vertical-align: top;
overflow: hidden;
word-wrap: break-word;
background: #eee;
@@ -107,22 +107,18 @@ footer .links {
#app-toc span.name { font-size: .8em; font-weight: bold; }
#app-toc span.detail { font-size: .7em; }
-#pagination { text-align: center; margin-top: 2em; }
-#pagination a { margin: .5em; padding: .2em; }
-#pagination a.active { border: 1pt solid black; border-radius: .2em; }
+.pagination { text-align: center; margin-top: 2em; }
+.pagination a { margin: .5em; padding: .2em; }
+.pagination a.active { border: 1pt solid black; border-radius: .2em; }
/* domain index */
-#dom-toc h3 {
- position: sticky;
- top: 0;
- background: #fff;
- padding-bottom: 4px;
-}
-#dom-toc a, #dom-top10 a { word-wrap: break-word; }
+#dom-top10 { text-align: right; }
+#dom-top10>div { margin: .4em; }
+#dom-top10 a, #dom-toc a { word-wrap: break-word; }
#dom-toc span { display: table; }
.found-in span, .snd { color: #586472; font-size: .85em; }
-.loadbar {
+.fillbar {
display: block;
background: #DDD;
width: 200px;
@@ -130,14 +126,15 @@ footer .links {
border-radius: 4px;
text-align: left;
}
-.loadbar span {
+.fillbar>i {
+ font-style: normal;
display: inline-block;
border-radius: 4px 0 0 4px;
background: #AC2B4A;
font-size: .8em;
- padding: 2px 0 2px 0;
text-align: center;
color: #FFF;
+ line-height: 1.8em;
}
@@ -149,8 +146,10 @@ p.subtitle { margin-top: .2em; }
.mg_top { margin-top: 2em; }
.right { text-align: right; }
.center { text-align: center; }
-.bg1 { background: #eee; }
-.border { border: 1pt solid #ccc; }
+.bg1 { background: #EEE; }
+.border { border: 1pt solid #CCC; }
+.large { font-size: 1.2em; }
+.stick-top { top: 0; position: sticky; padding: .8em 0 .5em; background: #FFF; }
/*#meta { margin-bottom: 2em; }*/
#meta .icons { margin-bottom: 2em; }
@@ -166,9 +165,9 @@ p.subtitle { margin-top: .2em; }
margin: 2em 0;
}
#stats .col1 { grid-column-start: 1; }
-#stats>div>h4 { margin: 0 0 .7em; }
-#stats>div>p { margin-top: .5em; }
-.percentile {
+.rank h4 { margin: 0 0 .7em; }
+.rank p { margin-top: .5em; }
+.pcbar {
display: inline-block;
background: #EEE;
border: 1px solid #000;
@@ -177,14 +176,15 @@ p.subtitle { margin-top: .2em; }
padding-right: 3px;
vertical-align: top;
}
-.percentile div {
+.pcbar>i {
+ display: block;
position: relative;
background: #000;
width: 3px;
height: 100%;
}
-.percentile.b div { background: #CA0D3A; }
-.percentile.g div { background: #6AC45C; }
+.pcbar.b>i { background: #CA0D3A; }
+.pcbar.g>i { background: #6AC45C; }
/* app bundle: domain tags */
.tags a {
@@ -198,7 +198,14 @@ p.subtitle { margin-top: .2em; }
display: inline-block;
margin: .12em;
}
-.tags a.trckr, .tags.trckr a { background: #F9A7A7;; border-color: #B06363; }
+.tags a:hover { background: #DDD; }
+.tags.large > * {
+ border-radius: .4em;
+ padding: 6pt 12pt;
+ margin: .36em;
+}
+.tags a.trckr, .tags.trckr a { background: #F9A7A7; border-color: #B06363; }
+.tags a.trckr:hover, .tags.trckr a:hover { background: #F99494; }
p.trckr { font-size: .9em; margin-left: .5em; }
/* app bundle: graphs */
@@ -253,10 +260,9 @@ p.trckr { font-size: .9em; margin-left: .5em; }
header h1 span { display: none; } /* header subtitle */
main { padding-left: 1em; padding-right: 1em; }
footer .col3 div { width: 100%; padding: 0; } /* 3 columns */
+ #app-toc { grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); }
#app-toc a { text-align: left; }
#app-toc div {
- display: inline-block;
- width: 100%;
margin: 0;
padding: .7em 0;
height: unset;
@@ -267,6 +273,9 @@ p.trckr { font-size: .9em; margin-left: .5em; }
float: left; width: 44px; height: 44px; margin: 0 .5em;
}
#stats { grid-template-columns: max-content; }
+ #dom-top10 { text-align: unset; }
+ .fillbar { width: 100%; }
+ .fillbar>i { line-height: 2.5em; }
}
@media(min-width: 651px) {
#meta .icons { float: right; }
@@ -282,8 +291,6 @@ p.trckr { font-size: .9em; margin-left: .5em; }
width: 40%;
margin-left: 1%;
}
- #dom-top10 { text-align: right; }
- #dom-top10 p { margin: .4em; }
.div-center { margin: 0 auto; width: max-content; max-width: 100%; }
- .loadbar { display: inline-block; }
+ .fillbar { display: inline-block; }
}
diff --git a/src/download_itunes.py b/src/download_itunes.py
index 9b4a9bf..d5ad19e 100755
--- a/src/download_itunes.py
+++ b/src/download_itunes.py
@@ -14,30 +14,23 @@ def read_from_disk(bundle_id, lang):
return mylib.json_read(fname_for(bundle_id, lang))
-def read_first_from_disk(bundle_id, langs=AVAILABLE_LANGS):
- for lang in langs:
- if mylib.file_exists(fname_for(bundle_id, lang)):
- return read_from_disk(bundle_id, lang)
- return None
-
-
-def app_names(bundle_id):
- def name_for(lang):
- try:
- return read_from_disk(bundle_id, lang)['trackCensoredName']
- except Exception:
- return None
- ret = {}
+def enum_all_from_disk(bundle_id):
for lang in AVAILABLE_LANGS:
- name = name_for(lang)
- if name:
- ret[lang] = name
- return ret
+ try:
+ yield lang, read_from_disk(bundle_id, lang)
+ except Exception:
+ pass
-def get_genres(bundle_id, langs=AVAILABLE_LANGS):
- json = read_first_from_disk(bundle_id, langs=langs)
- return list(zip(json['genreIds'], json['genres'])) if json else []
+def get_app_names(bundle_id):
+ return {lang: json['trackCensoredName']
+ for lang, json in enum_all_from_disk(bundle_id)}
+
+
+def enum_genres(bundle_id):
+ for lang, json in enum_all_from_disk(bundle_id):
+ for gid, name in zip(json['genreIds'], json['genres']):
+ yield lang, gid, name
def download_info(bundle_id, lang, force=False):
diff --git a/src/html_bundle.py b/src/html_bundle.py
index 2ab7ec1..4904af2 100755
--- a/src/html_bundle.py
+++ b/src/html_bundle.py
@@ -1,76 +1,51 @@
#!/usr/bin/env python3
import sys
-import time
-import math
import common_lib as mylib
-import download_itunes # get_genres
+import lib_graphs as Graph
+import lib_html as HTML
import bundle_combine # get_evaluated, fname_evaluated
import index_app_names # get_name
+import index_categories # get_categories
-def gen_dotgraph(sorted_arr):
- txt = ''
- for name, count, mark in sorted_arr:
- title = '{} ({})'.format(name, count) if count > 1 else name
- clss = ' class="trckr"' if mark else ''
- txt += ' {1}
* Potential trackers are highlighted
' - return ''.format( - 'trckr ' if onlyTrackers else '', txt, note if anyMark else '') + src += fn_a_html(name, domain_w_count(name, count), + attr_str=trkr_if(mark and not onlyTrackers)) + ' ' + if src: + if anyMark: + src += '* Potential trackers are highlighted
' + clss = ' trckr' if onlyTrackers else '' + return f'' else: return '– None –' -def gen_html(bundle_id, obj): +def gen_dotgraph(arr): + return Graph.dotgraph([(domain_w_count(title, num), num, trkr_if(f)) + for title, num, f in arr]) + + +def stat(col, title, ident, value, optional=None): + return Graph.rank_tile(title, value, optional, { + 'id': ident, 'class': 'col' + str(col)}) + + +def gen_page(bundle_id, obj): def round_num(num): return format(num, '.1f') # .rstrip('0').rstrip('.') @@ -81,48 +56,29 @@ def gen_html(bundle_id, obj): def as_percent(value): return round_num(value * 100) + '%' - def as_date(value): - return ''.format( - time.strftime('%Y-%m-%d %H:%M', time.gmtime(value)), - time.strftime('%Y-%m-%d, %H:%M', time.gmtime(value)) - ) - def seconds_to_time(seconds): seconds = int(seconds) minutes, seconds = divmod(seconds, 60) hours, minutes = divmod(minutes, 60) return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds) - def stat(col, title, ident, value, optional=None): - if optional: - value += '({})'.format(optional) - return ''' -- Rank: ?, - best: ?, - worst: ?
-Bundle-id:{ bundle_id }
| App Categories: | { - ', '.join([name for i, name in gernes]) + ', '.join([HTML.a_category(i, name) for i, name in gernes]) } |
| Last Update: | {as_date(obj['last_date'])} |
| Last Update: | {HTML.date_utc(obj['last_date'])} |
Download: json
@@ -156,11 +112,8 @@ def gen_html(bundle_id, obj): def process(bundle_ids): print('generating html: apps ...') for bid in mylib.appids_in_out(bundle_ids): - print(' ' + bid) - mylib.mkdir_out_app(bid) - json = bundle_combine.get_evaluated(bid) - with open(mylib.path_out_app(bid, 'index.html'), 'w') as fp: - fp.write(gen_html(bid, json)) + # print(' ' + bid) + gen_page(bid, bundle_combine.get_evaluated(bid)) mylib.symlink(bundle_combine.fname_evaluated(bid), mylib.path_out_app(bid, 'data.json')) print('') diff --git a/src/html_categories.py b/src/html_categories.py new file mode 100755 index 0000000..291ec6e --- /dev/null +++ b/src/html_categories.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import common_lib as mylib +import lib_html as HTML +import index_categories # enum_all_categories + + +def process(per_page=60): + print('generating html: category-index ...') + base = mylib.path_out('category') + parent = 'All Categories' + arr = [] + for cid, cat, apps in sorted(index_categories.enum_all_categories(), + key=lambda x: x[1].lower()): + arr.append((cid, cat)) + pre = HTML.h2(HTML.a_path([(parent, '../')], cat)) + _, a = HTML.write_app_pages(mylib.path_add(base, cid), apps, cat, + per_page, pre=pre) + print(' {} ({})'.format(cat, a)) + + src = ''.join([HTML.a(n, '{}/'.format(cid)) for cid, n in arr]) + HTML.write(base, ''' +{dom_str} {pct_bar}
' - fp.write(mylib.template_with_base(txt + ''' -Get full list -sorted by Occurrence frequency -or in Alphabetical order.
+Get full list sorted by + Occurrence frequency or in + Alphabetical order. +
Download: json
-''', title=title)) +''', title=title) -def gen_html_trinity(json, idx_dir, app_count, title): +def gen_html_trinity(idx_dir, app_count, json, title, symlink): + list1 = [(dom, len(ids)) for dom, ids in json['subdom'].items()] + list2 = [(dom, len(ids)) for dom, ids in json['pardom'].items()] + + def write_index(fname, title, button): + HTML.write(idx_dir, 'Present in: … applications
- The AppCheck database currently contains {:,} apps with a total of {:,} unique domains. -
-- Collected through {:,} recordings with {:,} individual requests. -
+The AppCheck database currently contains {:,} apps with a total of {:,} unique domains.
+Collected through {:,} recordings with {:,} individual requests.
@@ -24,7 +24,7 @@ def gen_root():
- If you're just interested in the results, go ahead to see all apps. + If you're just interested in the results, go ahead to see all apps.
@@ -32,10 +32,11 @@ def gen_root(): For mor infos follow this link.
With the release of iOS 14 some Privacy features are put into the spotlight. @@ -53,58 +54,51 @@ def gen_help():
| App Name | pre iOS 14 | post iOS 14 | |
|---|---|---|---|
| {} | -{} Download from AppStore | -{} | -{} | -
Go back to start page
''')) +Go back to start page
''', fname='404.html') def process(): diff --git a/src/index_app_names.py b/src/index_app_names.py index 67ffc35..9da7439 100755 --- a/src/index_app_names.py +++ b/src/index_app_names.py @@ -48,7 +48,7 @@ def process(bundle_ids): load_json_if_not_already() did_change = False for bid in mylib.appids_in_data(bundle_ids): - names = download_itunes.app_names(bid) + names = download_itunes.get_app_names(bid) if not names: mylib.err('index-app-names', 'could not load: {}'.format(bid)) continue diff --git a/src/index_categories.py b/src/index_categories.py new file mode 100755 index 0000000..4bc347a --- /dev/null +++ b/src/index_categories.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +import sys +import common_lib as mylib +import download_itunes # get_genres + +_dict_apps = None +_dict_names = None + + +def fname_app_categories(): + return mylib.path_data_index('app_categories.json') + + +def fname_category_names(): + return mylib.path_data_index('category_names.json') + + +def load_json_if_not_already(): + def load_json_from_disk(fname): + return mylib.json_read(fname) if mylib.file_exists(fname) else {} + + global _dict_apps, _dict_names + if not _dict_apps: + _dict_apps = load_json_from_disk(fname_app_categories()) + if not _dict_names: + _dict_names = load_json_from_disk(fname_category_names()) + + +def try_update_app(bid, genre_ids): + try: + if _dict_apps[bid] == genre_ids: + return False + except KeyError: + pass + _dict_apps[bid] = genre_ids + return True + + +def try_update_name(gid, lang, name): + try: + _dict_names[gid] + except KeyError: + _dict_names[gid] = {} + try: + if _dict_names[gid][lang]: + return False # key already exists + except KeyError: + pass + _dict_names[gid][lang] = name + return True # updated, need to persist changes + + +def reset_index(): + global _dict_apps + print(' full reset') + mylib.rm_file(fname_app_categories()) # rebuild from ground up + _dict_apps = None + + +def try_persist_changes(flag_apps, flag_names): + if flag_apps: + print(' write app-index') + mylib.json_write(fname_app_categories(), _dict_apps, pretty=False) + if flag_names: + print(' write name-index') + mylib.json_write(fname_category_names(), _dict_names, pretty=False) + + +def get_categories(bundle_id): + load_json_if_not_already() + try: + genres = _dict_apps[bundle_id] + except KeyError: + return [] + res = [] + for gid in genres: + for lang in ['us', 'de']: + try: + name = _dict_names[gid][lang] + except KeyError: + continue + res.append((gid, name)) + break + return res + + +def enum_all_categories(): + load_json_if_not_already() + reverse_index = {} + for bid, genre_ids in _dict_apps.items(): + for gid in genre_ids: + try: + reverse_index[gid].append(bid) + except KeyError: + reverse_index[gid] = [bid] + for gid, lang_dict in _dict_names.items(): + for lang in ['us', 'de']: + try: + name = lang_dict[lang] + except KeyError: + continue + yield gid, name, reverse_index[gid] + break + + +def process(bundle_ids, force=False): + print('writing index: categories ...') + if force and bundle_ids == ['*']: + reset_index() + + load_json_if_not_already() + write_app_index = False + write_name_index = False + for bid in mylib.appids_in_data(bundle_ids): + genre_ids = [] + for lang, gid, gname in download_itunes.enum_genres(bid): + if gid not in genre_ids: + genre_ids.append(gid) + if try_update_name(gid, lang, gname): + write_name_index = True + if try_update_app(bid, genre_ids): + write_app_index = True + + try_persist_changes(write_app_index, write_name_index) + print('') + + +if __name__ == '__main__': + args = sys.argv[1:] + if len(args) > 0: + process(args) + else: + # process(['*']) + mylib.usage(__file__, '[bundle_id] [...]') diff --git a/src/index_meta.py b/src/index_meta.py index 8fbbdce..dca5978 100755 --- a/src/index_meta.py +++ b/src/index_meta.py @@ -4,8 +4,6 @@ import sys import common_lib as mylib import bundle_combine # get_evaluated -_rank_dict = None - def fname_app_summary(): return mylib.path_data_index('app_summary.json') diff --git a/src/lib_graphs.py b/src/lib_graphs.py new file mode 100755 index 0000000..902f7f8 --- /dev/null +++ b/src/lib_graphs.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import math +import common_lib as mylib +import lib_html as HTML + + +def fill_bar(percent): + return ''.format(round(percent * 100)) + + +def percent_bar(percent): + return ''.format(round(percent * 100)) + + +def rank_tile(title, value, additional=None, attr={}, + percent=0.5, rank='?', best='?', worst='?'): + if additional: + value += '({})'.format(additional) + attr = HTML.attr_and(attr, {'class': 'rank'}) + return HTML.div(''' +Rank: {}, best: {}, worst: {}
+'''.format(title, percent_bar(percent), value, rank, best, worst), attr) + + +def dotgraph(arr): + ''' Needs list of (title, count, attr_str) tuples ''' + def D(title, count, attr_str=''): + return '{1}
{2}'.format( + attr_str, title, '' * count) + return '