From 69b287517765e67ffee7ee9e3a6a1a462bc29f6d Mon Sep 17 00:00:00 2001 From: relikd Date: Fri, 25 Sep 2020 02:56:16 +0200 Subject: [PATCH] Ranking --- out/static/lookup-domain.js | 5 +- out/static/lookup-rank.js | 10 -- out/static/lozad.js | 9 ++ out/static/ranking.js | 59 +++++++++ out/static/script.js | 48 ++++--- out/static/style.css | 27 +++- src/README.md | 21 ++-- src/html_categories.py | 2 +- src/html_index_apps.py | 3 +- src/html_index_domains.py | 5 +- src/html_ranking.py | 34 +++++ src/html_root.py | 2 +- src/index_rank.py | 61 ++++----- src/lib_html.py | 2 +- src/z_dependency.svg | 245 +++++++++++++++++++----------------- 15 files changed, 340 insertions(+), 193 deletions(-) create mode 100644 out/static/lozad.js create mode 100644 out/static/ranking.js create mode 100755 src/html_ranking.py diff --git a/out/static/lookup-domain.js b/out/static/lookup-domain.js index 129c410..c0a4d6c 100644 --- a/out/static/lookup-domain.js +++ b/out/static/lookup-domain.js @@ -31,11 +31,14 @@ function lookup_domain_js(fname_a, fname_b, id1, id2, id3) { let bid = apps[i][0]; let item = template.cloneNode(true); item.href = '/app/'+bid+'/'; - item.querySelector('img').src = '/app/'+bid+'/icon.png'; + let img = item.querySelector('img'); + img.classList = 'lozad'; + img.setAttribute('data-src', '/app/'+bid+'/icon.png'); item.querySelector('.name').innerHTML = apps[i][1]; item.querySelector('.detail').innerHTML = bid; dom_app_list.appendChild(item); } + const observer = lozad(); observer.observe(); }); }); } \ No newline at end of file diff --git a/out/static/lookup-rank.js b/out/static/lookup-rank.js index 2769796..681766c 100644 --- a/out/static/lookup-rank.js +++ b/out/static/lookup-rank.js @@ -20,16 +20,6 @@ function lookup_rank_js(bundle_id) { meta[1].innerHTML = fmt(best[i]); meta[2].innerHTML = fmt(worst[i]); } - // formatting - function dot1(x) { return Math.round(x * 10) / 10; } - function as_percent(x) { return dot1(x * 100) + '%'; } - function as_pm(x) { return dot1(x) + '/min'; } - function HHmmss(seconds) { - const h = Math.floor(seconds / 3600); - const m = Math.floor((seconds % 3600) / 60); - const s = Math.round(seconds % 60); - return (h<10?'0'+h:h)+':'+(m<10?'0'+m:m)+':'+(s<10?'0'+s:s); - } // order is important! update(0, 'sum_rec'); update(1, 'avg_time', HHmmss); diff --git a/out/static/lozad.js b/out/static/lozad.js new file mode 100644 index 0000000..d93feb3 --- /dev/null +++ b/out/static/lozad.js @@ -0,0 +1,9 @@ +/*! lozad.js - v1.9.0 - 2019-02-09 +* https://github.com/ApoorvSaxena/lozad.js +* Copyright (c) 2019 Apoorv Saxena; Licensed MIT */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):t.lozad=e()}(this,function(){"use strict";var g=Object.assign||function(t){for(var e=1;e'; + txt += ''; + txt += ''; + txt += ''; + txt += ''; + } + txt += ''; + } + return txt + ''; +} +function td(cols, id) { + var txt = '\n'; + for (var i = 0; i < cols.length; i++) { + txt += '' + cols[i] + ''; + } + return txt + ''; +} +function prep(x, i) { + return [ + i+1 + '. ', '' + x[1] + '', + x[2], HHmmss(x[3]), HHmmss(x[4]), as_pm(x[5]), as_pm(x[6]), x[7], x[8], + as_percent(x[9]), x[10], dot1(x[11]), new Date(x[12] * 1000).toISOString().slice(0, 16).replace('T',' ') + ]; +} +function update(col, asc) { + let table = document.getElementById('rank-list'); + let len = _data.length; + let txt = ''; + for (var i = 0; i < len; i++) { + txt += td(prep(_data[i], i), _data[i][0]) + } + table.innerHTML = th(col, asc, [ + '', 'Application', 'Number of recordings', 'Average recording time', + 'Cumulative recording time', 'Average requests per minute', + 'Total requests per minute', 'Number of domains', 'Number of subdomains', + 'Tracker percentage', 'Total number of requests', 'Average number of requests', + 'Last Update' + ]) + txt; + const observer = lozad(); observer.observe(); +} +function sort_by(col, asc) { + let i = col; + let o = asc; + _data.sort(function(a, b){ return a[i] < b[i] ? -o : a[i] > b[i] ? o : 0; }); + update(col, asc); +} +function rank_js(fname) { + loadJSON(fname, function(response) { + _data = JSON.parse(response); + update(12,-1); + }); +} \ No newline at end of file diff --git a/out/static/script.js b/out/static/script.js index fbf24cc..95b8d3d 100644 --- a/out/static/script.js +++ b/out/static/script.js @@ -1,25 +1,35 @@ (function(){// main entry - updateViewport(); - addEventListener('touchstart', function() {}); // :hover + updateViewport(); + addEventListener('touchstart', function() {}); // :hover })(); function updateViewport() {// show at least 2 columns on mobile devices - var viewport = document.head.querySelector("meta[name=viewport]"); - if (viewport && screen.width < 372) { - document.head.removeChild(viewport); - var x = document.createElement("meta"); - x.setAttribute("name", "viewport"); - x.setAttribute("content", "width=372"); - document.head.appendChild(x); - } + var viewport = document.head.querySelector('meta[name=viewport]'); + if (viewport && screen.width < 372) { + document.head.removeChild(viewport); + var x = document.createElement('meta'); + x.setAttribute('name', 'viewport'); + x.setAttribute('content', 'width=372'); + document.head.appendChild(x); + } } function loadJSON(url, callback, async=true) { - var xobj = new XMLHttpRequest(); - xobj.overrideMimeType("application/json"); - xobj.open('GET', url, async); - xobj.onreadystatechange = function () { - if (xobj.readyState == 4 && xobj.status == "200") { - callback(xobj.responseText); - } - }; - xobj.send(null); + var xobj = new XMLHttpRequest(); + xobj.overrideMimeType('application/json'); + xobj.open('GET', url, async); + xobj.onreadystatechange = function () { + if (xobj.readyState == 4 && xobj.status == '200') { + callback(xobj.responseText); + } + }; + xobj.send(null); +} +// formatting +function dot1(x) { return Math.round(x * 10) / 10; } +function as_percent(x) { return dot1(x * 100) + '%'; } +function as_pm(x) { return dot1(x) + '/min'; } +function HHmmss(seconds) { + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + const s = Math.round(seconds % 60); + return (h<10?'0'+h:h)+':'+(m<10?'0'+m:m)+':'+(s<10?'0'+s:s); } \ No newline at end of file diff --git a/out/static/style.css b/out/static/style.css index f25bf64..33daba6 100644 --- a/out/static/style.css +++ b/out/static/style.css @@ -100,7 +100,7 @@ footer .links { margin: .5em auto 1em; display: block; } -#app-toc img, img.app-icon { +#app-toc img, #rank-list img, img.app-icon { border-radius: 21.5%; border: .7px solid #ccc; } @@ -150,6 +150,11 @@ p.subtitle { margin-top: .2em; } .border { border: 1pt solid #CCC; } .large { font-size: 1.2em; } .stick-top { top: 0; position: sticky; padding: .8em 0 .5em; background: #FFF; } +.xscroll { overflow-x: scroll; max-width: 100%; } +.yscroll { overflow-y: scroll; max-height: 80vh; } +table.alternate td, table.alternate th { padding: .5em; } +table.alternate tr:nth-child(even) { background: #DDD; } +table.alternate tr:nth-child(odd) { background: #F9F9F9; } /*#meta { margin-bottom: 2em; }*/ #meta .icons { margin-bottom: 2em; } @@ -243,13 +248,27 @@ p.trckr { font-size: .9em; margin-left: .5em; } .cs1{stroke:#CA0D3A} /* Help needed */ -#help-links td { padding: .5em; } -#help-links tr:nth-child(even) { background: #DDD; } -#help-links tr:nth-child(odd) { background: #F9F9F9; } #help-links span.snd { display: table; } #help-links .notyet { color: #D11; } #help-links .done { color: #52C840; } +/* rank-list */ +#rank-list th span { display: block; margin-top: 4px; } +#rank-list th { vertical-align: bottom; } +#rank-list th a { cursor:pointer; border: unset; padding: 0 5px; margin: 0 2px; } +#rank-list th a:hover { fill: #F00; } +#rank-list th a.active { fill: #CA0D3A; } +#rank-list th:not(:first-child) { min-width: 12ex; } + +#rank-list img { width: 25px; height: 25px; vertical-align: bottom; } +#rank-list td { text-align: center; min-width: max-content; } +#rank-list td:nth-child(1) { + position: sticky; left: 0; + background: inherit; + text-align: right; +} +#rank-list td:nth-child(2) { text-align: left; } + /* responsive */ @media(max-width: 900px) { #stats { grid-template-columns: repeat(2, max-content); } diff --git a/src/README.md b/src/README.md index a4e281a..4656bec 100644 --- a/src/README.md +++ b/src/README.md @@ -40,22 +40,23 @@ Given A → B, B depends on A ``` digraph Dependency { - "." -> html_root + "." -> download_tracker "." -> download_itunes + "." -> bundle_combine download_itunes -> index_app_names download_itunes -> index_categories - index_categories -> html_categories - index_app_names -> html_bundle - index_app_names -> html_index_apps - index_app_names -> html_index_domains - index_app_names -> html_categories - index_rank -> html_index_domains - "." -> bundle_combine bundle_combine -> index_rank - index_rank -> html_bundle bundle_combine -> index_domains + index_categories -> html_categories + index_app_names -> html_index_apps + index_app_names -> html_categories + index_app_names -> index_rank + index_rank -> html_bundle + index_rank -> html_rank + index_rank -> html_index_domains index_domains -> html_index_domains - "." -> download_tracker + "." -> html_ranking + "." -> html_root } ``` [graphviz](http://www.webgraphviz.com/) \ No newline at end of file diff --git a/src/html_categories.py b/src/html_categories.py index 139386e..c189a2a 100755 --- a/src/html_categories.py +++ b/src/html_categories.py @@ -15,7 +15,7 @@ def process(affected=None, per_page=60): arr.append((cid, cat)) if affected and cid not in affected: continue - pre = HTML.h2(HTML.a_path([(parent, '..')], cat)) + pre = HTML.h2(HTML.a_path([(parent, '../')], cat)) _, a = HTML.write_app_pages(mylib.path_add(base, cid), apps, cat, per_page, pre=pre) print(' {} ({})'.format(cat, a)) diff --git a/src/html_index_apps.py b/src/html_index_apps.py index 378ae21..5e708fe 100755 --- a/src/html_index_apps.py +++ b/src/html_index_apps.py @@ -7,9 +7,10 @@ import lib_html as HTML def process(per_page=60): print('generating html: app-index ...') title = 'Apps (A–Z)' + header = HTML.h2(HTML.a_path([('Results', '/results/')], title)) p, a = HTML.write_app_pages(mylib.path_out('index', 'apps'), mylib.appids_in_out(), title, - per_page=per_page, pre=HTML.h2(title)) + per_page=per_page, pre=header) print(' {} apps'.format(a)) print(' {} pages'.format(p)) print('') diff --git a/src/html_index_domains.py b/src/html_index_domains.py index 0fdd331..fdd993a 100755 --- a/src/html_index_domains.py +++ b/src/html_index_domains.py @@ -98,6 +98,7 @@ def gen_lookup(html_dir, doms_dict, names_dict, title): {} + @@ -122,7 +123,9 @@ def gen_results(c_apps, c_domains, title):
  • List of Categories
  • List of Requested Domains
  • List of Trackers
    • -'''.format(title, c_apps, c_domains, c_recordings, c_logs), title=title) + +

    Or go ahead to the Ranking directly.

    +'''.format(title, c_apps, c_domains, c_recordings, c_logs), title=title) mylib.symlink(index_rank.fname_app_rank(), mylib.path_out('results', 'rank.json')) # after HTML.write diff --git a/src/html_ranking.py b/src/html_ranking.py new file mode 100755 index 0000000..c04ac3c --- /dev/null +++ b/src/html_ranking.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import lib_common as mylib +import lib_html as HTML +import index_rank + + +def process(): + print('generating html: ranking ...') + title = 'Ranking' + header = HTML.a_path([('Results', '/results/')], title) + base = mylib.path_out('ranking') + HTML.write(base, ''' +

    {}

    +

    + This ranking shows only the 500 most recently updated applications.
    + If you're missing an app, feel free to contribute a new app recording. +

    +
    +
    Loading …
    +
    +

    Download: json

    + + +'''.format(header), title=title) + mylib.symlink(index_rank.fname_ranking_list(), + mylib.path_add(base, 'data.json')) + print('') + + +if __name__ == '__main__': + process() diff --git a/src/html_root.py b/src/html_root.py index 4364ce9..3e694d0 100755 --- a/src/html_root.py +++ b/src/html_root.py @@ -67,7 +67,7 @@ def gen_help(): obj = mylib.json_read(mylib.path_root('src', 'help.json')) for land in sorted(obj.keys()): - txt += '\n

    {}:

    \n'.format(land) + txt += '\n

    {}:

    \n
    '.format(land) txt += HTML.tr(['', 'App Name', 'pre iOS 14', 'post iOS 14'], 'th') for i, x in enumerate(obj[land]): bid = x[2] diff --git a/src/index_rank.py b/src/index_rank.py index 4da92eb..1c076a0 100755 --- a/src/index_rank.py +++ b/src/index_rank.py @@ -3,6 +3,7 @@ import sys import lib_common as mylib import bundle_combine # get_evaluated +import index_app_names def fname_app_summary(): @@ -13,40 +14,27 @@ def fname_app_rank(): return mylib.path_data_index('app_rank.json') +def fname_ranking_list(): + return mylib.path_data_index('ranking_list.json') + + def json_to_list(json): return [ - json['sum_rec'], - json['avg_time'], - json['sum_time'], - json['avg_logs_pm'], - json['sum_logs_pm'], - len(json['pardom']), - len(json['subdom']), - json['tracker_percent'], + json['sum_rec'], # 0 + json['avg_time'], # 1 + json['sum_time'], # 2 + json['avg_logs_pm'], # 3 + json['sum_logs_pm'], # 4 + len(json['pardom']), # 5 + len(json['subdom']), # 6 + json['tracker_percent'], # 7 # v- not part of rank -v - json['sum_logs'], - json['avg_logs'], - json['last_date'], + json['sum_logs'], # 8 + json['avg_logs'], # 9 + json['last_date'], # 10 ] -def list_to_json(list): - return { - 'sum_rec': list[0], - 'avg_time': list[1], - 'sum_time': list[2], - 'avg_logs_pm': list[3], - 'sum_logs_pm': list[4], - 'pardom': list[5], - 'subdom': list[6], - 'tracker_percent': list[7], - # v- not part of rank -v - 'sum_logs': list[8], - 'avg_logs': list[9], - 'last_date': list[10], - } - - def write_summary_index(index, bundle_ids, deleteOnly=False): for bid in bundle_ids: # delete old value @@ -64,10 +52,24 @@ def write_summary_index(index, bundle_ids, deleteOnly=False): total[1] += val[8] index['_sum'] = total mylib.json_write(fname_app_summary(), index, pretty=False) + mylib.try_del(index, ['_sum']) + + +def write_ranking_list(index): + ret = [] + for bid, values in index.items(): + ret.append([bid, index_app_names.get_name(bid)] + values) + del(values[8:]) + ret.sort(key=lambda x: -x[2 + 10]) # sort by last update + # TODO: doesnt scale well, 100'000 apps ~> 12mb + if len(ret) > 500: # limit to most recent X entries + ret = ret[:500] + # ret.sort(key=lambda x: x[1].lower()) # sort by name + mylib.json_write(fname_ranking_list(), ret, pretty=False) def write_rank_index(index): - mylib.try_del(index, ['_sum', '_ranks', '_min', '_max']) + mylib.try_del(index, ['_ranks', '_min', '_max']) mins = [] maxs = [] if len(index) > 0: @@ -111,6 +113,7 @@ def process(bundle_ids, deleteOnly=False): index = mylib.json_safe_read(fname, {}) ids = mylib.appids_in_data(bundle_ids) write_summary_index(index, ids, deleteOnly=deleteOnly) + write_ranking_list(index) write_rank_index(index) print('') diff --git a/src/lib_html.py b/src/lib_html.py index 57b3541..d6eae65 100755 --- a/src/lib_html.py +++ b/src/lib_html.py @@ -65,7 +65,7 @@ def h2(inner, attr=None): def a_path(parts, suffix): ''' expects (name, url) tuples ''' - return ' / '.join(['{}'.format(url, title) + return ' / '.join(['{}'.format(url, title) for title, url in parts] + [suffix]) diff --git a/src/z_dependency.svg b/src/z_dependency.svg index b089fb0..4694e56 100644 --- a/src/z_dependency.svg +++ b/src/z_dependency.svg @@ -1,151 +1,166 @@ - - + + Dependency - + . - -. + +. - -html_root - -html_root + +download_tracker + +download_tracker - -.->html_root - - + +.->download_tracker + + download_itunes - -download_itunes + +download_itunes .->download_itunes - - + + -bundle_combine - -bundle_combine +bundle_combine + +bundle_combine -.->bundle_combine - - +.->bundle_combine + + - -download_tracker - -download_tracker + +html_ranking + +html_ranking - -.->download_tracker - - + +.->html_ranking + + + + +html_root + +html_root + + +.->html_root + + -index_app_names - -index_app_names +index_app_names + +index_app_names -download_itunes->index_app_names - - +download_itunes->index_app_names + + -index_categories - -index_categories +index_categories + +index_categories -download_itunes->index_categories - - +download_itunes->index_categories + + - -html_categories - -html_categories + +index_rank + +index_rank - -index_app_names->html_categories - - - - -html_bundle - -html_bundle - - -index_app_names->html_bundle - - - - -html_index_apps - -html_index_apps - - -index_app_names->html_index_apps - - - - -html_index_domains - -html_index_domains - - -index_app_names->html_index_domains - - - - -index_categories->html_categories - - - - -index_meta - -index_meta - - -index_meta->html_bundle - - - - -index_meta->html_index_domains - - - - -bundle_combine->index_meta - - + +bundle_combine->index_rank + + -index_domains - -index_domains +index_domains + +index_domains -bundle_combine->index_domains - - +bundle_combine->index_domains + + + + +index_app_names->index_rank + + + + +html_categories + +html_categories + + +index_app_names->html_categories + + + + +html_index_apps + +html_index_apps + + +index_app_names->html_index_apps + + + + +index_categories->html_categories + + + + +html_index_domains + +html_index_domains + + +index_rank->html_index_domains + + + + +html_bundle + +html_bundle + + +index_rank->html_bundle + + + + +html_rank + +html_rank + + +index_rank->html_rank + + -index_domains->html_index_domains - - +index_domains->html_index_domains + + \ No newline at end of file