relikd/appchk-web
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Dynamic app ranking

Browse Source
This commit is contained in:
relikd
2020-09-22 01:16:17 +02:00
parent 2349b2ca8d
commit 2fd3ebf7e6
3 changed files with 93 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
out/static/lookup-rank.js Normal file
View File

@@ -0,0 +1,43 @@
function lookup_rank_js(bundle_id) {
loadJSON('/stats/rank.json', function(response) {
let json = JSON.parse(response);
if (!json) { return; }
let rank = json[bundle_id];
let rank_max = json['_ranks'];
if (!rank || !rank_max) { return; }
let best = json['_min'];
let worst = json['_max'];
function update(i, id, fmt=String) {
let r = (rank[i] - 1) / (rank_max - 1);
let target = document.getElementById(id);
let bar = target.querySelector('.percentile');
bar.classList.add(r < 0.5 ? 'g' : 'b');
bar.firstChild.style.left = r * 100 + '%';
let meta = target.lastElementChild.children;
meta[0].innerHTML = rank[i];
meta[1].innerHTML = fmt(best[i]);
meta[2].innerHTML = fmt(worst[i]);
}
// formatting
function dot1(x) { return Math.round(x * 10) / 10; }
function as_percent(x) { return dot1(x * 100) + '%'; }
function as_pm(x) { return dot1(x) + '/min'; }
function HHmmss(seconds) {
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
const s = Math.round(seconds % 60);
return (h<10?'0'+h:h)+':'+(m<10?'0'+m:m)+':'+(s<10?'0'+s:s);
}
// order is important!
update(0, 'sum_rec');
update(1, 'avg_time', HHmmss);
update(2, 'sum_time', HHmmss);
update(3, 'avg_logs_pm', as_pm);
update(4, 'sum_logs_pm', as_pm);
update(5, 'pardom');
update(6, 'subdom');
update(7, 'tracker_percent', as_percent);
});
}

50
src/html_bundle.py
View File

@@ -7,7 +7,6 @@ import common_lib as mylib
import download_itunes # get_genres
import bundle_combine # get_evaluated, fname_evaluated
import index_app_names # get_name
import index_meta # get_rank
def gen_dotgraph(sorted_arr):
@@ -94,27 +93,22 @@ def gen_html(bundle_id, obj):
hours, minutes = divmod(minutes, 60)
return '{:02d}:{:02d}:{:02d}'.format(hours, minutes, seconds)
def stat(col, title, rank, value, optional=None, fmt=str, fmt2=None):
# percent = int(rank[0] / max_rank * 100)
r = rank[0] / max_rank
detail = fmt2(value) if fmt2 else fmt(value)
def stat(col, title, ident, value, optional=None):
if optional:
x = fmt(optional) if fmt2 else optional
detail += '<i class="snd mg_lr">({})</i>'.format(x)
return f'''
<div class="col{col}">
<h4>{title}</h4>
<div class="percentile {'g' if r < 0.5 else 'b'}"><div style="left: {as_percent(r)}"></div></div>
<b class="mg_lr">{detail}</b>
value += '<i class="snd mg_lr">({})</i>'.format(optional)
return '''
<div id="{}" class="col{}">
<h4>{}</h4>
<div class="percentile"><div style="left: 50%"></div></div>
<b class="mg_lr">{}</b>
<p class="snd">
Rank:&nbsp;<b>{rank[0]}</b>,
best:&nbsp;<i>{fmt(rank[1])}</i>,
worst:&nbsp;<i>{fmt(rank[2])}</i></p>
</div>'''
Rank:&nbsp;<b>?</b>,
best:&nbsp;<i>?</i>,
worst:&nbsp;<i>?</i></p>
</div>'''.format(ident, col, title, value)
name = index_app_names.get_name(bundle_id)
gernes = download_itunes.get_genres(bundle_id)
rank, max_rank = index_meta.get_rank(bundle_id)
obj['tracker'] = list(filter(lambda x: x[2], obj['subdom']))
return mylib.template_with_base(f'''
<h2 class="title">{name}</h2>
@@ -132,14 +126,14 @@ def gen_html(bundle_id, obj):
</table>
</div>
<div id="stats">
{ stat(1, 'Number of recordings:', rank['sum_rec'], obj['sum_rec']) }
{ stat(1, 'Average recording time:', rank['avg_time'], obj['avg_time'], fmt=seconds_to_time) }
{ stat(2, 'Cumulative recording time:', rank['sum_time'], obj['sum_time'], fmt=seconds_to_time) }
{ stat(1, 'Average number of requests:', rank['avg_logs_pm'], obj['avg_logs'], obj['avg_logs_pm'], fmt=as_pm, fmt2=round_num) }
{ stat(2, 'Total number of requests:', rank['sum_logs_pm'], obj['sum_logs'], obj['sum_logs_pm'], fmt=as_pm, fmt2=str) }
{ stat(1, 'Number of domains:', rank['pardom'], len(obj['pardom'])) }
{ stat(2, 'Number of subdomains:', rank['subdom'], len(obj['subdom'])) }
{ stat(3, 'Tracker percentage:', rank['tracker_percent'], obj['tracker_percent'], fmt=as_percent) }
{ stat(1, 'Number of recordings:', 'sum_rec', obj['sum_rec']) }
{ stat(1, 'Average recording time:', 'avg_time', seconds_to_time(obj['avg_time'])) }
{ stat(2, 'Cumulative recording time:', 'sum_time', seconds_to_time(obj['sum_time'])) }
{ stat(1, 'Average number of requests:', 'avg_logs_pm', round_num(obj['avg_logs']), as_pm(obj['avg_logs_pm'])) }
{ stat(2, 'Total number of requests:', 'sum_logs_pm', str(obj['sum_logs']), as_pm(obj['sum_logs_pm'])) }
{ stat(1, 'Number of domains:', 'pardom', len(obj['pardom'])) }
{ stat(2, 'Number of subdomains:', 'subdom', len(obj['subdom'])) }
{ stat(3, 'Tracker percentage:', 'tracker_percent', as_percent(obj['tracker_percent'])) }
</div>
<h3>Connections</h3>
<div>
@@ -152,7 +146,11 @@ def gen_html(bundle_id, obj):
{ gen_dotgraph(obj['subdom']) }
{ gen_dom_tags(obj['subdom'], isSub=True) }
</div>
<p class="right snd">Download: <a href="data.json" download="{bundle_id}.json">json</a></p>''', title=name)
<p class="right snd">Download: <a href="data.json" download="{bundle_id}.json">json</a></p>
<script type="text/javascript" src="/static/lookup-rank.js"></script>
<script type="text/javascript">
lookup_rank_js('{bundle_id}');
</script>''', title=name)
def process(bundle_ids):

59
src/index_meta.py
View File

@@ -22,32 +22,34 @@ def load_json_from_disk(fname):
def json_to_list(json):
return [
json['sum_rec'],
json['sum_logs'],
json['sum_logs_pm'],
json['sum_time'],
json['avg_logs'],
json['avg_logs_pm'],
json['avg_time'],
json['last_date'],
json['sum_time'],
json['avg_logs_pm'],
json['sum_logs_pm'],
len(json['pardom']),
len(json['subdom']),
json['tracker_percent']
json['tracker_percent'],
# v- not part of rank -v
json['sum_logs'],
json['avg_logs'],
json['last_date'],
]
def list_to_json(list):
return {
'sum_rec': list[0],
'sum_logs': list[1],
'sum_logs_pm': list[2],
'sum_time': list[3],
'avg_logs': list[4],
'avg_logs_pm': list[5],
'avg_time': list[6],
'last_date': list[7],
'pardom': list[8],
'subdom': list[9],
'tracker_percent': list[10]
'avg_time': list[1],
'sum_time': list[2],
'avg_logs_pm': list[3],
'sum_logs_pm': list[4],
'pardom': list[5],
'subdom': list[6],
'tracker_percent': list[7],
# v- not part of rank -v
'sum_logs': list[8],
'avg_logs': list[9],
'last_date': list[10],
}
@@ -65,7 +67,7 @@ def write_summary_index(index, bundle_ids, deleteOnly=False):
total = [0, 0]
for val in index.values():
total[0] += val[0]
total[1] += val[1]
total[1] += val[8]
index['_sum'] = total
mylib.json_write(fname_app_summary(), index, pretty=False)
@@ -75,7 +77,7 @@ def write_rank_index(index):
mins = []
maxs = []
if len(index) > 0:
for i in range(11): # equal to number of array entries
for i in range(8): # exclude unused columns
tmp = {}
# make temporary reverse index
for bid, val in index.items():
@@ -85,7 +87,7 @@ def write_rank_index(index):
tmp[val[i]] = [bid]
# read index position from temp reverse index
r = 1
ordered = sorted(tmp.items(), reverse=i in [0, 3, 6, 7])
ordered = sorted(tmp.items(), reverse=i in [0, 1, 2])
for idx, (_, ids) in enumerate(ordered):
for bid in ids:
index[bid][i] = r
@@ -95,7 +97,10 @@ def write_rank_index(index):
index['_ranks'] = len(index)
index['_min'] = mins
index['_max'] = maxs
mylib.json_write(fname_app_rank(), index, pretty=False)
# write evaluated file
fname = fname_app_rank()
mylib.json_write(fname, index, pretty=False)
mylib.symlink(fname, mylib.path_out('stats', 'rank.json'))
def get_total_counts():
@@ -105,18 +110,6 @@ def get_total_counts():
return [0, 0]
def get_rank(bundle_id):
''' Return tuples with (rank, max_rank, min_value, max_value) '''
global _rank_dict
if not _rank_dict:
_rank_dict = load_json_from_disk(fname_app_rank())
return list_to_json(list(zip(
_rank_dict[bundle_id],
_rank_dict['_min'],
_rank_dict['_max'],
))), _rank_dict['_ranks']
def process(bundle_ids, deleteOnly=False):
print('writing index: meta ...')
fname = fname_app_summary()