relikd/appchk-web
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Fix mutable params + compact names json

Browse Source
This commit is contained in:
relikd
2020-09-24 12:03:54 +02:00
parent 087bd5384b
commit 500f38b6b4
9 changed files with 102 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/download_itunes.py
View File

@@ -33,6 +33,16 @@ def enum_genres(bundle_id):
yield lang, gid, name
def choose_lang(obj):
''' expects dict with {'us': ..., 'de': ...} '''
for lang in AVAILABLE_LANGS:
try:
return obj[lang]
except KeyError:
pass
return None
def download_info(bundle_id, lang, force=False):
fname = fname_for(bundle_id, lang)
if force or not mylib.file_exists(fname):

2
src/html_categories.py
View File

@@ -21,7 +21,7 @@ def process(affected=None, per_page=60):
print(' {} ({})'.format(cat, a))
print(' .. {} categories'.format(len(arr)))
src = ''.join([HTML.a(n, '{}/'.format(cid)) for cid, n in arr])
src = ''.join([HTML.a_category(cid, n) for cid, n in arr])
HTML.write(base, '''
<h2>{}</h2>
<div class="tags large center">

53
src/index_app_names.py
View File

@@ -4,67 +4,58 @@ import sys
import lib_common as mylib
import download_itunes # app_names
_bundle_name_dict = None
_app_names_dict = None
def index_fname():
return mylib.path_data_index('app_names.json')
def fname_apps_all():
return mylib.path_data_index('app_names_all.json')
def missing():
return not mylib.file_exists(index_fname())
def fname_apps_compact():
return mylib.path_data_index('app_names_compact.json')
def load_json_if_not_already():
global _bundle_name_dict
if not _bundle_name_dict:
index_file = index_fname()
if mylib.file_exists(index_file):
_bundle_name_dict = mylib.json_read(index_file)
else:
_bundle_name_dict = {}
def write_json_to_disk():
mylib.json_write(index_fname(), _bundle_name_dict, pretty=False)
def get_name(bundle_id, langs=['us', 'de'], fallback='&lt; App-Name &gt;'):
load_json_if_not_already()
for lang in langs:
def get_name(bundle_id, fallback='&lt; App-Name &gt;'):
global _app_names_dict
if not _app_names_dict:
_app_names_dict = mylib.json_safe_read(fname_apps_compact(), {})
try:
return _bundle_name_dict[bundle_id][lang]
return _app_names_dict[bundle_id]
except KeyError:
continue
return fallback # None
return fallback
def process(bundle_ids, deleteOnly=False):
global _app_names_dict
print('writing index: app names ...')
if bundle_ids == ['*']:
print(' full reset')
mylib.rm_file(index_fname()) # rebuild from ground up
mylib.rm_file(fname_apps_all()) # rebuild from ground up
mylib.rm_file(fname_apps_compact())
load_json_if_not_already()
index = mylib.json_safe_read(fname_apps_all(), {})
did_change = False
for bid in mylib.appids_in_data(bundle_ids):
if deleteOnly:
did_change |= mylib.try_del(_bundle_name_dict, [bid])
did_change |= mylib.try_del(index, [bid])
continue
names = download_itunes.get_app_names(bid)
if not names:
mylib.err('index-app-names', 'could not load: {}'.format(bid))
continue
try:
if _bundle_name_dict[bid] == names:
if index[bid] == names:
continue
except KeyError:
pass
_bundle_name_dict[bid] = names
index[bid] = names
did_change = True
if did_change:
print(' writing')
write_json_to_disk()
mylib.json_write(fname_apps_all(), index, pretty=False)
_app_names_dict = {bid: download_itunes.choose_lang(names)
for bid, names in index.items()}
mylib.json_write(fname_apps_compact(), _app_names_dict, pretty=False)
else:
print(' no change')
print('')

81
src/index_categories.py
View File

@@ -12,50 +12,53 @@ def fname_app_categories():
return mylib.path_data_index('app_categories.json')
def fname_category_names():
return mylib.path_data_index('category_names.json')
def fname_cat_name_all():
return mylib.path_data_index('category_names_all.json')
def load_json_if_not_already():
def load_json_from_disk(fname):
return mylib.json_read(fname) if mylib.file_exists(fname) else {}
def fname_cat_name_compact():
return mylib.path_data_index('category_names_compact.json')
def load_json_if_not_already(noNames=False):
global _dict_apps, _dict_names
if not _dict_apps:
_dict_apps = load_json_from_disk(fname_app_categories())
if not _dict_names:
_dict_names = load_json_from_disk(fname_category_names())
_dict_apps = mylib.json_safe_read(fname_app_categories(), {})
if not _dict_names and not noNames:
_dict_names = mylib.json_safe_read(fname_cat_name_compact(), {})
def try_update_app(bid, genre_ids):
def try_update_app(index, bid, genre_ids):
try:
if _dict_apps[bid] == genre_ids:
if index[bid] == genre_ids:
return False
except KeyError:
pass
_dict_apps[bid] = genre_ids
index[bid] = genre_ids
return True
def try_update_name(gid, lang, name):
def try_update_name_all(index, cid, lang, name):
try:
_dict_names[gid]
index[cid]
except KeyError:
_dict_names[gid] = {}
index[cid] = {}
try:
if _dict_names[gid][lang]:
if index[cid][lang]:
return False # key already exists
except KeyError:
pass
_dict_names[gid][lang] = name
index[cid][lang] = name
return True # updated, need to persist changes
def reset_index():
global _dict_apps
print(' full reset')
global _dict_apps, _dict_names
mylib.rm_file(fname_app_categories()) # rebuild from ground up
mylib.rm_file(fname_cat_name_all())
mylib.rm_file(fname_cat_name_compact())
_dict_apps = None
_dict_names = None
def get_categories(bundle_id):
@@ -66,13 +69,7 @@ def get_categories(bundle_id):
return []
res = []
for gid in genres:
for lang in ['us', 'de']:
try:
name = _dict_names[gid][lang]
except KeyError:
continue
res.append((gid, name))
break
res.append((gid, _dict_names[gid]))
return res
@@ -85,40 +82,40 @@ def enum_all_categories():
reverse_index[gid].append(bid)
except KeyError:
reverse_index[gid] = [bid]
for gid, lang_dict in _dict_names.items():
for lang in ['us', 'de']:
try:
name = lang_dict[lang]
except KeyError:
continue
for gid, name in _dict_names.items():
yield gid, name, reverse_index[gid]
break
def process(bundle_ids, force=False):
global _dict_apps, _dict_names
print('writing index: categories ...')
if force and bundle_ids == ['*']:
print(' full reset')
reset_index()
load_json_if_not_already()
load_json_if_not_already(noNames=False)
name_index = mylib.json_safe_read(fname_cat_name_all(), {})
write_name_index = False
write_app_index = False
for bid in mylib.appids_in_data(bundle_ids):
genre_ids = []
for lang, gid, gname in download_itunes.enum_genres(bid):
if gid not in genre_ids:
genre_ids.append(gid)
if try_update_name(gid, lang, gname):
cateogory_ids = []
for lang, cid, gname in download_itunes.enum_genres(bid):
if cid not in cateogory_ids:
cateogory_ids.append(cid)
if try_update_name_all(name_index, cid, lang, gname):
write_name_index = True
if try_update_app(bid, genre_ids):
if try_update_app(_dict_apps, bid, cateogory_ids):
write_app_index = True
if write_name_index:
print(' write name-index')
mylib.json_write(fname_category_names(), _dict_names, pretty=False)
if write_app_index:
print(' write app-index')
mylib.json_write(fname_app_categories(), _dict_apps, pretty=False)
if write_name_index:
print(' write name-index')
mylib.json_write(fname_cat_name_all(), name_index, pretty=False)
_dict_names = {cid: download_itunes.choose_lang(names)
for cid, names in name_index.items()}
mylib.json_write(fname_cat_name_compact(), _dict_names, pretty=False)
print('')

6
src/index_domains.py
View File

@@ -15,10 +15,8 @@ def fname_tracker():
def load_json_from_disk(index_file):
if mylib.file_exists(index_file):
return mylib.json_read(index_file)
else:
return {'bundle': [], 'pardom': {}, 'subdom': {}}
return mylib.json_safe_read(
index_file, fallback={'bundle': [], 'pardom': {}, 'subdom': {}})
def delete_from_index(index, bundle_ids, deleteOnly=False):

8
src/index_meta.py
View File

@@ -13,10 +13,6 @@ def fname_app_rank():
return mylib.path_data_index('app_rank.json')
def load_json_from_disk(fname):
return mylib.json_read(fname) if mylib.file_exists(fname) else {}
def json_to_list(json):
return [
json['sum_rec'],
@@ -100,7 +96,7 @@ def write_rank_index(index):
def get_total_counts():
try:
return load_json_from_disk(fname_app_summary())['_sum']
return mylib.json_safe_read(fname_app_summary(), {})['_sum']
except KeyError:
return [0, 0]
@@ -112,7 +108,7 @@ def process(bundle_ids, deleteOnly=False):
print(' full reset')
mylib.rm_file(fname) # rebuild from ground up
index = load_json_from_disk(fname)
index = mylib.json_safe_read(fname, {})
ids = mylib.appids_in_data(bundle_ids)
write_summary_index(index, ids, deleteOnly=deleteOnly)
write_rank_index(index)

12
src/lib_common.py
View File

@@ -249,14 +249,14 @@ def enum_jsons(bundle_id):
yield fname, json.load(fp)
def appids_in_out(selection=['*']):
if selection != ['*']:
def appids_in_out(selection=None):
if selection and selection != ['*']:
return selection
return [os.path.basename(x) for x in glob.glob(path_out_app('*'))]
def appids_in_data(selection=['*']):
if selection != ['*']:
def appids_in_data(selection=None):
if selection and selection != ['*']:
return selection
global _all_data_bundle_ids
if not _all_data_bundle_ids:
@@ -287,6 +287,10 @@ def json_read(path):
return json.load(fp)
def json_safe_read(path, fallback=None):
return json_read(path) if file_exists(path) else fallback
def json_write(path, obj, pretty=False):
with open(path, 'w') as fp:
json.dump(obj, fp, indent=2 if pretty else None, sort_keys=pretty)

4
src/lib_graphs.py
View File

@@ -13,11 +13,11 @@ def percent_bar(percent):
return '<div class="pcbar"><i style="left: {}%"></i></div>'.format(round(percent * 100))
def rank_tile(title, value, additional=None, attr={},
def rank_tile(title, value, additional=None, attr=None,
percent=0.5, rank='?', best='?', worst='?'):
if additional:
value += '<i class="snd mg_lr">({})</i>'.format(additional)
attr = HTML.attr_and(attr, {'class': 'rank'})
attr = HTML.attr_and(attr or {}, {'class': 'rank'})
return HTML.div('''
<h4>{}</h4>
{} <b class="mg_lr">{}</b>

22
src/lib_html.py
View File

@@ -46,29 +46,27 @@ def attr_and(a, b):
# Basic building blocks
def xml(tag, inner, attr={}):
def xml(tag, inner, attr=None):
src = ''
if attr:
for key, val in attr.items():
if val:
src += ' {}="{}"'.format(key, val)
return '<{0}{1}>{2}</{0}>'.format(tag, src, inner)
def div(inner, attr={}):
def div(inner, attr=None):
return xml('div', inner, attr)
def h2(inner, attr={}):
def h2(inner, attr=None):
return xml('h2', inner, attr)
def a(inner, href, attr={}):
return xml('a', inner, attr_and(attr, {'href': href}))
def a_path(parts, suffix):
''' expects (name, url) tuples '''
return ' / '.join([a(*x) for x in parts] + [suffix])
return ' / '.join(['<a href="{}/">{}</a>'.format(url, title)
for title, url in parts] + [suffix])
# Simple constructs
@@ -137,8 +135,8 @@ def app_tile_template():
</div></a>'''
def app_tiles_all(bundle_ids, per_page=60, attr={}):
attr = attr_and(attr, {'id': 'app-toc', 'class': 'no-ul-all'})
def app_tiles_all(bundle_ids, per_page=60):
attr = {'id': 'app-toc', 'class': 'no-ul-all'}
c_apps = len(bundle_ids)
c_pages = int(math.ceil(c_apps / per_page))
for i, apps in apps_sorted_batch(bundle_ids, batch_size=per_page):
@@ -170,11 +168,11 @@ def write(path, content, title=None, fname='index.html'):
fp.write(base_template(content, title=title))
def write_app_pages(base, bundle_ids, title, per_page=60, attr={}, pre=''):
def write_app_pages(base, bundle_ids, title, per_page=60, pre=''):
pages = 0
entries = 0
mylib.rm_dir(base)
for i, count, src in app_tiles_all(bundle_ids, per_page, attr):
for i, count, src in app_tiles_all(bundle_ids, per_page):
pages += 1
entries += count
pth = base if i == 1 else mylib.path_add(base, str(i))