module refactoring + allow word mistakes for OEIS search

LP/FailedAttempts.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+from RuneText import rune_map, RuneText
+from NGrams import NGrams
+
+
+#########################################
+#  NGramShifter  :  Shift rune-pairs in a fixed-width running window
+#########################################
+
+class NGramShifter(object):
+    def __init__(self, gramsize=3):  # 3 is the only reasonable value though
+        self.gramsize = gramsize
+        self.prob = NGrams.load(gramsize)
+
+    def ngram_probability_heatmap(self, data):
+        gram_count = len(data) // self.gramsize
+        ret = [[] for _ in range(gram_count)]  # ret[x][y] x: parts, y: shifts
+        for y in range(29):
+            variant = data - y
+            for x in range(gram_count):
+                i = x * self.gramsize
+                gram = ''.join(r.rune for r in variant[i:i + self.gramsize])
+                ret[x].append((y, self.prob.get(gram, 0), gram))
+        # sort most probable first
+        for arr in ret:
+            arr.sort(key=lambda x: -x[1])  # (shift, probability)
+        return ret
+
+    def guess_single(self, data, interrupt_chr=None):
+        data = RuneText(data)
+        res = self.ngram_probability_heatmap(data)
+        fillup = ' ' * (2 * self.gramsize + 1)
+        all_interrupts = []
+        if interrupt_chr:
+            for i, x in enumerate(data):
+                if x.rune == interrupt_chr:
+                    all_interrupts.append(i)
+        for y in range(29):  # each row in output
+            line = ''
+            for i, obj in enumerate(res):  # each column per row
+                txt = ''
+                if obj[y][1] > 0:
+                    for u in range(self.gramsize):
+                        if (i * self.gramsize + u) in all_interrupts:
+                            txt += '|'  # mark with preceding
+                        txt += rune_map[obj[y][2][u]]
+                line += txt + fillup[len(txt):]
+            line = line.rstrip()
+            if line:
+                print(line)
+
+    def guess(self, data, interrupt_chr=None):
+        data = RuneText(data)  # create RuneText once and reuse
+        for i in range(self.gramsize):
+            print('offset:', i)
+            self.guess_single(data[i:], interrupt_chr)
+            print()
+
+
+# NGramShifter().guess('ᛈᚢᛟᚫᛈᚠᛖᚱᛋᛈᛈᚦᛗᚾᚪᚱᛚᚹᛈᛖᚩᛈᚢᛠᛁᛁᚻᛞᛚᛟᛠ', 'ᛟ')
+# NGramShifter().guess([1, 2, 4, 5, 7, 9, 0, 12], 'ᛟ')

LP/HeuristicLib.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+from NGrams import NGrams
+from RuneText import RUNES
+
+
+def normalized_probability(int_prob):
+    total = sum(int_prob)
+    return [x / total for x in int_prob]  # math.log(x / total, 10)
+
+
+PROB_INT = [0] * 29
+for k, v in NGrams.load(1, '').items():  # '-no-e', '-solved'
+    PROB_INT[RUNES.index(k)] = v
+PROB_NORM = normalized_probability(PROB_INT)
+# Target IoC. peace and war: 1.77368517 solved: 1.78021503, no e: 1.82715300
+N_total = (sum(PROB_INT) * (sum(PROB_INT) - 1)) / 29
+TARGET_IOC = sum(x * (x - 1) for x in PROB_INT) / N_total
+# TARGET_IOC = 1.78
+
+
+#########################################
+#  Probability  :  Count runes and do simple frequency analysis
+#########################################
+
+class Probability(object):
+    def __init__(self, numstream):
+        self.prob = [0] * 29
+        for r in numstream:
+            self.prob[r] += 1
+        self.N = len(numstream)
+
+    def IC(self):
+        X = sum(x * (x - 1) for x in self.prob)
+        return X / ((self.N * (self.N - 1)) / 29)
+
+    def IC_norm(self, target_ioc=TARGET_IOC):
+        return abs(self.IC() - target_ioc)
+
+    def similarity(self):
+        probs = normalized_probability(self.prob)
+        return sum((x - y) ** 2 for x, y in zip(PROB_NORM, probs))
+
+    @staticmethod
+    def IC_w_keylen(nums, keylen):
+        val = sum(Probability(nums[x::keylen]).IC() for x in range(keylen))
+        return val / keylen
+
+    @staticmethod
+    def target_diff(nums, keylen, target_ioc=TARGET_IOC):
+        val = sum(abs(Probability(nums[x::keylen]).IC() - target_ioc)
+                  for x in range(keylen))
+        return 1 - (val / keylen)

LP/HeuristicSearch.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+import itertools  # product, compress, combinations
+import bisect  # bisect_left, insort
+from lib import affine_decrypt
+
+
+#########################################
+#  GuessVigenere  :  Shift values around with a given keylength
+#########################################
+
+class GuessVigenere(object):
+    def __init__(self, nums):
+        self.nums = nums
+
+    def guess(self, keylength, score_fn):  # minimize score_fn
+        found = []
+        avg_score = 0
+        for offset in range(keylength):
+            bi = -1
+            bs = 9999999
+            for i in range(29):
+                shifted = [(x - i) % 29 for x in self.nums[offset::keylength]]
+                score = score_fn(shifted)
+                if score < bs:
+                    bs = score
+                    bi = i
+            avg_score += bs
+            found.append(bi)
+        return avg_score / keylength, found
+
+
+#########################################
+#  GuessAffine  :  Find greatest common affine key
+#########################################
+
+class GuessAffine(object):
+    def __init__(self, nums):
+        self.nums = nums
+
+    def guess(self, keylength, score_fn):  # minimize score_fn
+        found = []
+        avg_score = 0
+        for offset in range(keylength):
+            candidate = (None, None)
+            best = 9999999
+            for s in range(29):
+                for t in range(29):
+                    shifted = [affine_decrypt(x, (s, t))
+                               for x in self.nums[offset::keylength]]
+                    score = score_fn(shifted)
+                    if score < best:
+                        best = score
+                        candidate = (s, t)
+            avg_score += best
+            found.append(candidate)
+        return avg_score / keylength, found
+
+
+#########################################
+#  GuessPattern  :  Find a key that is rotated ABC BCA CAB, or ABC CAB BCA
+#########################################
+
+class GuessPattern(object):
+    def __init__(self, nums):
+        self.nums = nums
+
+    @staticmethod
+    def pattern(keylen, fn_pattern):
+        mask = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'[:keylen]
+        return fn_pattern(mask, keylen)
+
+    def split(self, keylen, mask, offset=0):
+        ret = {}
+        for _ in range(offset):
+            next(mask)
+        ret = {k: [] for k in '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'[:keylen]}
+        for n, k in zip(self.nums, mask):
+            ret[k].append(n)
+        return ret.values()
+
+    def zip(self, key_mask, offset=0):
+        for _ in range(offset):
+            next(key_mask)
+        return [(n - k) % 29 for n, k in zip(self.nums, key_mask)]
+
+    @staticmethod
+    def guess(parts, score_fn):  # minimize score_fn
+        found = []
+        avg_score = 0
+        for nums in parts:
+            best = 9999999
+            candidate = 0
+            for i in range(29):
+                score = score_fn([(x - i) % 29 for x in nums])
+                if score < best:
+                    best = score
+                    candidate = i
+            avg_score += best
+            found.append(candidate)
+        return avg_score / len(parts), found
+
+
+#########################################
+#  SearchInterrupt  :  Hill climbing algorithm for interrupt detection
+#########################################
+
+class SearchInterrupt(object):
+    def __init__(self, arr, interrupt_chr):  # remove all whitespace in arr
+        self.single_result = False  # if False, return list of equal likelihood
+        self.full = arr
+        self.stops = [i for i, n in enumerate(arr) if n == interrupt_chr]
+
+    def to_occurrence_index(self, interrupts):
+        return [self.stops.index(x) + 1 for x in interrupts]
+
+    def from_occurrence_index(self, interrupts):
+        return [self.stops[x - 1] for x in interrupts]
+
+    def join(self, interrupts=[]):  # rune positions, not occurrence index
+        ret = []
+        i = -1
+        for x in interrupts:
+            ret += self.full[i + 1:x]
+            i = x
+        return ret + self.full[i + 1:]
+
+    # Go over the full string but only look at the first {maxdepth} interrupts.
+    # Enumerate all possibilities and choose the one with the highest score.
+    # If first interrupt is set, add it to the resulting set. If not, ignore it
+    # Every iteration will add a single interrupt only, not the full set.
+    def sequential(self, score_fn, startAt=0, maxdepth=9):
+        found = [[]]
+
+        def best_in_one(i, depth, prefix=[]):
+            best_s = -8
+            best_p = []  # [match, match, ...]
+            irp = self.stops[i:i + depth]
+            for x in itertools.product([False, True], repeat=depth):
+                part = list(itertools.compress(irp, x))
+                score = score_fn(self.join(prefix + part))
+                if score >= best_s:
+                    if score > best_s or self.single_result:
+                        best_s = score
+                        best_p = [part]
+                    else:
+                        best_p.append(part)
+            return best_p, best_s
+
+        def best_in_all(i, depth):
+            best_s = -8
+            best_p = []  # [(prefix, [match, match, ...]), ...]
+            for pre in found:
+                parts, score = best_in_one(i, depth, prefix=pre)
+                if score >= best_s:
+                    if score > best_s or self.single_result:
+                        best_s = score
+                        best_p = [(pre, parts)]
+                    else:
+                        best_p.append((pre, parts))
+            return best_p, best_s
+
+        # first step: move maxdepth-sized window over data
+        i = startAt - 1  # in case loop isnt called
+        for i in range(startAt, len(self.stops) - maxdepth):
+            # print('.', end='')
+            parts, _ = best_in_all(i, maxdepth)
+            found = []
+            search = self.stops[i]
+            for prfx, candidates in parts:
+                bitSet = False
+                bitNotSet = False
+                for x in candidates:
+                    if len(x) > 0 and x[0] == search:
+                        bitSet = True
+                    else:
+                        bitNotSet = True
+                    if bitSet and bitNotSet:
+                        break
+                if bitSet:
+                    found.append(prfx + [search])
+                if bitNotSet:
+                    found.append(prfx)
+        # print('.')
+        # last step: all permutations for the remaining (< maxdepth) bits
+        i += 1
+        remaining, score = best_in_all(i, min(maxdepth, len(self.stops) - i))
+        found = [x + z for x, y in remaining for z in y]
+        return score, found
+
+    # Flip upto {maxdepth} bits anywhere in the full string.
+    # Choose the bitset with the highest score and repeat.
+    # If no better score found, increment number of testing bits and repeat.
+    # Either start with all interrupts set (topDown) or none set.
+    def genetic(self, score_fn, topDown=False, maxdepth=3):
+        current = self.stops if topDown else []
+
+        def evolve(lvl):
+            for x in itertools.combinations(self.stops, lvl + 1):
+                tmp = current[:]
+                for y in x:
+                    if y in current:
+                        tmp.pop(bisect.bisect_left(tmp, y))
+                    else:
+                        bisect.insort(tmp, y)
+                yield tmp, score_fn(self.join(tmp))
+
+        best = score_fn(self.join())
+        level = 0  # or start directly with maxdepth - 1
+        while level < maxdepth:
+            print('.', end='')
+            update = None
+            for interrupts, score in evolve(level):
+                if score > best:
+                    best = score
+                    update = interrupts
+            if update:
+                level = 0  # restart with 1-bit again
+                current = update
+                continue  # did optimize, so retry with same level
+            level += 1
+        print('.')
+        # find equally likely candidates
+        if self.single_result:
+            return best, [current]
+        all_of_them = [x for x, score in evolve(2) if score == best]
+        all_of_them.append(current)
+        return best, all_of_them
+
+
+# a = GuessInterrupt([2, 0, 1, 0, 14, 15, 0, 13, 24, 25, 25, 25], 0)
+# print(a.sequential(lambda x: (1.2 if len(x) == 11 else 0.1)))
+# print(a.sequential(lambda x: (1.1 if len(x) == 10 else 0.1)))
+# print(a.sequential(lambda x: (1.3 if len(x) == 9 else 0.1)))

LP/InterruptDB.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+import os
+from HeuristicSearch import SearchInterrupt
+from HeuristicLib import Probability
+from RuneText import RUNES, load_indices
+from LPath import FILES_ALL, FILES_UNSOLVED, LPath
+
+
+#########################################
+#  InterruptDB  :  Perform heuristic search on best possible interrupts.
+#########################################
+
+class InterruptDB(object):
+    def __init__(self, data, interrupt):
+        self.irp = interrupt
+        self.iguess = SearchInterrupt(data, interrupt)
+        self.irp_count = len(self.iguess.stops)
+
+    def make(self, dbname, name, keylen):
+        def fn(x):
+            return Probability.target_diff(x, keylen)  # used in db_norm
+            # return Probability.IC_w_keylen(x, keylen)  # used in db_high
+
+        if keylen == 0:
+            keylen = 1
+            score, skips = fn(self.iguess.join()), [[]]  # without interrupts
+        else:
+            score, skips = self.iguess.sequential(fn, startAt=0, maxdepth=99)
+            # score, skips = self.iguess.genetic(fn, topDown=False, maxdepth=4)
+        for i, interrupts in enumerate(skips):
+            skips[i] = self.iguess.to_occurrence_index(interrupts)
+
+        for nums in skips:
+            self.write(
+                name, score, self.irp, self.irp_count, keylen, nums, dbname)
+        return score, skips
+
+    def make_secondary(self, dbname, name, keylen, threshold):
+        scores = []
+
+        def fn(x):
+            score = Probability.target_diff(x, keylen)  # used in db_norm
+            # score = Probability.IC_w_keylen(x, keylen)  # used in db_high
+            if score >= threshold:
+                scores.append(score)
+                return 1
+            return -1
+
+        _, skips = self.iguess.sequential(fn, startAt=0, maxdepth=99)
+        for i, interrupts in enumerate(skips):
+            skips[i] = self.iguess.to_occurrence_index(interrupts)
+        ret = list(zip(scores, skips))
+        bestscore = max(ret)[0]
+        # exclude best results, as they are already present in the main db
+        filtered = [x for x in ret if x[0] < bestscore]
+        for score, nums in filtered:
+            self.write(
+                name, score, self.irp, self.irp_count, keylen, nums, dbname)
+        return len(filtered)
+
+    @staticmethod
+    def longest_no_interrupt(data, interrupt, irpmax=0):
+        def add(i):
+            nonlocal ret, prev
+            idx = prev.pop(0)
+            if idx == 0:
+                ret = []
+            ret.append((i - idx, idx))
+
+        prev = [0] * (irpmax + 1)
+        ret = []
+        for i, x in enumerate(data):
+            if x == interrupt:
+                prev.append(i + 1)
+                add(i)
+        add(i + 1)
+        length, pos = max(ret)
+        return pos, length
+
+    @staticmethod
+    def load(dbname):
+        if not os.path.isfile(LPath.InterruptDB(dbname)):
+            return {}
+        ret = {}
+        with open(LPath.InterruptDB(dbname), 'r') as f:
+            for line in f.readlines():
+                if line.startswith('#'):
+                    continue
+                line = line.rstrip()
+                name, irpc, score, irp, kl, nums = [x for x in line.split('|')]
+                val = [int(irpc), float(score), int(irp), int(kl)]
+                val.append([int(x) for x in nums.split(',')] if nums else [])
+                try:
+                    ret[name].append(val)
+                except KeyError:
+                    ret[name] = [val]
+        return ret
+
+    @staticmethod
+    def write(name, score, irp, irpmax, keylen, nums, dbname='db_main'):
+        with open(LPath.InterruptDB(dbname), 'a') as f:
+            nums = ','.join(map(str, nums))
+            f.write(f'{name}|{irpmax}|{score:.5f}|{irp}|{keylen}|{nums}\n')
+
+
+#########################################
+#  InterruptIndices  :  Read chapters and extract indices (cluster by runes)
+#########################################
+
+class InterruptIndices(object):
+    def __init__(self):
+        self.pos = InterruptIndices.read()
+
+    def consider(self, name, irp, limit):
+        nums = self.pos[name]['pos'][irp]
+        if len(nums) <= limit:
+            return self.pos[name]['total']
+        return nums[limit]  # number of runes, which is not last index
+
+    def total(self, name):
+        return self.pos[name]['total']
+
+    def longest_no_interrupt(self, name, irp, irpmax=0):
+        irpmax += 1
+        nums = self.pos[name]['pos'][irp] + [self.pos[name]['total']] * irpmax
+        ret = [(y - x, x) for x, y in zip(nums, nums[irpmax:])]
+        return sorted(ret, reverse=True)
+
+    @staticmethod
+    def write(dbname='db_indices'):
+        with open(LPath.InterruptDB(dbname), 'w') as f:
+            f.write('# file | total runes in file | interrupt | indices\n')
+            for name in FILES_ALL:
+                fname = f'pages/{name}.txt'
+                data = load_indices(fname, 0)
+                total = len(data)
+                nums = [[] for x in range(29)]
+                for idx, rune in enumerate(data):
+                    nums[rune].append(idx)
+                for irp, pos in enumerate(nums):
+                    f.write('{}|{}|{}|{}\n'.format(
+                        name, total, irp, ','.join(map(str, pos))))
+
+    @staticmethod
+    def read(dbname='db_indices'):
+        with open(LPath.InterruptDB(dbname), 'r') as f:
+            ret = {}
+            for line in f.readlines():
+                if line.startswith('#'):
+                    continue
+                line = line.strip()
+                name, total, irp, nums = line.split('|')
+                if name not in ret:
+                    ret[name] = {'total': int(total),
+                                 'pos': [[] for _ in range(29)]}
+                pos = ret[name]['pos']
+                pos[int(irp)] = list(map(int, nums.split(','))) if nums else []
+            return ret
+
+
+#########################################
+#  InterruptToWeb  :  Read interrupt DB and create html graphic / matrix
+#########################################
+
+class InterruptToWeb(object):
+    def __init__(self, dbname, template='template.html'):
+        with open(LPath.results(template), 'r') as f:
+            self.template = f.read()
+        self.indices = InterruptIndices()
+        self.scores = {}
+        db = InterruptDB.load(dbname)
+        for k, v in db.items():
+            for irpc, score, irp, kl, nums in v:
+                if k not in self.scores:
+                    self.scores[k] = [[] for _ in range(29)]
+                part = self.scores[k][irp]
+                while kl >= len(part):
+                    part.append((0, 0))  # (score, irpc)
+                oldc = part[kl][1]
+                if irpc > oldc or (irpc == oldc and score > part[kl][0]):
+                    part[kl] = (score, irpc)
+
+    def cls(self, x, low=0, high=1):
+        if x <= low:
+            return ' class="m0"'
+        return f' class="m{int((min(high, x) - low) / (high - low) * 14) + 1}"'
+
+    def table_reliable(self):
+        trh = '<tr class="rotate"><th></th>'
+        trtotal = '<tr class="small"><th>Total</th>'
+        trd = [f'<tr><th>{x}</th>' for x in RUNES]
+        del_row = [True] * 29
+        for name in FILES_ALL:
+            if name not in self.scores:
+                continue
+            total = self.indices.total(name)
+            trh += f'<th><div>{name}</div></th>'
+            trtotal += f'<td>{total}</td>'
+            for i in range(29):
+                scrs = self.scores[name][i][1:]
+                if not scrs:
+                    trd[i] += '<td>–</td>'
+                    continue
+                del_row[i] = False
+                worst_irpc = min([x[1] for x in scrs])
+                if worst_irpc == 0:
+                    if max([x[1] for x in scrs]) != 0:
+                        trd[i] += '<td>?</td>'
+                        continue
+                num = self.indices.consider(name, i, worst_irpc)
+                trd[i] += f'<td{self.cls(num, 384, 812)}>{num}</td>'
+
+        trh += '</tr>\n'
+        trtotal += '</tr>\n'
+        for i in range(29):
+            trd[i] += '</tr>\n'
+            if del_row[i]:
+                trd[i] = ''
+        return f'<table>{trh}{"".join(trd)}{trtotal}</table>'
+
+    def table_interrupt(self, irp, pmin=1.25, pmax=1.65):
+        maxkl = max(len(x[irp]) for x in self.scores.values())
+        trh = '<tr class="rotate"><th></th>'
+        trbest = '<tr class="small"><th>best</th>'
+        trd = [f'<tr><th>{x}</th>' for x in range(maxkl)]
+        for name in FILES_ALL:
+            maxscore = 0
+            bestkl = -1
+            try:
+                klarr = self.scores[name][irp]
+            except KeyError:
+                continue
+            trh += f'<th><div>{name}</div></th>'
+            for kl, (score, _) in enumerate(klarr):
+                if score < 0:
+                    trd[kl] += f'<td{self.cls(0)}>–</td>'
+                else:
+                    trd[kl] += f'<td{self.cls(score, pmin, pmax)}>{score:.2f}</td>'
+                if score > maxscore:
+                    maxscore = score
+                    bestkl = kl
+            trbest += f'<td>{bestkl}</td>'
+        trh += '</tr>\n'
+        trbest += '</tr>\n'
+        for i in range(29):
+            trd[i] += '</tr>\n'
+        return f'<table>{trh}{"".join(trd[1:])}{trbest}</table>'
+
+    def make(self, outfile, pmin=1.25, pmax=1.65):
+        nav = ''
+        txt = ''
+        for i in range(29):
+            has_entries = any(True for x in self.scores.values() if x[i])
+            if not has_entries:
+                continue
+            nav += f'<a href="#tb-i{i}">{RUNES[i]}</a>\n'
+            txt += f'<h3 id="tb-i{i}">Interrupt {i}: <b>{RUNES[i]}</b></h3>'
+            txt += self.table_interrupt(i, pmin, pmax)
+        html = self.template.replace('__NAVIGATION__', nav)
+        html = html.replace('__TAB_RELIABLE__', self.table_reliable())
+        html = html.replace('__INTERRUPT_TABLES__', txt)
+        with open(LPath.results(outfile), 'w') as f:
+            f.write(html)
+
+
+#########################################
+#  helper functions
+#########################################
+
+def create_initial_db(dbname, minkl=1, maxkl=32, max_irp=20, irpset=range(29)):
+    oldDB = InterruptDB.load(dbname)
+    oldValues = {k: set((a, b, c) for a, _, b, c, _ in v)
+                 for k, v in oldDB.items()}
+    for irp in irpset:  # interrupt rune index
+        for name in FILES_ALL:
+            data = load_indices(LPath.page(name), irp, maxinterrupt=max_irp)
+            db = InterruptDB(data, irp)
+            print('load:', name, 'interrupt:', irp, 'count:', db.irp_count)
+            for keylen in range(minkl, maxkl + 1):  # key length
+                if (db.irp_count, irp, keylen) in oldValues.get(name, []):
+                    print(f'{keylen}: skipped.')
+                    continue
+                score, interrupts = db.make(dbname, name, keylen)
+                print(f'{keylen}: {score:.4f}, solutions: {len(interrupts)}')
+
+
+def find_secondary_solutions(db_in, db_out, threshold=0.75, max_irp=20):
+    oldDB = InterruptDB.load(db_in)
+    search_set = set()
+    for name, arr in oldDB.items():
+        if name not in FILES_UNSOLVED:
+            continue
+        for irpc, score, irp, kl, nums in arr:
+            if score <= threshold or kl > 26 or kl < 3:
+                continue
+            search_set.add((name, irp, kl))
+    print('searching through', len(search_set), 'files.')
+    for name, irp, kl in search_set:
+        print('load:', name, 'interrupt:', irp, 'keylen:', kl)
+        data = load_indices(LPath.page(name), irp, maxinterrupt=max_irp)
+        db = InterruptDB(data, irp)
+        c = db.make_secondary(db_out, name, kl, threshold)
+        print('found', c, 'additional solutions')
+
+
+if __name__ == '__main__':
+    # find_secondary_solutions('db_high', 'db_high_secondary', threshold=1.4)
+    # find_secondary_solutions('db_norm', 'db_norm_secondary', threshold=0.55)
+    # create_initial_db('db_norm', minkl=1, maxkl=32, max_irp=20)
+    # InterruptToWeb('db_high').make('index_high.html')
+    InterruptToWeb('db_norm').make('index_norm.html', pmin=0.40, pmax=0.98)

LP/LPath.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+import os.path
+
+FILES_SOLVED = ['0_warning', '0_welcome', '0_wisdom', '0_koan_1',
+                '0_loss_of_divinity', 'jpg107-167', 'jpg229',
+                'p56_an_end', 'p57_parable']
+FILES_UNSOLVED = ['p0-2', 'p3-7', 'p8-14', 'p15-22', 'p23-26',
+                  'p27-32', 'p33-39', 'p40-53', 'p54-55']
+FILES_ALL = FILES_UNSOLVED + FILES_SOLVED
+
+LP_MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
+LP_ROOT_DIR = os.path.relpath(os.path.dirname(LP_MODULE_DIR))
+
+
+class LPath(object):
+    @staticmethod
+    def root(fname):
+        return os.path.join(LP_ROOT_DIR, fname)
+
+    @staticmethod
+    def page(fname):
+        return os.path.join(LP_ROOT_DIR, 'pages', fname + '.txt')
+
+    @staticmethod
+    def data(fname, ext='txt'):
+        return os.path.join(LP_ROOT_DIR, 'data', f'{fname}.{ext}')
+
+    @staticmethod
+    def tmp(fname, ext='txt'):
+        return os.path.join(LP_ROOT_DIR, 'tmp', f'{fname}.{ext}')
+
+    @staticmethod
+    def InterruptDB(fname):
+        return os.path.join(LP_ROOT_DIR, 'InterruptDB', fname + '.txt')
+
+    @staticmethod
+    def results(fname):
+        return os.path.join(LP_ROOT_DIR, 'InterruptDB', fname)

LP/NGrams.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+import re
+from RuneText import RUNES, re_norune, RuneText
+from LPath import LPath
+
+
+#########################################
+#  NGrams  :  loads and writes ngrams, also: translate english text to runes
+#########################################
+
+class NGrams(object):
+    @staticmethod
+    def translate(infile, outfile, stream=False):  # takes 10s
+        with open(infile, 'r') as f:
+            src = re.sub('[^A-Z]', '' if stream else ' ', f.read().upper())
+            if stream:
+                src.replace('\n', '')
+
+        with open(outfile, 'w') as f:
+            flag = False
+            for r in RuneText.from_text(src):
+                if r.kind != 'r':
+                    if not flag:
+                        f.write('\n')
+                        flag = True
+                    continue
+                f.write(r.rune)
+                flag = False
+
+    @staticmethod
+    def make(gramsize, infile, outfile):
+        with open(infile, 'r') as f:
+            data = re_norune.sub('', f.read())
+
+        res = {x: 0 for x in RUNES} if gramsize == 1 else {}
+        for i in range(len(data) - gramsize + 1):
+            ngram = data[i:i + gramsize]
+            try:
+                res[ngram] += 1
+            except KeyError:
+                res[ngram] = 1
+
+        with open(outfile, 'w') as f:
+            for x, y in sorted(res.items(), key=lambda x: -x[1]):
+                f.write(f'{x} {y}\n')
+
+    @staticmethod
+    def load(ngram=1, prefix=''):
+        ret = {}
+        with open(LPath.data(f'p{prefix}-{ngram}gram'), 'r') as f:
+            for line in f.readlines():
+                r, v = line.split()
+                ret[r] = int(v)
+        return ret
+
+
+def make_translation(stream=False):  # if true, ignore spaces / word bounds
+    NGrams.translate(LPath.data('baseline-text'),
+                     LPath.data('baseline-rune'), stream)
+
+
+def make_ngrams(max_ngram=1):
+    for i in range(1, max_ngram + 1):
+        print(f'generate {i}-gram file')
+        NGrams.make(i, infile=LPath.data('baseline-rune-words'),
+                    outfile=LPath.data(f'p-{i}gram'))
+        NGrams.make(i, infile=LPath.root('_solved.txt'),
+                    outfile=LPath.data(f'p-solved-{i}gram'))
+        NGrams.make(i, infile=LPath.data('baseline-rune-no-e'),
+                    outfile=LPath.data(f'p-no-e-{i}gram'))
+
+
+# make_translation(stream=False)
+# make_ngrams(5)

LP/RuneRunner.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+import sys
+from RuneText import RuneText
+import lib as utils
+
+
+#########################################
+#  RuneWriter  :  handle std output with highlight etc.
+#########################################
+
+class RuneWriter(object):
+    MARKS = {'_': '\x1b[0m', 'r': '\x1b[31;01m', 'b': '\x1b[30;01m'}
+
+    def __init__(self):
+        self.COLORS = True
+        self.VERBOSE = '-v' in sys.argv
+        self.QUIET = '-q' in sys.argv
+        self.BREAK_MODE = None
+        self.file_output = None
+        self.clear()
+
+    def clear(self):
+        self.mark = False
+        self.alternate = False
+        self._marked = ['_'] * 4
+        self.txt = [''] * 4
+
+    def is_empty(self):
+        return not self.txt[0]
+
+    def line_break_mode(self):
+        if self.BREAK_MODE is not None:  # if set by user
+            return self.BREAK_MODE
+        return 'l' if self.VERBOSE else 's'  # dynamically adapt to mode
+
+    def write(self, r=None, t=None, n1=None, n2=None):
+        m = ('b' if self.alternate else 'r') if self.mark else '_'
+        for i, v in enumerate([r, t, n1, n2]):
+            if v is None:
+                continue
+            if self.COLORS and self._marked[i] != m and i != 3:
+                self._marked[i] = m
+                prfx = self.MARKS[m]
+            else:
+                prfx = ''
+            self.txt[i] += prfx + v
+
+    # def rm(self, r=0, t=0, n1=0, n2=0):
+    #     for i, v in enumerate([r, t, n1, n2]):
+    #         if v > 0:
+    #             self.txt[i] = self.txt[i][:-v]
+
+    def stdout(self):
+        def print_f(x=''):
+            if self.file_output:
+                with open(self.file_output, 'a') as f:
+                    f.write(x + '\n')
+            else:
+                print(x)
+
+        if self.is_empty():
+            return
+        m = self.mark
+        self.mark = False  # flush closing color
+        self.write(r='', t='', n1='', n2='')
+        self.mark = m
+
+        if not self.QUIET or self.VERBOSE:
+            print_f()
+        if not self.QUIET:
+            print_f(self.txt[0])
+        print_f(self.txt[1])
+        if self.VERBOSE:
+            print_f(self.txt[2])
+            print_f(self.txt[3])
+        self.clear()
+
+
+#########################################
+#  RuneReader  :  handles parsing of the file and line breaks etc.
+#########################################
+
+class RuneReader(object):
+    def __init__(self):
+        self.data = None
+        self.loaded_file = None
+        self.words = {x: [] for x in range(20)}  # increase for longer words
+
+    def load(self, data=None, file=None, limit=None):
+        self.loaded_file = None
+        if not data:
+            with open(file, 'r') as f:
+                data = f.read()[:limit]
+                self.loaded_file = file
+        self.data = data if isinstance(data, RuneText) else RuneText(data)
+        self.generate_word_list()
+
+    def has_data(self):
+        if len(self.data) > 0:
+            return True
+        return False
+
+    def runes_no_whitespace(self):
+        return [x for x in self.data if x.kind == 'r']
+
+    def generate_word_list(self):
+        for x in self.words.values():
+            x.clear()
+        res = []
+        ai = 0
+        ari = 0
+        zri = 0
+        for zi, x in enumerate(self.data):
+            if x.kind == 'l':
+                continue
+            elif x.kind == 'r':
+                res.append(x)
+                zri += 1
+            else:
+                if len(res) > 0:
+                    xt = RuneText(res)
+                    self.words[len(xt)].append((ai, zi, ari, zri, xt))
+                res = []
+                ai = zi
+                ari = zri
+
+    # count_callback('c|w|l', count, is-first-flag)
+    def parse(self, rune_fn, count_fn, whitespace_fn, break_line_on='l'):
+        word_sum = 0
+        line_sum = 0
+        for i, x in enumerate(self.data):
+            if x.kind == 'r':
+                r = rune_fn(self.data, i, word_sum == 0)
+                count_fn('c', r.prime, word_sum == 0)
+                word_sum += r.prime
+            elif x.kind == 'l' and x.kind != break_line_on:
+                continue  # ignore all \n,\r if not forced explicitly
+            else:
+                if word_sum > 0:
+                    count_fn('w', word_sum, line_sum == 0)
+                    line_sum += word_sum
+                    word_sum = 0
+                if x.kind != 'l':  # still ignore \n,\r
+                    whitespace_fn(x)
+                if x.kind == break_line_on:
+                    count_fn('l', line_sum, line_sum == 0)
+                    line_sum = 0
+        if word_sum > 0:
+            count_fn('w', word_sum, line_sum == 0)
+            line_sum += word_sum
+        if line_sum > 0:
+            count_fn('l', line_sum, True)
+
+
+#########################################
+#  RuneRunner  :  Merge RuneWriter and RuneReader and stay in sync
+#########################################
+
+class RuneRunner(object):
+    def __init__(self):
+        self.input = RuneReader()
+        self.output = RuneWriter()
+        self.marked_chars = []
+        self.mark_alternate = []
+        self.next_mark = False
+        self.fn_cipher = None
+
+    def highlight_words_with_len(self, search_length):
+        found = [x for x in self.input.words[search_length]]
+        self.marked_chars = set(x for fp in found for x in range(fp[0], fp[1]))
+        return found
+
+    def highlight_rune(self, rune, mark_occurrences=[]):
+        ip = 0
+        tp = 0
+        ret = []
+        for i, x in enumerate(self.input.data):
+            if x.kind == 'r':
+                if x.rune == rune:
+                    ip += 1
+                    ret.append((ip, tp, i, ip in mark_occurrences))
+                tp += 1
+        self.marked_chars = set(i for _, _, i, _ in ret)
+        self.mark_alternate = set(i for _, _, i, f in ret if not f)
+        return ret
+
+    def reset_highlight(self):
+        self.marked_chars = []
+        self.mark_alternate = []
+
+    def start(self, fn_cipher):
+        self.fn_cipher = fn_cipher
+        self.next_mark = False
+        self.input.parse(
+            self.rune_callback, self.count_callback, self.whitespace_callback,
+            self.output.line_break_mode())
+
+    def rune_callback(self, encrypted_data, index, is_first):
+        if self.output.VERBOSE:
+            fillup = len(self.output.txt[2]) - len(self.output.txt[1])
+            if not is_first:
+                fillup += 1  # +1 cause n1 will add a '+'
+            if fillup > 0:
+                self.output.write(t=' ' * fillup)
+        if self.marked_chars:
+            x = encrypted_data[index]  # always search on original data
+            mt = index in self.marked_chars
+            mn = index + 1 in self.marked_chars
+            self.output.alternate = index in self.mark_alternate
+        else:
+            x, mt, mn = self.fn_cipher(encrypted_data, index)
+        self.output.mark = mt
+        self.output.write(r=x.rune, t=x.text)
+        self.next_mark = mn
+        return x
+
+    def count_callback(self, typ, num, is_first):
+        if typ == 'c':  # char
+            if self.output.VERBOSE:
+                self.output.write(n1=('' if is_first else '+') + str(num))
+            return
+        prm = utils.is_prime(num)
+        if typ == 'w':  # word
+            tt = ('' if is_first else ' + ') + str(num) + ('*' if prm else '')
+            self.output.write(n2=tt)
+            if prm and num > 109 and not (self.output.VERBOSE or self.output.QUIET):
+                self.output.write(t='__')
+        elif typ == 'l':  # line end (ignoring \n if mode is set to 's')
+            self.output.mark = False
+            # if not is_first:
+            sffx = ' = {}'.format(num) + ('*' if prm else '')
+            if utils.is_emirp(num):
+                sffx += '√'
+            if self.output.VERBOSE:
+                self.output.write(n2=sffx)
+            elif not self.output.QUIET:
+                self.output.write(t=sffx)
+            self.output.stdout()
+
+    def whitespace_callback(self, rune):
+        if not self.next_mark:  # dont mark whitespace after selection
+            self.output.mark = False
+        self.output.write(r=rune.rune, t=rune.text)
+        if self.output.VERBOSE:
+            self.output.write(n1=' ')

LP/RuneSolver.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+from RuneRunner import RuneRunner
+from RuneText import Rune, RuneText
+from lib import affine_decrypt
+
+
+#########################################
+#  RuneSolver  :  Generic parent class handles interrupts and text highlight
+#########################################
+
+class RuneSolver(RuneRunner):
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        self.INTERRUPT = 'ᚠ'
+        self.INTERRUPT_POS = []  # '1' for first occurrence of INTERRUPT
+
+    def highlight_interrupt(self):
+        return self.highlight_rune(self.INTERRUPT, self.INTERRUPT_POS)
+
+    def substitute_get(self, pos, keylen, search_term, found_term):
+        return found_term.zip_sub(search_term).description(count=True)
+
+    def substitute_supports_keylen(self):
+        return False
+
+    def run(self, data=None):
+        if data:
+            self.input.load(data=data)
+        self.interrupt_counter = 0
+        self.start(self.cipher_callback)
+
+    def cipher_callback(self, encrypted_data, index):
+        obj = encrypted_data[index]
+        is_interrupt = obj.rune == self.INTERRUPT
+        if is_interrupt:
+            self.interrupt_counter += 1
+        skip = is_interrupt and self.interrupt_counter in self.INTERRUPT_POS
+        mark_this = self.mark_char_at(index)
+        if not skip:
+            obj = self.cipher(obj, (index, encrypted_data))
+        mark_next = self.mark_char_at(index)
+        return obj, mark_this, mark_next
+
+    def cipher(self, rune, context):
+        raise NotImplementedError  # must subclass
+
+    def mark_char_at(self, position):
+        return False
+
+    def __str__(self):
+        txt = f'DATA: {len(self.input.data) if self.input.data else 0} bytes'
+        if self.input.loaded_file:
+            txt += f' (file: {self.input.loaded_file})'
+        else:
+            txt += f' (manual input)'
+        return txt + f'\ninterrupt jumps: {self.INTERRUPT_POS}'
+
+
+#########################################
+#  SequenceSolver  :  Decrypt runes with sequential function
+#########################################
+
+class SequenceSolver(RuneSolver):
+    def __init__(self):
+        super().__init__()
+        self.seq_index = 0
+        self.reset()
+
+    def reset(self):
+        super().reset()
+        self.FN = None
+
+    def run(self, data=None):
+        self.seq_index = 0
+        assert(self.FN)
+        super().run(data=data)
+
+    def cipher(self, rune, context):
+        x = self.FN(self.seq_index, rune)
+        self.seq_index += 1
+        return x
+
+    def __str__(self):
+        return super().__str__() + f'\nf(x): {self.FN}'
+
+
+#########################################
+#  RunningKeySolver  :  Decrypt runes with key; handles shift, rotation, etc.
+#########################################
+
+class RunningKeySolver(RuneSolver):
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        super().reset()
+        self.KEY_DATA = []  # the key material
+        self.KEY_INVERT = False  # ABCD -> ZYXW
+        self.KEY_SHIFT = 0  # ABCD -> DABC
+        self.KEY_ROTATE = 0  # ABCD -> ZABC
+        self.KEY_OFFSET = 0  # ABCD -> __ABCD
+        self.KEY_POST_PAD = 0  # ABCD -> ABCD__
+
+    def run(self, data=None):
+        self.k_current_pos = 0
+        self.k_len = len(self.KEY_DATA)
+        self.k_full_len = self.KEY_OFFSET + self.k_len + self.KEY_POST_PAD
+        super().run(data=data)
+
+    def mark_char_at(self, position):
+        return self.active_key_pos() != -1
+
+    def active_key_pos(self):
+        i = self.k_current_pos - self.KEY_OFFSET
+        if i >= 0 and i < self.k_len:
+            if self.KEY_DATA[i] != 29:  # placeholder for unknown
+                return i
+        return -1
+
+    def cipher(self, rune, context):
+        r_idx = rune.index
+        if self.KEY_INVERT:
+            r_idx = 28 - r_idx
+        pos = self.active_key_pos()
+        if pos == -1:
+            self.copy_unmodified(r_idx)
+        else:
+            i = (pos + self.KEY_SHIFT) % self.k_len
+            r_idx = (self.decrypt(r_idx, i) - self.KEY_ROTATE) % 29
+        # rotate_key
+        if self.k_full_len > 0:  # e.g., for key invert without a key
+            self.k_current_pos = (self.k_current_pos + 1) % self.k_full_len
+        return Rune(i=r_idx)
+
+    def decrypt(self, rune_index, key_index):  # must subclass
+        raise NotImplementedError
+
+    def copy_unmodified(self, rune_index):  # subclass if needed
+        pass
+
+    def key__str__(self):
+        return self.KEY_DATA  # you should override this
+
+    def key__str__basic_runes(self):
+        return RuneText(self.KEY_DATA).description(indexWhitespace=True)
+
+    def __str__(self):
+        txt = super().__str__()
+        txt += f'\nkey: {self.key__str__()}'
+        txt += f'\nkey invert: {self.KEY_INVERT}'
+        txt += f'\nkey offset: {self.KEY_OFFSET} runes'
+        txt += f'\nkey post pad: {self.KEY_POST_PAD} runes'
+        txt += f'\nkey shift: {self.KEY_SHIFT} indices'
+        txt += f'\nkey rotate: {self.KEY_ROTATE} indices'
+        return txt
+
+
+#########################################
+#  VigenereSolver  :  Decrypt runes with an array of indices
+#########################################
+
+class VigenereSolver(RunningKeySolver):
+    def decrypt(self, rune_index, key_index):
+        return rune_index - self.KEY_DATA[key_index]
+
+    def substitute_supports_keylen(self):
+        return True
+
+    def substitute_get(self, pos, keylen, search_term, found_term):
+        ret = [Rune(r='⁚')] * keylen
+        for i, r in enumerate(found_term.zip_sub(search_term)):
+            ret[(pos + i) % keylen] = r
+        return RuneText(ret).description(count=True, index=False)
+
+    def key__str__(self):
+        return self.key__str__basic_runes()
+
+
+#########################################
+#  AffineSolver  :  Decrypt runes with an array of (s, t) affine keys
+#########################################
+
+class AffineSolver(RunningKeySolver):
+    def decrypt(self, rune_index, key_index):
+        return affine_decrypt(rune_index, self.KEY_DATA[key_index])
+
+
+#########################################
+#  AutokeySolver  :  Decrypts runes by using previously decrypted ones as input
+#########################################
+
+class AutokeySolver(RunningKeySolver):
+    def run(self, data=None):
+        key = self.KEY_DATA[self.KEY_SHIFT:] + self.KEY_DATA[:self.KEY_SHIFT]
+        key = [29] * self.KEY_OFFSET + key + [29] * self.KEY_POST_PAD
+        self.running_key = key
+        super().run(data=data)
+
+    def decrypt(self, rune_index, _):
+        rune_index = (rune_index - self.running_key.pop(0)) % 29
+        self.running_key.append(rune_index)
+        return rune_index
+
+    def copy_unmodified(self, rune_index):
+        if self.k_len > 0:
+            self.running_key.pop(0)
+            self.running_key.append(rune_index)
+
+    def substitute_supports_keylen(self):
+        return True
+
+    def substitute_get(self, pos, keylen, search_term, found_term):
+        data = self.input.runes_no_whitespace()
+        ret = [Rune(r='⁚')] * keylen
+        for o in range(len(search_term)):
+            plain = search_term[o]
+            i = pos + o
+            while i >= 0:
+                plain = data[i] - plain
+                i -= keylen
+            ret[i + keylen] = plain
+        return RuneText(ret).description(count=True, index=False)
+
+    def key__str__(self):
+        return self.key__str__basic_runes()

LP/RuneText.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+import re  # load_indices
+
+white_rune = {'•': ' ', '⁘': '.', '⁚': ',', '⁖': ';', '⁜': '#'}
+white_text = {v: k for k, v in white_rune.items()}
+alphabet = [  # Using last value for display. Custom added: V
+    (2, 'ᚠ', ['F']), (3, 'ᚢ', ['V', 'U']), (5, 'ᚦ', ['TH']), (7, 'ᚩ', ['O']),
+    (11, 'ᚱ', ['R']), (13, 'ᚳ', ['K', 'C']), (17, 'ᚷ', ['G']),
+    (19, 'ᚹ', ['W']), (23, 'ᚻ', ['H']), (29, 'ᚾ', ['N']), (31, 'ᛁ', ['I']),
+    (37, 'ᛄ', ['J']), (41, 'ᛇ', ['EO']), (43, 'ᛈ', ['P']), (47, 'ᛉ', ['X']),
+    (53, 'ᛋ', ['Z', 'S']), (59, 'ᛏ', ['T']), (61, 'ᛒ', ['B']),
+    (67, 'ᛖ', ['E']), (71, 'ᛗ', ['M']), (73, 'ᛚ', ['L']),
+    (79, 'ᛝ', ['ING', 'NG']), (83, 'ᛟ', ['OE']), (89, 'ᛞ', ['D']),
+    (97, 'ᚪ', ['A']), (101, 'ᚫ', ['AE']), (103, 'ᚣ', ['Y']),
+    (107, 'ᛡ', ['IO', 'IA']), (109, 'ᛠ', ['EA'])
+]
+text_map = {t: r for _, r, ta in alphabet for t in ta}
+rune_map = {r: t for _, r, ta in alphabet for t in ta}
+primes_map = {r: p for p, r, _ in alphabet}
+RUNES = [r for _, r, _ in alphabet]  # array already sorted
+re_norune = re.compile('[^' + ''.join(RUNES) + ']')
+# del alphabet  # used in playground for GP display
+
+
+#########################################
+#  Rune  :  Stores a single rune. Incl. text, prime, index, and kind
+#########################################
+
+class Rune(object):
+    def __init__(self, r=None, t=None, i=None, p=None):
+        self._rune = r
+        self._text = t
+        self._index = i
+        self._prime = p
+        self._kind = None  # one of: r n s l w
+
+    def __repr__(self):
+        return f'<{self._rune}, {self._text}, {self._index}, {self._prime}>'
+
+    @property
+    def rune(self):
+        if self._rune is None:
+            self._rune = RUNES[self._index] if self._index < 29 else '•'
+        return self._rune
+
+    @property
+    def text(self, sameWhitespace=False):
+        if self._text is None:
+            if sameWhitespace:
+                self._text = rune_map.get(self.rune, ' ')
+            else:
+                r = self.rune
+                self._text = rune_map.get(r, white_rune.get(r, r))
+        return self._text
+
+    @property
+    def index(self):
+        if self._index is None:
+            r = self._rune
+            self._index = RUNES.index(r) if r in RUNES else 29
+        return self._index
+
+    @property
+    def prime(self):
+        if self._prime is None:
+            self._prime = primes_map.get(self.rune, 0)
+        return self._prime
+
+    @property
+    def kind(self):
+        if self._kind is None:
+            x = self.rune
+            if x in rune_map:
+                self._kind = 'r'  # rune
+            elif x == '⁜':
+                self._kind = 's'  # paragraph, but treat as sentence
+            elif x == '⁘':
+                self._kind = 's'  # sentence
+            elif x == '\n' or x == '\r':
+                self._kind = 'l'  # line end
+            elif x in '1234567890':
+                self._kind = 'n'  # number
+            else:
+                self._kind = 'w'  # whitespace (explicitly not n or s)
+        return self._kind
+
+    def __add__(self, o):
+        if isinstance(o, Rune):
+            o = o.index
+        if self.index == 29 or o == 29:
+            return self
+        return Rune(i=(self.index + o) % 29)
+
+    def __sub__(self, o):
+        if isinstance(o, Rune):
+            o = o.index
+        if self.index == 29 or o == 29:
+            return self
+        return Rune(i=(self.index - o) % 29)
+
+    def __radd__(self, o):
+        return self if self.index == 29 else Rune(i=(o + self.index) % 29)
+
+    def __rsub__(self, o):
+        return self if self.index == 29 else Rune(i=(o - self.index) % 29)
+
+    def __invert__(self):
+        return self if self.index == 29 else Rune(i=28 - self.index)
+
+
+#########################################
+#  RuneText  :  Stores multiple Rune objects. Allows arithmetic operations
+#########################################
+
+class RuneText(object):
+    def __init__(self, anything):
+        self._rune_sum = None
+        if not anything:
+            self._data = []
+        elif isinstance(anything, list):
+            if len(anything) > 0 and isinstance(anything[0], Rune):
+                self._data = anything
+            else:
+                self._data = [Rune(i=x) for x in anything]
+        else:
+            txt = anything.strip()
+            if not txt:
+                self._data = []
+            elif txt[0] in rune_map or txt[0] in white_rune:
+                self._data = [Rune(r=x) for x in txt]
+            else:
+                try:
+                    self._data = [
+                        Rune(i=int(x)) for x in txt.strip('[]').split(',')
+                    ]
+                except ValueError:
+                    self._data = self.from_text(txt)
+
+        self._data_len = len(self._data)
+
+    def __len__(self):
+        return self._data_len
+
+    def trim(self, maxlen):
+        if self._data_len > maxlen:
+            if self._rune_sum and self._rune_sum > 0:
+                self._rune_sum -= sum(x.prime for x in self._data[maxlen:])
+            self._data = self._data[:maxlen]
+            self._data_len = maxlen
+
+    @classmethod
+    def from_text(self, text):
+        res = []
+        text = text.strip().upper().replace('QU', 'CW')
+        tlen = len(text)
+        skip = 0
+        for i in range(tlen):
+            if skip:
+                skip -= 1
+                continue
+            char = text[i]
+            rune = None
+            wspace = white_text.get(char, None)
+            if wspace is not None:
+                rune = wspace
+            elif char in '\"\'\n\r\t1234567890':
+                rune = char
+            else:
+                if char in 'TINEOA' and i + 1 < tlen:
+                    bichar = char + text[i + 1]
+                    rune = text_map.get(bichar, None)
+                    if rune is not None:
+                        char = bichar
+                        skip = 1
+                    elif char == 'I' and i + 2 < tlen:
+                        trichar = bichar + text[i + 2]
+                        rune = text_map.get(trichar, None)
+                        if rune:
+                            char = trichar
+                            skip = 2
+                if not rune:
+                    rune = text_map.get(char, None)
+            if not rune:
+                raise ValueError(f'Unkn0n char: {i} "{char}"')
+            res.append(Rune(r=rune, t=char))
+        return res
+
+    def description(self, count=False, index=True, indexWhitespace=False):
+        return None if len(self) == 0 else \
+            self.rune + (f' ({len(self)})' if count else '') + ' - ' + \
+            self.text + (f' ({len(self.text)})' if count else '') + \
+            (f' - {self.index if indexWhitespace else self.index_rune_only}'
+             if index else '')
+
+    def zip_sub(self, other):
+        if len(self) != len(other):
+            raise IndexError('RuneText length mismatch')
+        return RuneText([x - y for x, y in zip(self._data, other._data)])
+
+    @property
+    def text(self):
+        return ''.join(x.text for x in self._data)
+
+    @property
+    def rune(self):
+        return ''.join(x.rune for x in self._data)
+
+    @property
+    def index(self):
+        return [x.index for x in self._data if x.kind != 'l']
+
+    @property
+    def index_rune_only(self):
+        return [x.index for x in self._data if x.index != 29]
+
+    @property
+    def prime(self):
+        return [x.prime for x in self._data]
+
+    @property
+    def prime_sum(self):
+        if self._rune_sum is None:
+            self._rune_sum = sum(self.prime)
+        return self._rune_sum
+
+    def __getitem__(self, key):
+        if isinstance(key, str):
+            return [getattr(x, key) for x in self._data]
+        else:
+            return self._data[key]
+
+    def __setitem__(self, key, value):
+        self._data[key] = value
+
+    def __add__(self, other):
+        return RuneText([x + other for x in self._data])
+
+    def __sub__(self, other):
+        return RuneText([x - other for x in self._data])
+
+    def __radd__(self, other):
+        return RuneText([other + x for x in self._data])
+
+    def __rsub__(self, other):
+        return RuneText([other - x for x in self._data])
+
+    def __invert__(self):
+        return RuneText([~x for x in self._data])
+
+    def __repr__(self):
+        return f'RuneText<{len(self._data)}>'
+
+
+#########################################
+#  load page and convert to indices for faster access
+#########################################
+
+def load_indices(fname, interrupt, maxinterrupt=None, minlen=None, limit=None):
+    with open(fname, 'r') as f:
+        data = RuneText(re_norune.sub('', f.read())).index_rune_only[:limit]
+    if maxinterrupt is not None:
+        # incl. everything up to but not including next interrupt
+        # e.g., maxinterrupt = 0 will return text until first interrupt
+        for i, x in enumerate(data):
+            if x != interrupt:
+                continue
+            if maxinterrupt == 0:
+                if minlen and i < minlen:
+                    continue
+                return data[:i]
+            maxinterrupt -= 1
+    return data

LP/__init__.py

@@ -0,0 +1,17 @@
+import sys
+if True:
+    sys.path.append(__path__[0])
+
+import lib as utils
+from LPath import FILES_ALL, FILES_UNSOLVED, FILES_SOLVED
+from LPath import LPath as path
+from RuneSolver import VigenereSolver, AffineSolver, AutokeySolver, SequenceSolver
+from RuneText import Rune, RuneText
+from RuneText import RUNES, alphabet, load_indices
+
+from HeuristicSearch import GuessVigenere, GuessAffine, GuessPattern
+from HeuristicSearch import SearchInterrupt
+from HeuristicLib import Probability
+from InterruptDB import InterruptDB
+
+from FailedAttempts import NGramShifter

LP/lib.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+import math
+
+
+def is_prime(num):
+    if isinstance(num, str):
+        num = int(num)
+    if num in [2, 3, 5]:
+        return True
+    if num & 1 and num % 5 > 0:
+        for i in range(2, math.floor(math.sqrt(num)) + 1):
+            if i & 1 and (num % i) == 0:
+                return False
+        return True
+    return False
+
+
+def rev(num):  # or int(str(num)[::-1])
+    if isinstance(num, str):
+        num = int(num)
+    revs = 0
+    while (num > 0):
+        remainder = num % 10
+        revs = (revs * 10) + remainder
+        num = num // 10
+    return revs
+
+
+def is_emirp(num):
+    return is_prime(rev(num))
+
+
+def power(x, y, p):
+    res = 1
+    x %= p
+    while (y > 0):
+        if (y & 1):
+            res = (res * x) % p
+        y = y >> 1
+        x = (x * x) % p
+    return res
+
+
+def sqrtNormal(n, p):
+    n %= p
+    for x in range(2, p):
+        if ((x * x) % p == n):
+            return x
+    return None
+
+
+# Assumption: p is of the form 3*i + 4 where i >= 1
+def sqrtFast(n, p):
+    if (p % 4 != 3):
+        # raise ValueError('Invalid Input')
+        return sqrtNormal(n, p)
+    # Try "+(n ^ ((p + 1)/4))"
+    n = n % p
+    x = power(n, (p + 1) // 4, p)
+    if ((x * x) % p == n):
+        return x
+    # Try "-(n ^ ((p + 1)/4))"
+    x = p - x
+    if ((x * x) % p == n):
+        return x
+    return None
+
+
+def elliptic_curve(x, a, b, r):
+    y2 = (x ** 3 + a * x + b) % r
+    y = sqrtFast(y2, r) if y2 > 0 else 0
+    if y is None:
+        return None, None
+    return y, -y % r
+
+
+AFFINE_INV = None
+
+
+def affine_inverse(s, n=29):
+    def fn(s, n):
+        g = [n, s]
+        u = [1, 0]
+        v = [0, 1]
+        y = [None]
+        i = 1
+        while g[i] != 0:
+            y.append(g[i - 1] // g[i])
+            g.append(g[i - 1] - y[i] * g[i])
+            u.append(u[i - 1] - y[i] * u[i])
+            v.append(v[i - 1] - y[i] * v[i])
+            i += 1
+        return v[-2] % n
+
+    global AFFINE_INV
+    if AFFINE_INV is None:
+        AFFINE_INV = [fn(x, n) for x in range(n)]
+    return AFFINE_INV[s]
+
+
+def affine_decrypt(x, key, n=29):  # key: (s, t)
+    return ((x - key[1]) * affine_inverse(key[0], n)) % n
+
+
+def autokey_reverse(data, keylen, pos, search_term):
+    ret = [29] * keylen
+    for o in range(len(search_term)):
+        plain = search_term[o]
+        i = pos + o
+        while i >= 0:
+            plain = (data[i] - plain) % 29
+            i -= keylen
+        ret[i + keylen] = plain
+    return ret
+
+# alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+# cipher = 'YDIDWYASDDJVAPJMMBIASDTJVAMD'
+# indices = [affine_decrypt(alphabet.index(x), (5, 9), 26) for x in cipher]
+# print(''.join(alphabet[x] for x in indices))