filter by key score similarity + baseline probability for text without e

HeuristicLib.py

@@ -12,11 +12,14 @@ def normalized_probability(int_prob):
 RUNES = 'ᚠᚢᚦᚩᚱᚳᚷᚹᚻᚾᛁᛄᛇᛈᛉᛋᛏᛒᛖᛗᛚᛝᛟᛞᚪᚫᚣᛡᛠ'
 re_norune = re.compile('[^' + RUNES + ']')
 PROB_INT = [0] * 29
-for k, v in NGrams.load().items():
+# for k, v in NGrams.load(1, '-no-e').items():  # 1.8271530001197518
+for k, v in NGrams.load().items():  # 1.7736851725202398
     PROB_INT[RUNES.index(k)] = v
 PROB_NORM = normalized_probability(PROB_INT)
 K_r = 1 / 29   # 0.034482758620689655
 K_p = sum(x ** 2 for x in PROB_INT)  # 0.06116195419412538
+N_total = (sum(PROB_INT) * (sum(PROB_INT) - 1)) / 29
+PROB_TARGET = sum(x * (x - 1) for x in PROB_INT) / N_total
 
 
 #########################################

HeuristicSearch.py

@@ -14,6 +14,7 @@ class GuessVigenere(object):
 
     def guess(self, keylength, score_fn):  # minimize score_fn
         found = []
+        avg_score = 0
         for offset in range(keylength):
             bi = -1
             bs = 9999999
@@ -23,8 +24,9 @@ class GuessVigenere(object):
                 if score < bs:
                     bs = score
                     bi = i
+            avg_score += bs
             found.append(bi)
-        return found
+        return avg_score / keylength, found
 
 
 #########################################
@@ -37,6 +39,7 @@ class GuessAffine(object):
 
     def guess(self, keylength, score_fn):  # minimize score_fn
         found = []
+        avg_score = 0
         for offset in range(keylength):
             candidate = (None, None)
             best = 9999999
@@ -48,8 +51,9 @@ class GuessAffine(object):
                     if score < best:
                         best = score
                         candidate = (s, t)
+            avg_score += best
             found.append(candidate)
-        return found
+        return avg_score / keylength, found
 
 
 #########################################
@@ -114,7 +118,7 @@ class SearchInterrupt(object):
         # first step: move maxdepth-sized window over data
         i = startAt - 1  # in case loop isnt called
         for i in range(startAt, len(self.stops) - maxdepth):
-            print('.', end='')
+            # print('.', end='')
             parts, _ = best_in_all(i, maxdepth)
             found = []
             search = self.stops[i]
@@ -132,7 +136,7 @@ class SearchInterrupt(object):
                     found.append(prfx + [search])
                 if bitNotSet:
                     found.append(prfx)
-        print('.')
+        # print('.')
         # last step: all permutations for the remaining (< maxdepth) bits
         i += 1
         remaining, score = best_in_all(i, min(maxdepth, len(self.stops) - i))

InterruptDB.py

@@ -17,8 +17,6 @@ FILES_ALL = FILES_UNSOLVED + FILES_SOLVED
 #########################################
 
 class InterruptDB(object):
-    DB_NAME = 'InterruptDB/db_main.txt'
-
     def __init__(self, data, interrupt):
         self.irp = interrupt
         self.iguess = SearchInterrupt(data, interrupt)
@@ -37,14 +35,13 @@ class InterruptDB(object):
             skips[i] = self.iguess.to_occurrence_index(interrupts)
         return score, skips
 
-    def make_keylength(self, name, keylen, db_path=DB_NAME):
+    def make_keylength(self, name, keylen, dbname='db_main'):
         score, interrupts = self.make(keylen)
         for nums in interrupts:
-            self.write(name, score, self.irp, self.irp_count, keylen, nums,
-                       db_path=db_path)
+            self.write(name, score, self.irp, self.irp_count, keylen, nums, dbname)
         return score, interrupts
 
-    def find_secondary(self, name, keylen, threshold, db_path=DB_NAME):
+    def find_secondary(self, name, keylen, threshold, dbname='db_main'):
         scores = []
 
         def fn(x):
@@ -62,16 +59,15 @@ class InterruptDB(object):
         # exclude best results, as they are already present in the main db
         filtered = [x for x in ret if x[0] < bestscore]
         for score, nums in filtered:
-            self.write(name, score, self.irp, self.irp_count, keylen, nums,
-                       db_path=db_path)
+            self.write(name, score, self.irp, self.irp_count, keylen, nums, dbname)
         return len(filtered)
 
     @staticmethod
-    def load(db_path=DB_NAME):
-        if not os.path.isfile(db_path):
+    def load(dbname='db_main'):
+        if not os.path.isfile(f'InterruptDB/{dbname}.txt'):
             return {}
         ret = {}
-        with open(db_path, 'r') as f:
+        with open(f'InterruptDB/{dbname}.txt', 'r') as f:
             for line in f.readlines():
                 if line.startswith('#'):
                     continue
@@ -86,8 +82,8 @@ class InterruptDB(object):
         return ret
 
     @staticmethod
-    def write(name, score, irp, irpmax, keylen, nums, db_path=DB_NAME):
-        with open(db_path, 'a') as f:
+    def write(name, score, irp, irpmax, keylen, nums, dbname='db_main'):
+        with open(f'InterruptDB/{dbname}.txt', 'a') as f:
             nums = ','.join(map(str, nums))
             f.write(f'{name}|{irpmax}|{score:.5f}|{irp}|{keylen}|{nums}\n')
 
@@ -97,8 +93,6 @@ class InterruptDB(object):
 #########################################
 
 class InterruptIndices(object):
-    DB_NAME = 'InterruptDB/db_indices.txt'
-
     def __init__(self):
         self.pos = InterruptIndices.read()
 
@@ -112,8 +106,8 @@ class InterruptIndices(object):
         return self.pos[name]['total']
 
     @staticmethod
-    def write():
-        with open(InterruptIndices.DB_NAME, 'w') as f:
+    def write(dbname='db_indices'):
+        with open(f'InterruptDB/{dbname}.txt', 'w') as f:
             f.write('# file | total runes in file | interrupt | indices\n')
             for name in FILES_ALL:
                 fname = f'pages/{name}.txt'
@@ -127,8 +121,8 @@ class InterruptIndices(object):
                         name, total, irp, ','.join(map(str, pos))))
 
     @staticmethod
-    def read():
-        with open(InterruptIndices.DB_NAME, 'r') as f:
+    def read(dbname='db_indices'):
+        with open(f'InterruptDB/{dbname}.txt', 'r') as f:
             ret = {}
             for line in f.readlines():
                 if line.startswith('#'):
@@ -242,25 +236,25 @@ class InterruptToWeb(object):
 #  helper functions
 #########################################
 
-def create_initial_db(min_kl=1, max_kl=32, max_irp=20):
-    oldDB = InterruptDB.load()
+def create_initial_db(dbname, minkl=1, maxkl=32, max_irp=20, irpset=range(29)):
+    oldDB = InterruptDB.load(dbname)
     oldValues = {k: set((a, b, c) for a, _, b, c, _ in v)
                  for k, v in oldDB.items()}
-    for irp in range(29):  # interrupt rune index
-        for name in FILES_ALL:  # filename
+    for irp in irpset:  # interrupt rune index
+        for name in FILES_UNSOLVED:  # filename
             fname = f'pages/{name}.txt'
             data = load_indices(fname, irp, maxinterrupt=max_irp)
             db = InterruptDB(data, irp)
             print('load:', fname, 'interrupt:', irp, 'count:', db.irp_count)
-            for keylen in range(min_kl, max_kl + 1):  # key length
+            for keylen in range(minkl, maxkl + 1):  # key length
                 if (db.irp_count, irp, keylen) in oldValues.get(name, []):
                     print(f'{keylen}: skipped.')
                     continue
-                score, interrupts = db.make_keylength(name, keylen)
+                score, interrupts = db.make_keylength(name, keylen, dbname)
                 print(f'{keylen}: {score:.4f}, solutions: {len(interrupts)}')
 
 
-def find_secondary_solutions(max_irp=20, threshold=1.4):
+def find_secondary_solutions(dbname, max_irp=20, threshold=1.4):
     oldDB = InterruptDB.load()
     search_set = set()
     for name, arr in oldDB.items():
@@ -277,11 +271,11 @@ def find_secondary_solutions(max_irp=20, threshold=1.4):
         data = load_indices(fname, irp, maxinterrupt=max_irp)
         db = InterruptDB(data, irp)
         c = db.find_secondary(name, kl, threshold,
-                              db_path='InterruptDB/db_secondary.txt')
+                              db_path=f'InterruptDB/{dbname}.txt')
         print('found', c, 'additional solutions')
 
 
 if __name__ == '__main__':
-    # find_secondary_solutions()
-    # create_initial_db(min_kl=1, max_kl=32, max_irp=20)
+    # find_secondary_solutions('db_secondary')
+    # create_initial_db('db_main', minkl=1, maxkl=32, max_irp=20)
     InterruptToWeb('InterruptDB/template.html').make('InterruptDB/index.html')

NGrams.py

@@ -45,9 +45,9 @@ class NGrams(object):
                 f.write(f'{x} {y}\n')
 
     @staticmethod
-    def load(ngram=1):
+    def load(ngram=1, prefix=''):
         ret = {}
-        with open(f'data/p-{ngram}gram.txt', 'r') as f:
+        with open(f'data/p{prefix}-{ngram}gram.txt', 'r') as f:
             for line in f.readlines():
                 r, v = line.split()
                 ret[r] = int(v)
@@ -61,3 +61,5 @@ class NGrams(object):
 #                 outfile=f'data/p-{i}gram.txt')
 #     NGrams.make(i, infile='_solved.txt',
 #                 outfile=f'data/p-solved-{i}gram.txt')
+#     NGrams.make(i, infile='data/baseline-rune-no-e.txt',
+#                 outfile=f'data/p-no-e-{i}gram.txt')

data/p-no-e-1gram.txt

@@ -0,0 +1,29 @@
+ᚪ 23112
+ᚩ 21225
+ᛋ 15220
+ᚾ 14570
+ᛏ 14452
+ᛁ 14440
+ᛚ 11276
+ᚱ 10100
+ᚢ 9400
+ᛞ 8939
+ᚳ 8253
+ᚣ 6936
+ᚻ 6830
+ᚹ 6035
+ᛒ 4750
+ᚠ 4581
+ᛗ 4397
+ᚷ 4185
+ᛈ 4038
+ᛝ 3665
+ᚦ 3590
+ᛡ 1185
+ᛄ 501
+ᛉ 170
+ᛖ 6
+ᛇ 0
+ᛟ 0
+ᚫ 0
+ᛠ 0

data/p-no-e-2gram.txt

@@ -0,0 +1,553 @@
+ᚪᚾ 5122
+ᚩᚢ 3947
+ᚾᛞ 3077
+ᛁᚾ 2987
+ᛏᚩ 2893
+ᚩᚱ 2780
+ᚪᛏ 2739
+ᚪᛋ 2495
+ᛋᛏ 2291
+ᛚᛚ 2239
+ᛋᚪ 2228
+ᚪᛚ 2203
+ᛁᛋ 2125
+ᛏᚪ 2100
+ᛁᛏ 1955
+ᚾᚩ 1912
+ᚩᚾ 1881
+ᚱᚩ 1877
+ᚩᚹ 1874
+ᛁᛚ 1822
+ᚪᚱ 1813
+ᚻᚪ 1722
+ᚹᚪ 1702
+ᚱᚪ 1633
+ᚾᛏ 1624
+ᛗᚪ 1598
+ᛋᚩ 1536
+ᚢᛏ 1532
+ᚾᚪ 1530
+ᚪᛞ 1515
+ᚩᚩ 1491
+ᛞᛋ 1460
+ᚩᚠ 1446
+ᚦᚪ 1442
+ᚻᛁ 1417
+ᚳᚻ 1403
+ᚣᚩ 1400
+ᛚᚪ 1384
+ᚻᚩ 1291
+ᛏᛋ 1253
+ᚠᚩ 1240
+ᛏᛁ 1228
+ᛞᚪ 1196
+ᛚᛞ 1177
+ᛚᚣ 1153
+ᛚᚩ 1150
+ᚳᚪ 1126
+ᚪᛁ 1113
+ᚷᚻ 1104
+ᚪᚳ 1104
+ᚹᛁ 1086
+ᚾᚳ 1072
+ᚢᚱ 1059
+ᛁᚷ 1052
+ᛞᚩ 1047
+ᛋᛒ 1047
+ᛋᚻ 1040
+ᚪᚣ 1040
+ᚾᛋ 1034
+ᚩᛚ 1032
+ᛋᛋ 994
+ᚳᚩ 992
+ᚩᛗ 990
+ᛋᛁ 961
+ᚷᚪ 961
+ᚢᛋ 954
+ᛁᚳ 946
+ᚢᚾ 941
+ᚣᛋ 918
+ᛚᛋ 886
+ᛈᚪ 886
+ᚱᛁ 879
+ᚾᛁ 871
+ᚩᛏ 857
+ᚳᛁ 856
+ᚹᚩ 852
+ᚢᛚ 845
+ᚹᚻ 830
+ᚣᚪ 826
+ᚻᛏ 824
+ᛞᛁ 822
+ᛁᛞ 815
+ᛒᚢ 814
+ᛡᚾ 811
+ᚪᛒ 806
+ᛒᚣ 785
+ᛒᚩ 778
+ᛁᚱ 746
+ᛚᛁ 732
+ᛋᚳ 728
+ᛏᚹ 715
+ᛈᚩ 706
+ᛒᛁ 704
+ᚩᛋ 698
+ᚩᚳ 693
+ᚦᛁ 690
+ᛏᚣ 672
+ᚢᚳ 671
+ᚾᚣ 655
+ᚳᚳ 633
+ᚢᛈ 631
+ᚷᚩ 628
+ᛝᚪ 617
+ᚠᚪ 608
+ᚢᚷ 599
+ᚩᛞ 598
+ᛒᚱ 594
+ᚠᛁ 589
+ᛏᚱ 573
+ᚱᛏ 561
+ᚩᚪ 559
+ᚪᛈ 559
+ᛏᛡ 553
+ᚱᛋ 553
+ᛏᚢ 552
+ᛋᛈ 547
+ᚪᛗ 545
+ᚱᚣ 530
+ᛁᚦ 530
+ᛗᛁ 521
+ᛗᚩ 514
+ᛋᚹ 503
+ᚹᚾ 503
+ᛋᚢ 502
+ᛒᚪ 500
+ᛁᛗ 498
+ᛏᚳ 464
+ᚪᚷ 452
+ᚾᚻ 451
+ᚢᛝ 444
+ᚱᚳ 443
+ᚱᛞ 442
+ᚠᚱ 433
+ᚷᚱ 433
+ᛋᛗ 427
+ᛏᛏ 421
+ᚳᛏ 421
+ᚱᛗ 420
+ᚷᛁ 420
+ᚪᚹ 416
+ᛋᚠ 413
+ᚣᚹ 404
+ᛝᛋ 400
+ᛞᚣ 399
+ᚱᛚ 397
+ᚳᚾ 391
+ᚦᚩ 388
+ᛚᚳ 384
+ᚩᛈ 384
+ᛗᛈ 378
+ᛏᛒ 370
+ᛏᚠ 369
+ᚢᛁ 366
+ᛋᚾ 360
+ᚾᚹ 358
+ᚩᛝ 357
+ᚳᛋ 349
+ᚪᚢ 349
+ᛏᚦ 348
+ᚳᛝ 346
+ᛞᚹ 339
+ᚣᛒ 335
+ᛞᚾ 334
+ᛈᚢ 333
+ᚢᚪ 333
+ᛄᚢ 324
+ᛈᛁ 317
+ᚳᚣ 316
+ᛈᚱ 310
+ᛏᛝ 308
+ᚣᚠ 305
+ᛞᛚ 304
+ᛈᛈ 303
+ᚾᚦ 299
+ᚱᚢ 298
+ᚣᛁ 294
+ᚩᛁ 291
+ᚩᚻ 291
+ᚩᚦ 286
+ᚱᚷ 286
+ᛝᚩ 283
+ᛞᚳ 280
+ᛒᛚ 277
+ᛏᚻ 277
+ᚣᚳ 276
+ᚩᛒ 274
+ᛁᚠ 269
+ᛋᛚ 269
+ᚪᚻ 268
+ᚩᚣ 268
+ᚳᚢ 265
+ᛚᚹ 264
+ᛞᛒ 264
+ᛋᚦ 263
+ᚣᚷ 262
+ᚳᚱ 259
+ᚳᛚ 258
+ᛞᛞ 255
+ᛚᚢ 255
+ᛞᚦ 254
+ᚾᚠ 254
+ᛋᚷ 254
+ᚦᛝ 251
+ᚱᚾ 251
+ᛚᛝ 250
+ᚾᛚ 249
+ᚾᛒ 248
+ᛏᚷ 248
+ᚣᚦ 247
+ᛞᛝ 247
+ᛈᛚ 243
+ᚻᚢ 234
+ᚠᚠ 234
+ᛞᚷ 232
+ᛏᛚ 229
+ᛞᚻ 227
+ᛗᚢ 227
+ᛞᛏ 227
+ᚢᛗ 225
+ᛏᛗ 224
+ᚱᛝ 223
+ᚱᚱ 223
+ᚠᚢ 221
+ᛝᛁ 219
+ᛗᛋ 216
+ᛞᚢ 215
+ᛏᚾ 213
+ᛁᚢ 212
+ᛋᛞ 210
+ᛞᚠ 207
+ᚣᛏ 207
+ᚣᚾ 207
+ᚪᚦ 206
+ᚱᚦ 205
+ᚣᚻ 204
+ᛋᚣ 202
+ᚳᚹ 202
+ᛚᛗ 202
+ᛝᚠ 201
+ᛝᛏ 200
+ᛚᛏ 199
+ᛏᛞ 193
+ᛝᛒ 193
+ᛞᚱ 189
+ᛏᛈ 188
+ᚱᛒ 188
+ᚱᚹ 187
+ᚪᚠ 183
+ᛚᚠ 182
+ᛝᚦ 181
+ᛈᛋ 179
+ᚱᛡ 177
+ᛚᛒ 177
+ᚩᚷ 173
+ᚣᛝ 172
+ᚠᛚ 167
+ᛞᛈ 166
+ᛗᚣ 165
+ᛝᚹ 164
+ᚦᚱ 162
+ᚢᛒ 162
+ᚣᛞ 162
+ᚻᚣ 162
+ᚾᚾ 160
+ᚢᛞ 159
+ᚾᚢ 157
+ᛈᚣ 157
+ᛋᛝ 157
+ᚢᚦ 156
+ᛝᚳ 156
+ᚣᛗ 155
+ᚠᛏ 155
+ᛞᛗ 153
+ᛁᛈ 152
+ᚹᛋ 150
+ᚣᛈ 148
+ᚷᛚ 146
+ᛗᛝ 146
+ᚻᛝ 144
+ᚻᚹ 140
+ᚻᛋ 139
+ᚹᛝ 138
+ᚾᛝ 134
+ᛝᚢ 134
+ᛄᚩ 134
+ᚣᛚ 134
+ᛝᚻ 132
+ᚠᚣ 130
+ᛚᚦ 122
+ᚦᛋ 118
+ᚱᛈ 116
+ᛝᛚ 116
+ᚠᛒ 114
+ᛋᚱ 114
+ᛈᛝ 113
+ᚹᚦ 112
+ᛡᚢ 110
+ᚾᛗ 108
+ᚠᚻ 108
+ᛗᛗ 108
+ᚠᛋ 107
+ᚾᚷ 107
+ᛁᛒ 106
+ᛝᚷ 105
+ᛝᛗ 101
+ᚹᛚ 100
+ᚻᚻ 100
+ᛝᛈ 100
+ᚳᛡ 97
+ᚻᚾ 97
+ᛚᚷ 97
+ᚣᚣ 96
+ᛗᛒ 96
+ᛋᛡ 96
+ᚱᚠ 95
+ᛝᛞ 95
+ᛚᚱ 94
+ᛈᚳ 94
+ᚠᚦ 93
+ᚻᚦ 92
+ᚦᛚ 92
+ᛚᛈ 91
+ᚱᚻ 91
+ᛚᚻ 90
+ᛒᛒ 89
+ᚹᛞ 88
+ᚾᛡ 88
+ᚪᛝ 86
+ᚢᚹ 85
+ᛒᛋ 85
+ᚾᛈ 85
+ᛈᚻ 84
+ᛞᛄ 84
+ᛡᛚ 82
+ᚠᚷ 80
+ᚠᚳ 76
+ᚢᚩ 75
+ᚳᚠ 73
+ᛝᚱ 73
+ᛋᛄ 72
+ᚣᚱ 71
+ᛚᛡ 71
+ᚹᚹ 70
+ᚣᚢ 69
+ᚷᛋ 69
+ᚻᚠ 69
+ᛈᛏ 68
+ᛝᛝ 68
+ᛁᚹ 68
+ᚻᚳ 67
+ᚳᛒ 66
+ᚷᚷ 64
+ᛗᚾ 62
+ᛗᚦ 61
+ᚻᛒ 61
+ᛗᚹ 61
+ᚳᚦ 58
+ᛏᛄ 57
+ᛈᚹ 57
+ᚷᚢ 56
+ᚪᚪ 56
+ᛚᚾ 55
+ᚻᚱ 55
+ᛝᚾ 54
+ᛗᚠ 54
+ᚦᚢ 53
+ᚦᚠ 51
+ᚠᛗ 51
+ᚹᚢ 51
+ᚻᛗ 50
+ᚷᚾ 50
+ᚹᚠ 50
+ᚦᚣ 50
+ᛁᛉ 50
+ᚢᚠ 49
+ᚾᚱ 48
+ᚷᚳ 47
+ᛈᚠ 44
+ᚠᛈ 44
+ᛒᛝ 44
+ᚠᚹ 44
+ᚹᚣ 43
+ᚹᛒ 43
+ᛝᚣ 43
+ᚹᛗ 42
+ᚻᛈ 42
+ᛁᚪ 41
+ᚹᚱ 41
+ᚹᛏ 41
+ᚣᛄ 41
+ᚦᚦ 40
+ᚠᚾ 39
+ᚹᚳ 39
+ᛡᛏ 38
+ᚦᚹ 37
+ᚻᚷ 36
+ᚢᚻ 36
+ᛈᛒ 36
+ᛗᚳ 36
+ᚢᚣ 36
+ᚾᛄ 35
+ᛄᚪ 35
+ᚻᛚ 34
+ᚳᛈ 33
+ᛡᛋ 33
+ᚻᛞ 33
+ᚳᚷ 33
+ᚳᛞ 33
+ᛞᛡ 33
+ᛁᛁ 31
+ᛗᚷ 31
+ᛗᚻ 31
+ᛒᛏ 30
+ᚠᛞ 30
+ᚳᛗ 30
+ᚦᚳ 30
+ᚷᛒ 30
+ᚦᚾ 30
+ᚦᚻ 30
+ᛉᛉ 30
+ᚦᛒ 29
+ᛗᛏ 28
+ᚷᛈ 28
+ᛗᛚ 28
+ᚩᛄ 28
+ᚩᛉ 27
+ᚷᛝ 27
+ᚷᚠ 27
+ᛝᛄ 26
+ᛉᛁ 26
+ᚹᚷ 25
+ᚷᚣ 24
+ᚱᛄ 22
+ᚦᛗ 21
+ᚦᚷ 21
+ᚦᛈ 21
+ᛡᚪ 21
+ᛈᛡ 20
+ᚪᛄ 20
+ᛈᚦ 20
+ᚢᛡ 20
+ᛈᚷ 20
+ᚷᚹ 19
+ᛚᛄ 19
+ᛗᛞ 18
+ᚦᛏ 18
+ᛉᚢ 17
+ᛡᚱ 16
+ᛈᛗ 16
+ᛉᚩ 16
+ᚹᛈ 15
+ᚻᛄ 15
+ᚷᛞ 15
+ᛈᚾ 15
+ᚠᛄ 14
+ᚪᛉ 13
+ᚷᛗ 13
+ᚾᛉ 13
+ᛒᚻ 13
+ᛁᚻ 12
+ᚹᛄ 12
+ᛡᚹ 12
+ᛡᛒ 11
+ᚷᛏ 11
+ᚢᛄ 11
+ᚳᛄ 11
+ᛈᛄ 10
+ᛗᚱ 10
+ᛒᛡ 10
+ᚪᚩ 9
+ᚦᛞ 9
+ᛡᛗ 9
+ᛁᚩ 9
+ᛉᚪ 8
+ᛉᚹ 8
+ᛁᛄ 8
+ᛉᛚ 8
+ᛒᚹ 7
+ᛉᛡ 7
+ᛡᛞ 7
+ᛉᛝ 6
+ᛒᚦ 6
+ᚢᛉ 6
+ᛗᛄ 6
+ᛡᚳ 6
+ᛈᛞ 6
+ᛉᛗ 6
+ᛡᚻ 5
+ᛡᚣ 5
+ᚷᚦ 5
+ᚷᛡ 5
+ᛉᛏ 5
+ᛁᚣ 5
+ᚢᚢ 5
+ᚻᛡ 5
+ᛡᛁ 5
+ᛋᛉ 5
+ᛞᛉ 5
+ᚠᛝ 4
+ᛉᚦ 4
+ᚦᛄ 4
+ᛉᚠ 4
+ᛉᛒ 4
+ᛝᛉ 4
+ᛡᚩ 4
+ᛏᛉ 4
+ᛒᛗ 3
+ᛉᛋ 3
+ᚦᛖ 3
+ᛉᚳ 3
+ᛉᚱ 3
+ᛉᚻ 3
+ᛡᚷ 3
+ᛡᛈ 3
+ᚱᛉ 3
+ᛉᚷ 3
+ᚷᛄ 3
+ᛚᛉ 3
+ᛒᚾ 3
+ᛒᚳ 3
+ᛒᛄ 2
+ᚣᛉ 2
+ᛄᛋ 2
+ᛗᛡ 2
+ᛉᚾ 2
+ᛉᛞ 2
+ᚹᛉ 2
+ᛡᚦ 2
+ᛉᛈ 2
+ᛄᛚ 1
+ᛁᛖ 1
+ᛖᚾ 1
+ᛖᛒ 1
+ᛄᚾ 1
+ᚾᛖ 1
+ᛖᛚ 1
+ᛄᚹ 1
+ᚣᛡ 1
+ᛖᚳ 1
+ᛄᛁ 1
+ᛡᚠ 1
+ᛖᛈ 1
+ᛄᚷ 1
+ᛒᚠ 1
+ᛄᚳ 1
+ᛒᚷ 1
+ᚳᛉ 1
+ᛈᛉ 1
+ᚻᛉ 1
+ᛡᛄ 1
+ᛒᛈ 1
+ᚳᛖ 1
+ᛖᚱ 1

probability.py

@@ -7,7 +7,10 @@ from InterruptDB import InterruptDB
 
 RUNES = 'ᚠᚢᚦᚩᚱᚳᚷᚹᚻᚾᛁᛄᛇᛈᛉᛋᛏᛒᛖᛗᛚᛝᛟᛞᚪᚫᚣᛡᛠ'
 INVERT = False
-MIN_SCORE = 1.4
+IOC_MIN_SCORE = 1.3
+KEY_MAX_SCORE = 0.05
+AFF_MAX_SCORE = 0.04
+IRP_F_ONLY = False
 session_files = []
 
 
@@ -25,7 +28,12 @@ def break_cipher(fname, candidates, solver, key_fn):
     slvr.output.QUIET = True
     slvr.output.COLORS = False
     slvr.KEY_INVERT = INVERT
+    key_max_score = KEY_MAX_SCORE
+    if key_fn.__name__ == 'GuessAffine':
+        key_max_score = AFF_MAX_SCORE
     for irp_count, score, irp, kl, skips in candidates:
+        if IRP_F_ONLY and irp != 0:
+            continue
         data = load_indices(filename, irp, maxinterrupt=irp_count)
         if INVERT:
             data = [28 - x for x in data]
@@ -34,10 +42,12 @@ def break_cipher(fname, candidates, solver, key_fn):
             score, RUNES[irp], len(iguess.stops), key_fn.__name__))
         testcase = iguess.join(iguess.from_occurrence_index(skips))
 
-        key = key_fn(testcase).guess(kl, fn_similarity)
+        key_score, key = key_fn(testcase).guess(kl, fn_similarity)
+        if key_score > key_max_score:
+            continue
+        print(f'  key_score: {key_score:.4f}, {key}')
         print('  skip:', skips)
-        print('  key:', key)
-        txtname = f'{key_fn.__name__}.{score:.3f}_{fname}_{kl}.{irp}'
+        txtname = f'{key_fn.__name__}.{key_score:.4f}_{fname}_{kl}.{irp}'
         if INVERT:
             txtname += '.inv'
         while txtname in session_files:
@@ -45,7 +55,7 @@ def break_cipher(fname, candidates, solver, key_fn):
         session_files.append(txtname)
         outfile = f'out/{txtname}.txt'
         with open(outfile, 'w') as f:
-            f.write(f'{kl}, {score:.4f}, {key}, {skips}\n')
+            f.write(f'{irp}, {kl}, {score:.4f}, {key}, {skips}\n')
         slvr.output.file_output = outfile
         slvr.INTERRUPT = RUNES[irp]
         slvr.INTERRUPT_POS = skips
@@ -56,8 +66,7 @@ def break_cipher(fname, candidates, solver, key_fn):
 #########################################
 #  main
 #########################################
-# db = InterruptDB.load('InterruptDB/db_secondary.txt')
-db = InterruptDB.load()
+db = InterruptDB.load()  # 'db_secondary'
 
 for fname in [
     'p0-2',  # ???
@@ -85,7 +94,7 @@ for fname in [
         continue
     print()
     print(f'loading file: pages/{fname}.txt')
-    candidates = [x for x in db[fname] if x[1] >= MIN_SCORE]
+    candidates = [x for x in db[fname] if x[1] >= IOC_MIN_SCORE]
     if not candidates:
         maxscore = max(x[1] for x in db[fname])
         print('No candidates. Highest score is only', maxscore)