Refactoring I.

- Revamp whole DB to Display flow
- Filter Pipeline, arbitrary filtering and sorting
- Binary tree arrays for faster lookup & manipulation
- DB: introducing custom functions
- DB scheme: split req into heap & cache
- cache written by GlassVPN only
- heap written by Main App only
- Introducing DB separation: DBCore, DBCommon, DBAppOnly
- Introducing DB data sources: TestDataSource, GroupedDomainDataSource, RecordingsDB, DomainFilter
- Background sync: Move entries from cache to heap and notify all observers
- GlassVPN: Binary tree filter lookup
- GlassVPN: Reusing prepared statement
This commit is contained in:
relikd
2020-06-02 21:45:08 +02:00
parent 10b43a0f67
commit b17fb3c354
36 changed files with 2214 additions and 1482 deletions

View File

@@ -1,8 +1,8 @@
import UIKit
extension UIAlertController {
func presentIn(_ viewController: UIViewController?) {
viewController?.present(self, animated: true)
func presentIn(_ viewController: UIViewController) {
viewController.present(self, animated: true)
}
}

View File

@@ -0,0 +1,77 @@
import Foundation
//extension Collection {
// subscript(ifExist i: Index?) -> Iterator.Element? {
// guard let i = i else { return nil }
// return indices.contains(i) ? self[i] : nil
// }
//}
extension Range where Bound == Int {
@inline(__always) func arr() -> [Bound] { self.map { $0 } }
}
// MARK: - Sorted Array
extension Array {
typealias CompareFn = (Element, Element) -> Bool
/// Binary tree search operation.
/// - Warning: Array must be sorted already.
/// - Parameter mustExist: Determine whether to return low index or `nil` if element is missing.
/// - Returns: Index or `nil` (only if `mustExist = true` and element does not exist).
/// - Complexity: O(log *n*), where *n* is the length of the array.
func binTreeIndex(of element: Element, compare fn: CompareFn, mustExist: Bool = false) -> Int? {
var lo = 0, hi = self.count - 1
while lo <= hi {
let mid = (lo + hi)/2
if fn(self[mid], element) {
lo = mid + 1
} else if fn(element, self[mid]) {
hi = mid - 1
} else {
return mid
}
}
return mustExist ? nil : lo // not found, would be inserted at position lo
}
/// Binary tree insert operation
/// - Warning: Array must be sorted already.
/// - Returns: Index at which `elem` was inserted
/// - Complexity: O(log *n*), where *n* is the length of the array.
@discardableResult mutating func binTreeInsert(_ elem: Element, compare fn: CompareFn) -> Int {
let newIndex = binTreeIndex(of: elem, compare: fn)!
insert(elem, at: newIndex)
return newIndex
}
/// Binary tree remove operation
/// - Warning: Array must be sorted already.
/// - Returns: Index of removed `elem` or `nil` if it does not exist
/// - Complexity: O(log *n*), where *n* is the length of the array.
@discardableResult mutating func binTreeRemove(_ elem: Element, compare fn: CompareFn) -> Int? {
if let i = binTreeIndex(of: elem, compare: fn, mustExist: true) {
remove(at: i)
return i
}
return nil
}
/// Sorted synchronous comparison between elements
/// - Parameter sortedSubset: Must be a strict subset of the sorted array.
/// - Returns: List of elements that are **not** present in `sortedSubset`.
/// - Complexity: O(*m*+*n*), where *n* is the length of the array and *m* the length of the `sortedSubset`.
/// If indices are found earlier, *n* may be significantly less (on average: `n/2`)
func difference(toSubset sortedSubset: [Element], compare fn: CompareFn) -> [Element] {
var result: [Element] = []
var iter = makeIterator()
for rhs in sortedSubset {
while let lhs = iter.next(), fn(lhs, rhs) {
result.append(lhs)
}
}
return result
}
}

View File

@@ -1,40 +0,0 @@
import Foundation
extension GroupedDomain {
/// Return new `GroupedDomain` by adding `total` and `blocked` counts. Set `lastModified` to the maximum of the two.
static func +(a: GroupedDomain, b: GroupedDomain) -> Self {
GroupedDomain(domain: a.domain, total: a.total + b.total, blocked: a.blocked + b.blocked,
lastModified: max(a.lastModified, b.lastModified), options: a.options ?? b.options )
}
/// Return new `GroupedDomain` by subtracting `total` and `blocked` counts.
static func -(a: GroupedDomain, b: GroupedDomain) -> Self {
GroupedDomain(domain: a.domain, total: a.total - b.total, blocked: a.blocked - b.blocked,
lastModified: a.lastModified, options: a.options )
}
}
extension Array where Element == GroupedDomain {
func merge(_ domain: String, options opt: FilterOptions? = nil) -> GroupedDomain {
var b: Int32 = 0, t: Int32 = 0, m: Timestamp = 0
for x in self {
b += x.blocked
t += x.total
m = Swift.max(m, x.lastModified)
}
return GroupedDomain(domain: domain, total: t, blocked: b, lastModified: m, options: opt)
}
}
extension Recording {
var fallbackTitle: String { get { "Unnamed Recording #\(id)" } }
var duration: Timestamp? { get { stop == nil ? nil : stop! - start } }
var durationString: String? { get { stop == nil ? nil : TimeFormat.from(duration!) } }
}
extension Timestamp {
/// - Returns: Time string with format `yyyy-MM-dd HH:mm:ss`
func asDateTime() -> String { dateTimeFormat.string(from: self) }
func toDate() -> Date { Date(timeIntervalSince1970: Double(self)) }
static func now() -> Timestamp { Timestamp(Date().timeIntervalSince1970) }
static func past(minutes: Int) -> Timestamp { now() - Timestamp(minutes * 60) }
}

View File

@@ -16,102 +16,6 @@ struct QLog {
}
}
extension Collection {
subscript(ifExist i: Index?) -> Iterator.Element? {
guard let i = i else { return nil }
return indices.contains(i) ? self[i] : nil
}
}
var listOfSLDs: [String : [String : Bool]] = {
let path = Bundle.main.url(forResource: "third-level", withExtension: "txt")
let content = try! String(contentsOf: path!)
var res: [String : [String : Bool]] = [:]
content.enumerateLines { line, _ in
let dom = line.split(separator: ".")
let tld = String(dom.first!)
let sld = String(dom.last!)
if res[tld] == nil { res[tld] = [:] }
res[tld]![sld] = true
}
return res
}()
extension String {
/// Check if string is equal to `domain` or ends with `.domain`
func isSubdomain(of domain: String) -> Bool { self == domain || self.hasSuffix("." + domain) }
/// Split string into top level domain part and host part
func splitDomainAndHost() -> (domain: String, host: String?) {
let lastChr = last?.asciiValue ?? 0
guard lastChr > UInt8(ascii: "9") || lastChr < UInt8(ascii: "0") else { // IP address
return (domain: "# IP connection", host: self)
}
var parts = components(separatedBy: ".")
guard let tld = parts.popLast(), let sld = parts.popLast() else {
return (domain: self, host: nil) // no subdomains, just plain SLD
}
var ending = sld + "." + tld
if listOfSLDs[tld]?[sld] ?? false, let rld = parts.popLast() {
ending = rld + "." + ending
}
return (domain: ending, host: parts.joined(separator: "."))
}
/// Returns `true` if String matches list of known second level domains (e.g., `co.uk`).
func isKnownSLD() -> Bool {
let parts = components(separatedBy: ".")
return parts.count == 2 && listOfSLDs[parts.last!]?[parts.first!] ?? false
}
}
extension Timer {
@discardableResult static func repeating(_ interval: TimeInterval, call selector: Selector, on target: Any, userInfo: Any? = nil) -> Timer {
Timer.scheduledTimer(timeInterval: interval, target: target, selector: selector,
userInfo: userInfo, repeats: true)
}
}
extension DateFormatter {
convenience init(withFormat: String) {
self.init()
dateFormat = withFormat
}
func with(format: String) -> Self {
dateFormat = format
return self
}
func string(from ts: Timestamp) -> String {
string(from: Date.init(timeIntervalSince1970: Double(ts)))
}
}
struct TimeFormat {
static func from(_ duration: Timestamp) -> String {
String(format: "%02d:%02d", duration / 60, duration % 60)
}
static func from(_ duration: TimeInterval, millis: Bool = false) -> String {
let t = Int(duration)
if millis {
let mil = Int(duration * 1000) % 1000
return String(format: "%02d:%02d.%03d", t / 60, t % 60, mil)
}
return String(format: "%02d:%02d", t / 60, t % 60)
}
static func since(_ date: Date, millis: Bool = false) -> String {
from(Date().timeIntervalSince(date), millis: millis)
}
/// Formatted duration string, e.g., `20 min` or `7 days`
/// - Parameters:
/// - minutes: Duration in minutes
/// - style: Default: `.short`
static func short(minutes: Int, style: DateComponentsFormatter.UnitsStyle = .short) -> String? {
let dcf = DateComponentsFormatter()
dcf.maximumUnitCount = 1
dcf.allowedUnits = [.day, .hour, .minute]
dcf.unitsStyle = style
return dcf.string(from: DateComponents(minute: minutes))
}
}
extension UIColor {
static var sysBg: UIColor { get { if #available(iOS 13.0, *) { return .systemBackground } else { return .white } }}
static var sysFg: UIColor { get { if #available(iOS 13.0, *) { return .label } else { return .black } }}

View File

@@ -1,9 +1,11 @@
import Foundation
let NotifyVPNStateChanged = NSNotification.Name("GlassVPNStateChanged") // VPNState!
let NotifyDNSFilterChanged = NSNotification.Name("PSIDNSFilterSettingsChanged") // nil!
let NotifyDNSFilterChanged = NSNotification.Name("PSIDNSFilterSettingsChanged") // domain: String?
let NotifyDateFilterChanged = NSNotification.Name("PSIDateFilterSettingsChanged") // nil!
let NotifyLogHistoryReset = NSNotification.Name("PSILogHistoryReset") // nil!
let NotifyLogHistoryReset = NSNotification.Name("PSILogHistoryReset") // domain: String?
let NotifySyncInsert = NSNotification.Name("PSISyncInsert") // SQLiteRowRange!
let NotifySyncRemove = NSNotification.Name("PSISyncRemove") // SQLiteRowRange!
let NotifyRecordingChanged = NSNotification.Name("PSIRecordingChanged") // (Recording, deleted: Bool)!

View File

@@ -1,7 +1,7 @@
import Foundation
let dateTimeFormat = DateFormatter(withFormat: "yyyy-MM-dd HH:mm:ss")
var currentVPNState: VPNState = .off
let sync = SyncUpdate(periodic: 7)
public enum VPNState : Int {
case on = 1, inbetween, off

View File

@@ -0,0 +1,51 @@
import UIKit
extension NSMutableAttributedString {
func withColor(_ color: UIColor, fromBack: Int) -> Self {
let l = length - fromBack
let r = (l < 0) ? NSMakeRange(0, length) : NSMakeRange(l, fromBack)
self.addAttribute(.foregroundColor, value: color, range: r)
return self
}
}
extension String {
/// Check if string is equal to `domain` or ends with `.domain`
func isSubdomain(of domain: String) -> Bool { self == domain || self.hasSuffix("." + domain) }
/// Extract second or third level domain name
func extractDomain() -> String {
let lastChr = last?.asciiValue ?? 0
guard lastChr > UInt8(ascii: "9") || lastChr < UInt8(ascii: "0") else { // IP address
return "# IP"
}
var parts = components(separatedBy: ".")
guard let tld = parts.popLast(), let sld = parts.popLast() else {
return self // no subdomains, just plain SLD
}
if listOfSLDs[tld]?[sld] ?? false, let rld = parts.popLast() {
return rld + "." + sld + "." + tld
}
return sld + "." + tld
}
/// Returns `true` if String matches list of known second level domains (e.g., `co.uk`).
func isKnownSLD() -> Bool {
let parts = components(separatedBy: ".")
return parts.count == 2 && listOfSLDs[parts.last!]?[parts.first!] ?? false
}
}
var listOfSLDs: [String : [String : Bool]] = {
let path = Bundle.main.url(forResource: "third-level", withExtension: "txt")
let content = try! String(contentsOf: path!)
var res: [String : [String : Bool]] = [:]
content.enumerateLines { line, _ in
let dom = line.split(separator: ".")
let tld = String(dom.first!)
let sld = String(dom.last!)
if res[tld] == nil { res[tld] = [:] }
res[tld]![sld] = true
}
return res
}()

View File

@@ -1,58 +1,27 @@
import UIKit
extension GroupedDomain {
var detailCellText: String { get {
return blocked > 0
? "\(lastModified.asDateTime())\(blocked)/\(total) blocked"
: "\(lastModified.asDateTime())\(total)"
}
}
}
extension FilterOptions {
func tableRowImage() -> UIImage? {
let blocked = contains(.blocked)
let ignored = contains(.ignored)
if blocked { return UIImage(named: ignored ? "block_ignore" : "shield-x") }
if ignored { return UIImage(named: "quicklook-not") }
return nil
}
}
extension NSMutableAttributedString {
func withColor(_ color: UIColor, fromBack: Int) -> Self {
let l = length - fromBack
let r = (l < 0) ? NSMakeRange(0, length) : NSMakeRange(l, fromBack)
self.addAttribute(.foregroundColor, value: color, range: r)
return self
}
}
// MARK: Pull-to-Refresh
extension UIRefreshControl {
convenience init(call: Selector, on: UITableViewController) {
self.init()
addTarget(on, action: call, for: .valueChanged)
addTarget(self, action: #selector(endRefreshing), for: .valueChanged)
}
}
// MARK: TableView extensions
extension IndexPath {
/// Convenience init with `section: 0`
public init(row: Int) { self.init(row: row, section: 0) }
}
extension UIRefreshControl {
convenience init(call: Selector, on target: Any) {
self.init()
addTarget(target, action: call, for: .valueChanged)
}
}
// MARK: - UITableView
extension UITableView {
/// Returns `true` if this `tableView` is the currently frontmost visible
var isFrontmost: Bool { window?.isKeyWindow ?? false }
/// If frontmost window, perform `deleteRows()`; If not, perform `reloadData()`
func safeDeleteRow(_ index: Int, with animation: UITableView.RowAnimation = .automatic) {
isFrontmost ? deleteRows(at: [IndexPath(row: index)], with: animation) : reloadData()
func safeDeleteRows(_ indices: [Int], with animation: UITableView.RowAnimation = .automatic) {
isFrontmost ? deleteRows(at: indices.map {IndexPath(row: $0)}, with: animation) : reloadData()
}
/// If frontmost window, perform `reloadRows()`; If not, perform `reloadData()`
func safeReloadRow(_ index: Int, with animation: UITableView.RowAnimation = .automatic) {
@@ -69,71 +38,44 @@ extension UITableView {
}
// MARK: - Incremental Update Delegate
// MARK: - EditableRows
enum IncrementalDataSourceUpdateOperation {
case ReloadTable, Update, Insert, Delete, Move
public enum RowAction {
case ignore, block, delete
}
protocol IncrementalDataSourceUpdate : UITableViewController {
var dataSource: [GroupedDomain] { get set }
func shouldLiveUpdateIncrementalDataSource() -> Bool
/// - Warning: Called on a background thread!
/// - Parameters:
/// - operation: Row update action
/// - row: Which row index is affected? `IndexPath(row: row)`
/// - moveTo: Only set for `Move` operation, otherwise `-1`
func didUpdateIncrementalDataSource(_ operation: IncrementalDataSourceUpdateOperation, row: Int, moveTo: Int)
protocol EditableRows {
func editableRowUserInfo(_ index: IndexPath) -> Any?
func editableRowActions(_ index: IndexPath) -> [(RowAction, String)]
func editableRowActionColor(_ index: IndexPath, _ action: RowAction) -> UIColor?
@discardableResult func editableRowCallback(_ atIndexPath: IndexPath, _ action: RowAction, _ userInfo: Any?) -> Bool
}
extension IncrementalDataSourceUpdate {
func shouldLiveUpdateIncrementalDataSource() -> Bool { true }
func didUpdateIncrementalDataSource(_: IncrementalDataSourceUpdateOperation, row: Int, moveTo: Int) {}
// TODO: custom handling if cell is being edited
func insertRow(_ obj: GroupedDomain, at index: Int) {
dataSource.insert(obj, at: index)
if shouldLiveUpdateIncrementalDataSource() {
DispatchQueue.main.sync { tableView.safeInsertRow(index, with: .left) }
}
didUpdateIncrementalDataSource(.Insert, row: index, moveTo: -1)
}
func moveRow(_ obj: GroupedDomain, from: Int, to: Int) {
dataSource.remove(at: from)
dataSource.insert(obj, at: to)
if shouldLiveUpdateIncrementalDataSource() {
DispatchQueue.main.sync {
if tableView.isFrontmost {
let source = IndexPath(row: from)
let cell = tableView.cellForRow(at: source)
cell?.detailTextLabel?.text = obj.detailCellText
tableView.moveRow(at: source, to: IndexPath(row: to))
} else {
tableView.reloadData()
}
extension EditableRows where Self: UITableViewDelegate {
func getRowActionsIOS9(_ index: IndexPath) -> [UITableViewRowAction]? {
let userInfo = editableRowUserInfo(index)
return editableRowActions(index).compactMap { a,t in
let x = UITableViewRowAction(style: a == .delete ? .destructive : .normal, title: t) { self.editableRowCallback($1, a, userInfo) }
if let color = editableRowActionColor(index, a) {
x.backgroundColor = color
}
return x
}
didUpdateIncrementalDataSource(.Move, row: from, moveTo: to)
}
func replaceRow(_ obj: GroupedDomain, at index: Int) {
dataSource[index] = obj
if shouldLiveUpdateIncrementalDataSource() {
DispatchQueue.main.sync { tableView.safeReloadRow(index) }
}
didUpdateIncrementalDataSource(.Update, row: index, moveTo: -1)
}
func deleteRow(at index: Int) {
dataSource.remove(at: index)
if shouldLiveUpdateIncrementalDataSource() {
DispatchQueue.main.sync { tableView.safeDeleteRow(index) }
}
didUpdateIncrementalDataSource(.Delete, row: index, moveTo: -1)
}
func replaceData(with newData: [GroupedDomain]) {
dataSource = newData
if shouldLiveUpdateIncrementalDataSource() {
DispatchQueue.main.sync { tableView.reloadData() }
}
didUpdateIncrementalDataSource(.ReloadTable, row: -1, moveTo: -1)
@available(iOS 11.0, *)
func getRowActionsIOS11(_ index: IndexPath) -> UISwipeActionsConfiguration? {
let userInfo = editableRowUserInfo(index)
return UISwipeActionsConfiguration(actions: editableRowActions(index).compactMap { a,t in
let x = UIContextualAction(style: a == .delete ? .destructive : .normal, title: t) { $2(self.editableRowCallback(index, a, userInfo)) }
x.backgroundColor = editableRowActionColor(index, a)
return x
})
}
func editableRowUserInfo(_ index: IndexPath) -> Any? { nil }
}
protocol EditActionsRemove : EditableRows {}
extension EditActionsRemove where Self: UITableViewController {
func editableRowActions(_: IndexPath) -> [(RowAction, String)] { [(.delete, "Remove")] }
func editableRowActionColor(_: IndexPath, _: RowAction) -> UIColor? { nil }
}

View File

@@ -0,0 +1,74 @@
import Foundation
private let dateTimeFormat = DateFormatter(withFormat: "yyyy-MM-dd HH:mm:ss")
extension DateFormatter {
convenience init(withFormat: String) {
self.init()
dateFormat = withFormat
}
}
extension Timestamp {
/// Time string with format `yyyy-MM-dd HH:mm:ss`
func asDateTime() -> String {
dateTimeFormat.string(from: Date.init(timeIntervalSince1970: Double(self)))
}
/// Convert `Timestamp` to `Date`
func toDate() -> Date {
Date(timeIntervalSince1970: Double(self))
}
/// Current time as `Timestamp` (second accuracy)
static func now() -> Timestamp {
Timestamp(Date().timeIntervalSince1970)
}
/// Create `Timestamp` with `now() - minutes * 60`
static func past(minutes: Int) -> Timestamp {
now() - Timestamp(minutes * 60)
}
}
extension Timer {
/// Recurring timer maintains a strong reference to `target`.
@discardableResult static func repeating(_ interval: TimeInterval, call selector: Selector, on target: Any, userInfo: Any? = nil) -> Timer {
Timer.scheduledTimer(timeInterval: interval, target: target, selector: selector,
userInfo: userInfo, repeats: true)
}
}
struct TimeFormat {
/// Time string with format `HH:mm`
static func from(_ duration: Timestamp) -> String {
String(format: "%02d:%02d", duration / 60, duration % 60)
}
/// Duration string with format `HH:mm` or `HH:mm.sss`
static func from(_ duration: TimeInterval, millis: Bool = false) -> String {
let t = Int(duration)
if millis {
let mil = Int(duration * 1000) % 1000
return String(format: "%02d:%02d.%03d", t / 60, t % 60, mil)
}
return String(format: "%02d:%02d", t / 60, t % 60)
}
/// Duration string with format `HH:mm` or `HH:mm.sss` since reference date
static func since(_ date: Date, millis: Bool = false) -> String {
from(Date().timeIntervalSince(date), millis: millis)
}
/// Formatted duration string, e.g., `20 min` or `7 days`
/// - Parameters:
/// - minutes: Duration in minutes
/// - style: Default: `.short`
static func short(minutes: Int, style: DateComponentsFormatter.UnitsStyle = .short) -> String? {
let dcf = DateComponentsFormatter()
dcf.maximumUnitCount = 1
dcf.allowedUnits = [.day, .hour, .minute]
dcf.unitsStyle = style
return dcf.string(from: DateComponents(minute: minutes))
}
}

View File

@@ -1,9 +1,9 @@
import Foundation
fileprivate extension FileManager {
func exportDir() -> URL {
try! url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
}
// func exportDir() -> URL {
// try! url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: true)
// }
func appGroupDir() -> URL {
containerURL(forSecurityApplicationGroupIdentifier: "group.de.uni-bamberg.psi.AppCheck")!
}
@@ -13,7 +13,7 @@ fileprivate extension FileManager {
}
extension URL {
static func exportDir() -> URL { FileManager.default.exportDir() }
// static func exportDir() -> URL { FileManager.default.exportDir() }
static func appGroupDir() -> URL { FileManager.default.appGroupDir() }
static func internalDB() -> URL { FileManager.default.internalDB() }
}