Refactoring I.

- Revamp whole DB to Display flow
- Filter Pipeline, arbitrary filtering and sorting
- Binary tree arrays for faster lookup & manipulation
- DB: introducing custom functions
- DB scheme: split req into heap & cache
- cache written by GlassVPN only
- heap written by Main App only
- Introducing DB separation: DBCore, DBCommon, DBAppOnly
- Introducing DB data sources: TestDataSource, GroupedDomainDataSource, RecordingsDB, DomainFilter
- Background sync: Move entries from cache to heap and notify all observers
- GlassVPN: Binary tree filter lookup
- GlassVPN: Reusing prepared statement

main/Data Source/DomainFilter.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+enum DomainFilter {
+	static private var data: [String: FilterOptions] = {
+		AppDB?.loadFilters() ?? [:]
+	}()
+	
+	/// Get filter with given `domain` name
+	@inline(__always) static subscript(_ domain: String) -> FilterOptions? {
+		data[domain]
+	}
+	
+	/// Update local memory object by loading values from persistent db.
+	/// - Note: Will trigger `NotifyDNSFilterChanged` notification.
+	static func reload() {
+		data = AppDB?.loadFilters() ?? [:]
+		NotifyDNSFilterChanged.post()
+	}
+	
+	/// Get list of domains (sorted by name) which do contain the given filter
+	static func list(where matching: FilterOptions) -> [String] {
+		data.compactMap { $1.contains(matching) ? $0 : nil }.sorted()
+	}
+	
+	/// Get total number of blocked and ignored domains. Shown in settings overview.
+	static func counts() -> (blocked: Int, ignored: Int) {
+		data.reduce(into: (0, 0)) {
+			if $1.1.contains(.blocked) { $0.0 += 1 }
+			if $1.1.contains(.ignored) { $0.1 += 1 } }
+	}
+	
+	/// Union `filter` with set.
+	/// - Note: Will trigger `NotifyDNSFilterChanged` notification.
+	static func update(_ domain: String, add filter: FilterOptions) {
+		update(domain, set: (data[domain] ?? FilterOptions()).union(filter))
+	}
+
+	/// Subtract `filter` from set.
+	/// - Note: Will trigger `NotifyDNSFilterChanged` notification.
+	static func update(_ domain: String, remove filter: FilterOptions) {
+		update(domain, set: data[domain]?.subtracting(filter))
+	}
+
+	/// Update persistent db, local memory object, and post notification to subscribers
+	/// - Parameter set: Remove a filter with `nil` or `.none`
+	static private func update(_ domain: String, set: FilterOptions?) {
+		AppDB?.setFilter(domain, set)
+		data[domain] = (set == FilterOptions.none) ? nil : set
+		NotifyDNSFilterChanged.post(domain)
+	}
+}

main/Data Source/GroupedDomainDataSource.swift

@@ -0,0 +1,250 @@
+import UIKit
+
+// ##########################
+// #
+// #    MARK: DataSource
+// #
+// ##########################
+
+class GroupedDomainDataSource {
+	
+	private var tsLatest: Timestamp = 0
+	
+	private let parent: String?
+	let pipeline: FilterPipeline<GroupedDomain>
+	
+	init(withDelegate tvc: FilterPipelineDelegate, parent p: String?) {
+		parent = p
+		pipeline = .init(withDelegate: tvc)
+		pipeline.setDataSource { [unowned self] in self.dataSourceCallback() }
+		pipeline.setSorting {
+			$0.lastModified > $1.lastModified
+		}
+		if #available(iOS 10.0, *) {
+			tvc.tableView.refreshControl = UIRefreshControl(call: #selector(reloadFromSource), on: self)
+		}
+		NotifyLogHistoryReset.observe(call: #selector(reloadFromSource), on: self)
+		NotifyDNSFilterChanged.observe(call: #selector(didChangeDomainFilter), on: self)
+		NotifySyncInsert.observe(call: #selector(syncInsert), on: self)
+		NotifySyncRemove.observe(call: #selector(syncRemove), on: self)
+	}
+	
+	/// Callback fired only when pipeline resets data source
+	private func dataSourceCallback() -> [GroupedDomain] {
+		guard let db = AppDB else { return [] }
+		let earliest = sync.tsEarliest
+		tsLatest = earliest
+		var log = db.dnsLogsGrouped(since: earliest, parentDomain: parent) ?? []
+		for (i, val) in log.enumerated() {
+			log[i].options = DomainFilter[val.domain]
+			tsLatest = max(tsLatest, val.lastModified)
+		}
+		return log
+	}
+	
+	/// Pause recurring background updates to force reload `dataSource`.
+	/// Callback fired on user action `pull-to-refresh`, or another background task triggered `NotifyLogHistoryReset`.
+	/// - Parameter sender: May be either `UIRefreshControl` or `Notification`
+	///                     (optional: pass single domain as the notification object).
+	@objc func reloadFromSource(sender: Any? = nil) {
+		weak var refreshControl = sender as? UIRefreshControl
+		let notification = sender as? Notification
+		sync.pause()
+		if let affectedDomain = notification?.object as? String {
+			partiallyReloadFromSource(affectedDomain)
+			sync.start()
+		} else {
+			pipeline.reload(fromSource: true, whenDone: {
+				sync.start()
+				refreshControl?.endRefreshing()
+			})
+		}
+	}
+	
+	/// Callback fired when user editslist of `blocked` or `ignored` domains in settings. (`NotifyDNSFilterChanged` notification)
+	@objc private func didChangeDomainFilter(_ notification: Notification) {
+		guard let domain = notification.object as? String else {
+			reloadFromSource()
+			return
+		}
+		if let (i, obj) = pipeline.dataSourceGet(where: { $0.domain == domain }) {
+			var y = obj
+			y.options = DomainFilter[domain]
+			pipeline.update(y, at: i)
+		}
+	}
+	
+	
+	// MARK: Table View Data Source
+	
+	@inline(__always) var numberOfRows: Int { get { pipeline.displayObjectCount() } }
+	
+	@inline(__always) subscript(_ row: Int) -> GroupedDomain { pipeline.displayObject(at: row) }
+	
+	
+	// MARK: partial updates
+	
+	/// Callback fired when background sync added new entries to the list. (`NotifySyncInsert` notification)
+	@objc private func syncInsert(_ notification: Notification) {
+		let range = notification.object as! SQLiteRowRange
+		guard let latest = AppDB?.dnsLogsGrouped(range: range, parentDomain: parent) else {
+			assertionFailure("NotifySyncInsert fired with empty range")
+			return
+		}
+		for x in latest {
+			if let (i, obj) = pipeline.dataSourceGet(where: { $0.domain == x.domain }) {
+				pipeline.update(obj + x, at: i)
+			} else {
+				var y = x
+				y.options = DomainFilter[x.domain]
+				pipeline.addNew(y)
+			}
+			tsLatest = max(tsLatest, x.lastModified)
+		}
+	}
+	
+	/// Callback fired when background sync removed old entries from the list. (`NotifySyncRemove` notification)
+	@objc private func syncRemove(_ notification: Notification) {
+		let range = notification.object as! SQLiteRowRange
+		guard let outdated = AppDB?.dnsLogsGrouped(range: range, parentDomain: parent),
+			outdated.count > 0 else {
+				assertionFailure("NotifySyncRemove fired with empty range")
+				return
+		}
+		var listOfDeletes: [Int] = []
+		for x in outdated {
+			guard let (i, obj) = pipeline.dataSourceGet(where: { $0.domain == x.domain }) else {
+				assertionFailure("Try to remove non-existent element")
+				continue // should never happen
+			}
+			if obj.total > x.total {
+				pipeline.update(obj - x, at: i)
+			} else {
+				listOfDeletes.append(i)
+			}
+		}
+		pipeline.remove(indices: listOfDeletes.sorted())
+	}
+}
+
+
+// ################################
+// #
+// #    MARK: - Delete History
+// #
+// ################################
+
+extension GroupedDomainDataSource {
+	
+	/// Callback fired when user performs row edit -> delete action
+	func deleteHistory(domain: String, since ts: Timestamp) {
+		let flag = (parent != nil)
+		DispatchQueue.global().async {
+			guard let db = AppDB, db.dnsLogsDelete(domain, strict: flag, since: ts) > 0 else {
+				return // nothing has changed
+			}
+			db.vacuum()
+			NotifyLogHistoryReset.postAsyncMain(domain) // calls deleteReloadFromSource(:)
+		}
+	}
+	
+	/// Reload a single data source entry. Callback fired by `reloadFromSource()`
+	private func partiallyReloadFromSource(_ affectedFQDN: String) {
+		let affectedParent = affectedFQDN.extractDomain()
+		guard parent == nil || parent == affectedParent else {
+			return // does not affect current table
+		}
+		let affected = (parent == nil ? affectedParent : affectedFQDN)
+		guard let old = pipeline.dataSourceGet(where: { $0.domain == affected }) else {
+			// can only happen if delete sheet is open while background sync removed the element
+			return
+		}
+		var removeOld = true
+		if let new = AppDB?.dnsLogsGrouped(since: sync.tsEarliest, upto: tsLatest, matchingDomain: affected, parentDomain: parent) {
+			assert(new.count < 2)
+			for var x in new {
+				x.options = DomainFilter[x.domain]
+				if old.object.domain == x.domain {
+					pipeline.update(x, at: old.index)
+					removeOld = false
+				} else {
+					pipeline.addNew(x)
+				}
+			}
+		}
+		if removeOld { pipeline.remove(indices: [old.index]) }
+	}
+}
+
+
+// ##########################
+// #
+// #    MARK: - Edit Row
+// #
+// ##########################
+
+protocol GroupedDomainEditRow : EditableRows, FilterPipelineDelegate {
+	var source: GroupedDomainDataSource { get set }
+}
+
+extension GroupedDomainEditRow  {
+	
+	func editableRowActions(_ index: IndexPath) -> [(RowAction, String)] {
+		let x = source[index.row]
+		if x.domain.starts(with: "#") {
+			return [(.delete, "Delete")]
+		}
+		let b = x.options?.contains(.blocked) ?? false
+		let i = x.options?.contains(.ignored) ?? false
+		return [(.delete, "Delete"), (.block, b ? "Unblock" : "Block"), (.ignore, i ? "Unignore" : "Ignore")]
+	}
+	
+	func editableRowActionColor(_: IndexPath, _ action: RowAction) -> UIColor? {
+		action == .block ? .systemOrange : nil
+	}
+	
+	func editableRowUserInfo(_ index: IndexPath) -> Any? { source[index.row] }
+	
+	func editableRowCallback(_ index: IndexPath, _ action: RowAction, _ userInfo: Any?) -> Bool {
+		let entry = userInfo as! GroupedDomain
+		switch action {
+		case .ignore: showFilterSheet(entry, .ignored)
+		case .block:  showFilterSheet(entry, .blocked)
+		case .delete:
+			AlertDeleteLogs(entry.domain, latest: entry.lastModified) {
+				self.source.deleteHistory(domain: entry.domain, since: $0)
+			}.presentIn(self)
+		}
+		return true
+	}
+	
+	private func showFilterSheet(_ entry: GroupedDomain, _ filter: FilterOptions) {
+		if entry.options?.contains(filter) ?? false {
+			DomainFilter.update(entry.domain, remove: filter)
+		} else {
+			// TODO: alert sheet
+			DomainFilter.update(entry.domain, add: filter)
+		}
+	}
+}
+
+// MARK: Extensions
+extension TVCDomains : GroupedDomainEditRow {
+	override func tableView(_ tableView: UITableView, editActionsForRowAt indexPath: IndexPath) -> [UITableViewRowAction]? {
+		getRowActionsIOS9(indexPath)
+	}
+	@available(iOS 11.0, *)
+	override func tableView(_ tableView: UITableView, trailingSwipeActionsConfigurationForRowAt indexPath: IndexPath) -> UISwipeActionsConfiguration? {
+		getRowActionsIOS11(indexPath)
+	}
+}
+
+extension TVCHosts : GroupedDomainEditRow {
+	override func tableView(_ tableView: UITableView, editActionsForRowAt indexPath: IndexPath) -> [UITableViewRowAction]? {
+		getRowActionsIOS9(indexPath)
+	}
+	@available(iOS 11.0, *)
+	override func tableView(_ tableView: UITableView, trailingSwipeActionsConfigurationForRowAt indexPath: IndexPath) -> UISwipeActionsConfiguration? {
+		getRowActionsIOS11(indexPath)
+	}
+}

main/Data Source/RecordingsDB.swift

@@ -0,0 +1,43 @@
+import Foundation
+
+enum RecordingsDB {
+	/// Get last started recording (where `start` is set, but `stop` is not)
+	static func getCurrent() -> Recording? { AppDB?.recordingGetOngoing() }
+	
+	/// Create new recording and set `start` timestamp to `now()`
+	static func startNew() -> Recording? { try? AppDB?.recordingStartNew() }
+	
+	/// Finalize recording by setting the `stop` timestamp to `now()`
+	static func stop(_ r: inout Recording) { AppDB?.recordingStop(&r) }
+	
+	/// Get list of all recordings
+	static func list() -> [Recording] { AppDB?.recordingGetAll() ?? [] }
+	
+	/// Copy log entries from generic `heap` table  to recording specific `recLog` table
+	static func persist(_ r: Recording) { AppDB?.recordingLogsPersist(r) }
+	
+	/// Get list of domains that occured during the recording
+	static func details(_ r: Recording) -> [RecordLog] {
+		AppDB?.recordingLogsGetGrouped(r) ?? []
+	}
+	
+	/// Update `title`, `appid`, and `notes` and post `NotifyRecordingChanged` notification.
+	static func update(_ r: Recording) {
+		AppDB?.recordingUpdate(r)
+		NotifyRecordingChanged.post((r, false))
+	}
+	
+	/// Delete whole recording including all entries and post `NotifyRecordingChanged` notification.
+	static func delete(_ r: Recording) {
+		if (try? AppDB?.recordingDelete(r)) == true {
+			NotifyRecordingChanged.post((r, true))
+		}
+	}
+	
+	/// Delete individual entries from recording while keeping the recording alive.
+	/// - Returns: `true` if at least one row is deleted.
+	static func deleteDetails(_ r: Recording, domain: String) -> Bool {
+		((try? AppDB?.recordingLogsDelete(r.id, matchingDomain: domain)) ?? 0) > 0
+	}
+}
+

main/Data Source/SyncUpdate.swift

@@ -0,0 +1,44 @@
+import Foundation
+
+class SyncUpdate {
+	private var timer: Timer!
+	private var paused: Int = 1 // first start() will decrement
+	private(set) var tsEarliest: Timestamp
+	
+	init(periodic interval: TimeInterval) {
+		tsEarliest = Pref.DateFilter.lastXMinTimestamp() ?? 0
+		NotifyDateFilterChanged.observe(call: #selector(didChangeDateFilter), on: self)
+		timer = Timer.repeating(interval, call: #selector(periodicUpdate), on: self)
+	}
+	
+	@objc private func didChangeDateFilter() {
+		let lastXFilter = Pref.DateFilter.lastXMinTimestamp() ?? 0
+		if tsEarliest < lastXFilter {
+			if let excess = AppDB?.dnsLogsRowRange(between: tsEarliest, and: lastXFilter) {
+				NotifySyncRemove.post(excess)
+			}
+		} else if tsEarliest > lastXFilter {
+			if let missing = AppDB?.dnsLogsRowRange(between: lastXFilter, and: tsEarliest) {
+				NotifySyncInsert.post(missing)
+			}
+		}
+		tsEarliest = lastXFilter
+	}
+	
+	func pause() { paused += 1 }
+	func start() { if paused > 0 { paused -= 1 } }
+	
+	@objc private func periodicUpdate() {
+		guard paused == 0, let db = AppDB else { return }
+		if let inserted = db.dnsLogsPersist() { // move cache -> heap
+			NotifySyncInsert.post(inserted)
+		}
+		if let lastXFilter = Pref.DateFilter.lastXMinTimestamp(), tsEarliest < lastXFilter {
+			if let removed = db.dnsLogsRowRange(between: tsEarliest, and: lastXFilter) {
+				NotifySyncRemove.post(removed)
+			}
+			tsEarliest = lastXFilter
+		}
+		// TODO: periodic hard delete old logs (will reset rowids!)
+	}
+}

main/Data Source/TestDataSource.swift

@@ -0,0 +1,41 @@
+import Foundation
+
+#if IOS_SIMULATOR
+
+private let db = AppDB!
+private var pStmt: OpaquePointer?
+
+class TestDataSource {
+	
+	static func load() {
+		QLog.Debug("SQLite path: \(URL.internalDB())")
+		
+		let deleted = db.dnsLogsDelete("test.com", strict: false)
+		QLog.Debug("Deleting \(deleted) rows matching 'test.com'")
+		
+		QLog.Debug("Writing 33 test logs")
+		pStmt = try! db.logWritePrepare()
+		try? db.logWrite(pStmt, "keeptest.com", blocked: false)
+		for _ in 1...4 { try? db.logWrite(pStmt, "test.com", blocked: false) }
+		for _ in 1...7 { try? db.logWrite(pStmt, "i.test.com", blocked: false) }
+		for i in 1...8 { try? db.logWrite(pStmt, "b.test.com", blocked: i>5) }
+		for i in 1...13 { try? db.logWrite(pStmt, "bi.test.com", blocked: i%2==0) }
+		
+		db.dnsLogsPersist()
+		
+		QLog.Debug("Creating 4 filters")
+		db.setFilter("b.test.com", .blocked)
+		db.setFilter("i.test.com", .ignored)
+		db.setFilter("bi.test.com", [.blocked, .ignored])
+		
+		QLog.Debug("Done")
+		
+		Timer.repeating(2, call: #selector(insertRandom), on: self)
+	}
+	
+	@objc static func insertRandom() {
+		//QLog.Debug("Inserting 1 periodic log entry")
+		try? db.logWrite(pStmt, "\(arc4random() % 5).count.test.com", blocked: true)
+	}
+}
+#endif