Persist cache with pull-to-refresh + Sync rate limiting

main/Common Classes/FilterPipeline.swift

@@ -24,7 +24,7 @@ class FilterPipeline<T> {
 	}
 	
 	/// Set a new `dataSource` query and immediately apply all filters and sorting.
-	/// - Note: You must call `reload(fromSource:)` manually!
+	/// - Note: You must call `reload(fromSource:whenDone:)` manually!
 	/// - Note: Always use `[unowned self]`
 	func setDataSource(query: @escaping DataSourceQuery) {
 		sourceQuery = query
@@ -49,14 +49,8 @@ class FilterPipeline<T> {
 		return (i, dataSource[i])
 	}
 	
-	/// Search and return list of `dataSource` elements that match the given `predicate`.
-	/// - Returns: Sorted list of indices and objects  in `dataSource`.
-	/// - Complexity: O(*m* + *n*), where *n* is the length of the `dataSource` and *m* is the number of matches.
-//	func dataSourceAll(where predicate: ((T) -> Bool)) -> [(index: Int, object: T)] {
-//		dataSource.enumerated().compactMap { predicate($1) ? ($0, $1) : nil }
-//	}
-	
 	/// Re-query data source and re-built filter and display sorting order.
+	/// - Note: Will call `reloadData()` before `whenDone` closure is executed. But only if `cellAnimations` are enabled.
 	/// - Parameter fromSource: If `false` only re-built filter and sort order
 	func reload(fromSource: Bool, whenDone: @escaping () -> Void) {
 		DispatchQueue.global().async {
@@ -65,7 +59,7 @@ class FilterPipeline<T> {
 			}
 			self.resetFilters()
 			DispatchQueue.main.sync {
-				self.delegate?.tableView.reloadData()
+				self.reloadTableCells()
 				whenDone()
 			}
 		}
@@ -86,6 +80,7 @@ class FilterPipeline<T> {
 	
 	/// Add new filter layer. Each layer is applied upon the previous layer. Therefore, each filter
 	/// can only restrict the display further. A filter cannot introduce previously removed elements.
+	/// - Note: Will call `reloadData()` if `cellAnimations` are enabled.
 	/// - Parameters:
 	///   - identifier: Use this id to find the filter again. For reload and remove operations.
 	///   - otherId: If `nil` or non-existent the new filter will be appended at the end.
@@ -100,10 +95,11 @@ class FilterPipeline<T> {
 			pipeline.append(newFilter)
 			display?.apply(moreRestrictive: newFilter.selection)
 		}
-		if cellAnimations { delegate?.tableView.reloadData() }
+		reloadTableCells()
 	}
 	
 	/// Find and remove filter with given identifier. Will automatically update remaining filters and display sorting.
+	/// - Note: Will call `reloadData()` if `cellAnimations` are enabled.
 	func removeFilter(withId ident: String) {
 		guard let i = indexOfFilter(ident) else { return }
 		pipeline.remove(at: i)
@@ -113,29 +109,23 @@ class FilterPipeline<T> {
 		} else {
 			resetFilters(startingAt: i)
 		}
-		if cellAnimations { delegate?.tableView.reloadData() }
+		reloadTableCells()
 	}
 	
 	/// Start filter evaluation on all entries from previous filter.
+	/// - Note: Will call `reloadData()` if `cellAnimations` are enabled.
 	func reloadFilter(withId ident: String) {
 		guard let i = indexOfFilter(ident) else { return }
 		resetFilters(startingAt: i)
-		if cellAnimations { delegate?.tableView.reloadData() }
+		reloadTableCells()
 	}
 	
-	/// Remove last `k` filters from the filter pipeline. Thus showing more entries from previous layers.
-//	func popLastFilter(k: Int = 1) {
-//		guard k > 0, k <= pipeline.count else { return }
-//		pipeline.removeLast(k)
-//		display?.reset(toLessRestrictive: lastFilterLayerIndices())
-//		if cellAnimations { delegate?.tableView.reloadData() }
-//	}
-	
 	/// Sets the sort and display order. You should set the `delegate` to automatically update your `tableView`.
+	/// - Note: Will call `reloadData()` if `cellAnimations` are enabled.
 	/// - Parameter predicate: Return `true` if first element should be sorted before second element.
 	func setSorting(_ predicate: @escaping PipelineSorting<T>.Predicate) {
 		display = .init(predicate, pipe: self)
-		if cellAnimations { delegate?.tableView.reloadData() }
+		reloadTableCells()
 	}
 	
 	/// Re-built filter and display sorting order.
@@ -188,6 +178,11 @@ class FilterPipeline<T> {
 		}
 	}
 	
+	/// Reload table but only if `cellAnimations` is enabled.
+	func reloadTableCells() {
+		if cellAnimations { delegate?.tableView.reloadData() }
+	}
+	
 	/// Add new element to the original `dataSource` and immediately apply filter and sorting.
 	/// - Complexity: O((*m*+1) log *n*), where *m* is the number of filters and *n* the number of elements in each filter.
 	func addNew(_ obj: T) {
@@ -301,14 +296,6 @@ class PipelineFilter<T> {
 		selection.binTreeRemove(index, compare: (<))
 	}
 	
-	/// Find `selection` index for corresponding `dataSource` index
-	/// - Parameter index: Index of object in original `dataSource`
-	/// - Returns: Index in `selection` or `nil` if element does not exist.
-	/// - Complexity: O(log *n*), where *n* is the length of the `selection`.
-	fileprivate func index(ofDataSource index: Int) -> Int? {
-		selection.binTreeIndex(of: index, compare: (<), mustExist: true)
-	}
-	
 	/// Perform filter check and update internal `selection` indices.
 	/// - Parameters:
 	///   - obj: Object that was inserted or updated.
@@ -317,7 +304,7 @@ class PipelineFilter<T> {
 	///            `idx` contains the selection filter index or `nil` if the value should be removed.
 	/// - Complexity: O(log *n*), where *n* is the length of the `selection`.
 	fileprivate func update(_ obj: T, at index: Int) -> (keep: Bool, idx: Int?) {
-		let currentIndex = self.index(ofDataSource: index)
+		let currentIndex = selection.binTreeIndex(of: index, compare: (<), mustExist: true)
 		if shouldPersist(obj) {
 			return (true, currentIndex ?? selection.binTreeInsert(index, compare: (<)))
 		}

main/DB/DBAppOnly.swift

@@ -109,9 +109,9 @@ extension SQLiteDatabase {
 		return (before > after) ? nil : (before, after)
 	}
 	
-	/// `DELETE FROM heap; DELETE FROM cache;`
+	/// `DELETE FROM cache; DELETE FROM heap;`
 	func dnsLogsDeleteAll() throws {
-		try? run(sql: "DELETE FROM heap; DELETE FROM cache;")
+		try? run(sql: "DELETE FROM cache; DELETE FROM heap;")
 		vacuum()
 	}
 	

main/Data Source/GroupedDomainDataSource.swift

@@ -56,6 +56,7 @@ class GroupedDomainDataSource {
 			sync.continue()
 		} else {
 			pipeline.reload(fromSource: true, whenDone: {
+				sync.syncNow() // sync outstanding entries in cache
 				sync.continue()
 				refreshControl?.endRefreshing()
 			})

main/Data Source/SyncUpdate.swift

@@ -1,6 +1,7 @@
 import Foundation
 
 class SyncUpdate {
+	private var lastSync: TimeInterval = 0
 	private var timer: Timer!
 	private var paused: Int = 1 // first start() will decrement
 	private(set) var tsEarliest: Timestamp
@@ -9,47 +10,65 @@ class SyncUpdate {
 		tsEarliest = Pref.DateFilter.lastXMinTimestamp() ?? 0
 		NotifyDateFilterChanged.observe(call: #selector(didChangeDateFilter), on: self)
 		timer = Timer.repeating(interval, call: #selector(periodicUpdate), on: self)
+		syncNow() // because timer will only fire after interval
 	}
 	
 	@objc private func periodicUpdate() { if paused == 0 { syncNow() } }
 	
 	@objc private func didChangeDateFilter() {
-		let lastXFilter = Pref.DateFilter.lastXMinTimestamp() ?? 0
+		DispatchQueue.global().async {
-		let before = tsEarliest
+			self.set(newEarliest: Pref.DateFilter.lastXMinTimestamp() ?? 0)
-		tsEarliest = lastXFilter
-		if before < lastXFilter {
-			DispatchQueue.global().async {
-				if let excess = AppDB?.dnsLogsRowRange(between: before, and: lastXFilter) {
-					NotifySyncRemove.postAsyncMain(excess)
-				}
-			}
-		} else if before > lastXFilter {
-			DispatchQueue.global().async {
-				if let missing = AppDB?.dnsLogsRowRange(between: lastXFilter, and: before) {
-					NotifySyncInsert.postAsyncMain(missing)
-				}
-			}
 		}
 	}
 	
+	/// This will immediately resume timer updates, ignoring previous `pause()` requests.
 	func start() { paused = 0 }
+	
+	/// All calls must be balanced with `continue()` calls.
+	/// Can be nested within other `pause-continue` pairs.
+	/// - Warning: An execution branch that results in unbalanced pairs will completely disable updates!
 	func pause() { paused += 1 }
+	
+	/// Must be balanced with a `pause()` call. A `continue()` without a `pause()` is a `nop`.
+	/// - Note: Internally the sync timer keeps running. The `pause` will simply ignore execution during that time.
 	func `continue`() { if paused > 0 { paused -= 1 } }
 	
+	/// Persist logs from cache and notify all observers. (`NotifySyncInsert`)
+	/// Determine rows of outdated entries that should be removed and notify observers as well. (`NotifySyncRemove`)
+	/// - Note: This method is rate limited. Sync will be performed at most once per second.
+	/// - Note: This method returns immediatelly. Syncing is done in a background thread.
 	func syncNow() {
-		self.pause() // reduce concurrent load
+		let now = Date().timeIntervalSince1970
+		guard (now - lastSync) > 1 else { return } // rate limiting
+		lastSync = now
 		
-		if let inserted = AppDB?.dnsLogsPersist() { // move cache -> heap
+		DispatchQueue.global().async {
-			NotifySyncInsert.post(inserted)
+			self.pause() // reduce concurrent load
-		}
+			
-		if let lastXFilter = Pref.DateFilter.lastXMinTimestamp(), tsEarliest < lastXFilter {
+			if let inserted = AppDB?.dnsLogsPersist() { // move cache -> heap
-			if let removed = AppDB?.dnsLogsRowRange(between: tsEarliest, and: lastXFilter) {
+				NotifySyncInsert.postAsyncMain(inserted)
-				NotifySyncRemove.post(removed)
 			}
-			tsEarliest = lastXFilter
+			if let lastXFilter = Pref.DateFilter.lastXMinTimestamp() {
-		}
+				self.set(newEarliest: lastXFilter)
-		// TODO: periodic hard delete old logs (will reset rowids!)
+			}
+			// TODO: periodic hard delete old logs (will reset rowids!)
 			
-		self.continue()
+			self.continue()
+		}
+	}
+	
+	/// - Warning: Always call from a background thread!
+	private func set(newEarliest: Timestamp) {
+		let current = tsEarliest
+		tsEarliest = newEarliest
+		if current < newEarliest {
+			if let excess = AppDB?.dnsLogsRowRange(between: current, and: newEarliest) {
+				NotifySyncRemove.postAsyncMain(excess)
+			}
+		} else if current > newEarliest {
+			if let missing = AppDB?.dnsLogsRowRange(between: newEarliest, and: current) {
+				NotifySyncInsert.postAsyncMain(missing)
+			}
+		} // else: nothing changed
 	}
 }

main/Requests/TVCHostDetails.swift

@@ -27,6 +27,7 @@ class TVCHostDetails: UITableViewController {
 			self?.dataSource = AppDB?.timesForDomain(self?.fullDomain ?? "", since: sync.tsEarliest) ?? []
 			DispatchQueue.main.sync {
 				self?.tableView.reloadData()
+				sync.syncNow() // sync outstanding entries in cache
 				refreshControl?.endRefreshing()
 			}
 		}