From b4b89f8bb4807941e0fcf9d602425ba00551ea67 Mon Sep 17 00:00:00 2001 From: relikd Date: Fri, 5 Jun 2020 18:12:31 +0200 Subject: [PATCH] Persist cache with pull-to-refresh + Sync rate limiting --- main/Common Classes/FilterPipeline.swift | 47 +++++------- main/DB/DBAppOnly.swift | 4 +- .../Data Source/GroupedDomainDataSource.swift | 1 + main/Data Source/SyncUpdate.swift | 71 ++++++++++++------- main/Requests/TVCHostDetails.swift | 1 + 5 files changed, 66 insertions(+), 58 deletions(-) diff --git a/main/Common Classes/FilterPipeline.swift b/main/Common Classes/FilterPipeline.swift index 7c916b1..e2eab87 100644 --- a/main/Common Classes/FilterPipeline.swift +++ b/main/Common Classes/FilterPipeline.swift @@ -24,7 +24,7 @@ class FilterPipeline { } /// Set a new `dataSource` query and immediately apply all filters and sorting. - /// - Note: You must call `reload(fromSource:)` manually! + /// - Note: You must call `reload(fromSource:whenDone:)` manually! /// - Note: Always use `[unowned self]` func setDataSource(query: @escaping DataSourceQuery) { sourceQuery = query @@ -49,14 +49,8 @@ class FilterPipeline { return (i, dataSource[i]) } - /// Search and return list of `dataSource` elements that match the given `predicate`. - /// - Returns: Sorted list of indices and objects in `dataSource`. - /// - Complexity: O(*m* + *n*), where *n* is the length of the `dataSource` and *m* is the number of matches. -// func dataSourceAll(where predicate: ((T) -> Bool)) -> [(index: Int, object: T)] { -// dataSource.enumerated().compactMap { predicate($1) ? ($0, $1) : nil } -// } - /// Re-query data source and re-built filter and display sorting order. + /// - Note: Will call `reloadData()` before `whenDone` closure is executed. But only if `cellAnimations` are enabled. /// - Parameter fromSource: If `false` only re-built filter and sort order func reload(fromSource: Bool, whenDone: @escaping () -> Void) { DispatchQueue.global().async { @@ -65,7 +59,7 @@ class FilterPipeline { } self.resetFilters() DispatchQueue.main.sync { - self.delegate?.tableView.reloadData() + self.reloadTableCells() whenDone() } } @@ -86,6 +80,7 @@ class FilterPipeline { /// Add new filter layer. Each layer is applied upon the previous layer. Therefore, each filter /// can only restrict the display further. A filter cannot introduce previously removed elements. + /// - Note: Will call `reloadData()` if `cellAnimations` are enabled. /// - Parameters: /// - identifier: Use this id to find the filter again. For reload and remove operations. /// - otherId: If `nil` or non-existent the new filter will be appended at the end. @@ -100,10 +95,11 @@ class FilterPipeline { pipeline.append(newFilter) display?.apply(moreRestrictive: newFilter.selection) } - if cellAnimations { delegate?.tableView.reloadData() } + reloadTableCells() } /// Find and remove filter with given identifier. Will automatically update remaining filters and display sorting. + /// - Note: Will call `reloadData()` if `cellAnimations` are enabled. func removeFilter(withId ident: String) { guard let i = indexOfFilter(ident) else { return } pipeline.remove(at: i) @@ -113,29 +109,23 @@ class FilterPipeline { } else { resetFilters(startingAt: i) } - if cellAnimations { delegate?.tableView.reloadData() } + reloadTableCells() } /// Start filter evaluation on all entries from previous filter. + /// - Note: Will call `reloadData()` if `cellAnimations` are enabled. func reloadFilter(withId ident: String) { guard let i = indexOfFilter(ident) else { return } resetFilters(startingAt: i) - if cellAnimations { delegate?.tableView.reloadData() } + reloadTableCells() } - /// Remove last `k` filters from the filter pipeline. Thus showing more entries from previous layers. -// func popLastFilter(k: Int = 1) { -// guard k > 0, k <= pipeline.count else { return } -// pipeline.removeLast(k) -// display?.reset(toLessRestrictive: lastFilterLayerIndices()) -// if cellAnimations { delegate?.tableView.reloadData() } -// } - /// Sets the sort and display order. You should set the `delegate` to automatically update your `tableView`. + /// - Note: Will call `reloadData()` if `cellAnimations` are enabled. /// - Parameter predicate: Return `true` if first element should be sorted before second element. func setSorting(_ predicate: @escaping PipelineSorting.Predicate) { display = .init(predicate, pipe: self) - if cellAnimations { delegate?.tableView.reloadData() } + reloadTableCells() } /// Re-built filter and display sorting order. @@ -188,6 +178,11 @@ class FilterPipeline { } } + /// Reload table but only if `cellAnimations` is enabled. + func reloadTableCells() { + if cellAnimations { delegate?.tableView.reloadData() } + } + /// Add new element to the original `dataSource` and immediately apply filter and sorting. /// - Complexity: O((*m*+1) log *n*), where *m* is the number of filters and *n* the number of elements in each filter. func addNew(_ obj: T) { @@ -301,14 +296,6 @@ class PipelineFilter { selection.binTreeRemove(index, compare: (<)) } - /// Find `selection` index for corresponding `dataSource` index - /// - Parameter index: Index of object in original `dataSource` - /// - Returns: Index in `selection` or `nil` if element does not exist. - /// - Complexity: O(log *n*), where *n* is the length of the `selection`. - fileprivate func index(ofDataSource index: Int) -> Int? { - selection.binTreeIndex(of: index, compare: (<), mustExist: true) - } - /// Perform filter check and update internal `selection` indices. /// - Parameters: /// - obj: Object that was inserted or updated. @@ -317,7 +304,7 @@ class PipelineFilter { /// `idx` contains the selection filter index or `nil` if the value should be removed. /// - Complexity: O(log *n*), where *n* is the length of the `selection`. fileprivate func update(_ obj: T, at index: Int) -> (keep: Bool, idx: Int?) { - let currentIndex = self.index(ofDataSource: index) + let currentIndex = selection.binTreeIndex(of: index, compare: (<), mustExist: true) if shouldPersist(obj) { return (true, currentIndex ?? selection.binTreeInsert(index, compare: (<))) } diff --git a/main/DB/DBAppOnly.swift b/main/DB/DBAppOnly.swift index bb90bd7..8e191c6 100644 --- a/main/DB/DBAppOnly.swift +++ b/main/DB/DBAppOnly.swift @@ -109,9 +109,9 @@ extension SQLiteDatabase { return (before > after) ? nil : (before, after) } - /// `DELETE FROM heap; DELETE FROM cache;` + /// `DELETE FROM cache; DELETE FROM heap;` func dnsLogsDeleteAll() throws { - try? run(sql: "DELETE FROM heap; DELETE FROM cache;") + try? run(sql: "DELETE FROM cache; DELETE FROM heap;") vacuum() } diff --git a/main/Data Source/GroupedDomainDataSource.swift b/main/Data Source/GroupedDomainDataSource.swift index c3ea206..6999beb 100644 --- a/main/Data Source/GroupedDomainDataSource.swift +++ b/main/Data Source/GroupedDomainDataSource.swift @@ -56,6 +56,7 @@ class GroupedDomainDataSource { sync.continue() } else { pipeline.reload(fromSource: true, whenDone: { + sync.syncNow() // sync outstanding entries in cache sync.continue() refreshControl?.endRefreshing() }) diff --git a/main/Data Source/SyncUpdate.swift b/main/Data Source/SyncUpdate.swift index 21b4792..28813f7 100644 --- a/main/Data Source/SyncUpdate.swift +++ b/main/Data Source/SyncUpdate.swift @@ -1,6 +1,7 @@ import Foundation class SyncUpdate { + private var lastSync: TimeInterval = 0 private var timer: Timer! private var paused: Int = 1 // first start() will decrement private(set) var tsEarliest: Timestamp @@ -9,47 +10,65 @@ class SyncUpdate { tsEarliest = Pref.DateFilter.lastXMinTimestamp() ?? 0 NotifyDateFilterChanged.observe(call: #selector(didChangeDateFilter), on: self) timer = Timer.repeating(interval, call: #selector(periodicUpdate), on: self) + syncNow() // because timer will only fire after interval } @objc private func periodicUpdate() { if paused == 0 { syncNow() } } @objc private func didChangeDateFilter() { - let lastXFilter = Pref.DateFilter.lastXMinTimestamp() ?? 0 - let before = tsEarliest - tsEarliest = lastXFilter - if before < lastXFilter { - DispatchQueue.global().async { - if let excess = AppDB?.dnsLogsRowRange(between: before, and: lastXFilter) { - NotifySyncRemove.postAsyncMain(excess) - } - } - } else if before > lastXFilter { - DispatchQueue.global().async { - if let missing = AppDB?.dnsLogsRowRange(between: lastXFilter, and: before) { - NotifySyncInsert.postAsyncMain(missing) - } - } + DispatchQueue.global().async { + self.set(newEarliest: Pref.DateFilter.lastXMinTimestamp() ?? 0) } } + /// This will immediately resume timer updates, ignoring previous `pause()` requests. func start() { paused = 0 } + + /// All calls must be balanced with `continue()` calls. + /// Can be nested within other `pause-continue` pairs. + /// - Warning: An execution branch that results in unbalanced pairs will completely disable updates! func pause() { paused += 1 } + + /// Must be balanced with a `pause()` call. A `continue()` without a `pause()` is a `nop`. + /// - Note: Internally the sync timer keeps running. The `pause` will simply ignore execution during that time. func `continue`() { if paused > 0 { paused -= 1 } } + /// Persist logs from cache and notify all observers. (`NotifySyncInsert`) + /// Determine rows of outdated entries that should be removed and notify observers as well. (`NotifySyncRemove`) + /// - Note: This method is rate limited. Sync will be performed at most once per second. + /// - Note: This method returns immediatelly. Syncing is done in a background thread. func syncNow() { - self.pause() // reduce concurrent load + let now = Date().timeIntervalSince1970 + guard (now - lastSync) > 1 else { return } // rate limiting + lastSync = now - if let inserted = AppDB?.dnsLogsPersist() { // move cache -> heap - NotifySyncInsert.post(inserted) - } - if let lastXFilter = Pref.DateFilter.lastXMinTimestamp(), tsEarliest < lastXFilter { - if let removed = AppDB?.dnsLogsRowRange(between: tsEarliest, and: lastXFilter) { - NotifySyncRemove.post(removed) + DispatchQueue.global().async { + self.pause() // reduce concurrent load + + if let inserted = AppDB?.dnsLogsPersist() { // move cache -> heap + NotifySyncInsert.postAsyncMain(inserted) } - tsEarliest = lastXFilter + if let lastXFilter = Pref.DateFilter.lastXMinTimestamp() { + self.set(newEarliest: lastXFilter) + } + // TODO: periodic hard delete old logs (will reset rowids!) + + self.continue() } - // TODO: periodic hard delete old logs (will reset rowids!) - - self.continue() + } + + /// - Warning: Always call from a background thread! + private func set(newEarliest: Timestamp) { + let current = tsEarliest + tsEarliest = newEarliest + if current < newEarliest { + if let excess = AppDB?.dnsLogsRowRange(between: current, and: newEarliest) { + NotifySyncRemove.postAsyncMain(excess) + } + } else if current > newEarliest { + if let missing = AppDB?.dnsLogsRowRange(between: newEarliest, and: current) { + NotifySyncInsert.postAsyncMain(missing) + } + } // else: nothing changed } } diff --git a/main/Requests/TVCHostDetails.swift b/main/Requests/TVCHostDetails.swift index 01c8c9a..25c2fb8 100644 --- a/main/Requests/TVCHostDetails.swift +++ b/main/Requests/TVCHostDetails.swift @@ -27,6 +27,7 @@ class TVCHostDetails: UITableViewController { self?.dataSource = AppDB?.timesForDomain(self?.fullDomain ?? "", since: sync.tsEarliest) ?? [] DispatchQueue.main.sync { self?.tableView.reloadData() + sync.syncNow() // sync outstanding entries in cache refreshControl?.endRefreshing() } }