From f0258fb246d7cfb2b6a4fcf99ffcb0d77bf1eecf Mon Sep 17 00:00:00 2001 From: relikd Date: Sat, 26 Jan 2019 18:47:48 +0100 Subject: [PATCH] Proper caching for downloads --- README.md | 2 +- baRSS.xcodeproj/project.pbxproj | 2 - baRSS/Categories/FeedMeta+Ext.m | 6 +- baRSS/FeedDownload.m | 106 +++++++++++++++++++++++++------- 4 files changed, 90 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e2f782a..2c656d8 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ ToDo - [ ] Other - [ ] App Icon - [ ] Translate text to different languages - - [ ] Download with ephemeral url session? + - [x] Download with ephemeral url session? - [ ] Add Sandboxing - [ ] Disable Startup checkbox (or other workaround) diff --git a/baRSS.xcodeproj/project.pbxproj b/baRSS.xcodeproj/project.pbxproj index 9552e18..1dd0d82 100644 --- a/baRSS.xcodeproj/project.pbxproj +++ b/baRSS.xcodeproj/project.pbxproj @@ -7,7 +7,6 @@ objects = { /* Begin PBXBuildFile section */ - 540CD14921C094A2004AB594 /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = 540CD14821C094A2004AB594 /* README.md */; }; 540F704521B6C16C0022E69D /* FeedMeta+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 540F704421B6C16C0022E69D /* FeedMeta+Ext.m */; }; 54195883218A061100581B79 /* Feed+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 54195882218A061100581B79 /* Feed+Ext.m */; }; 54195886218E1BDB00581B79 /* NSMenu+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 54195885218E1BDB00581B79 /* NSMenu+Ext.m */; }; @@ -385,7 +384,6 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( - 540CD14921C094A2004AB594 /* README.md in Resources */, 546FC4472118A8E6007CC3A3 /* Preferences.xib in Resources */, 54E88321211B509D00064188 /* ModalFeedEdit.xib in Resources */, 54ACC28621061B3C0020715F /* Assets.xcassets in Resources */, diff --git a/baRSS/Categories/FeedMeta+Ext.m b/baRSS/Categories/FeedMeta+Ext.m index beab2a9..7fb1ecc 100644 --- a/baRSS/Categories/FeedMeta+Ext.m +++ b/baRSS/Categories/FeedMeta+Ext.m @@ -55,11 +55,13 @@ static const int32_t RefreshUnitValues[] = {1, 60, 3600, 86400, 604800}; // smhd if (self.errorCount < 0) self.errorCount = 0; int16_t n = self.errorCount + 1; // always increment errorCount (can be used to indicate bad feeds) + // TODO: remove logging + NSLog(@"ERROR: Feed download failed: %@ (errorCount: %d)", self.url, n); + if ([self.scheduled timeIntervalSinceNow] > 30) // forced, early update. Scheduled is still in the futute. + return; // Keep error counter low. Not enough time has passed (e.g., temporary server outage) NSTimeInterval retryWaitTime = pow(2, (n > 13 ? 13 : n)) * 60; // 2^N (between: 2 minutes and 5.7 days) self.errorCount = n; [self scheduleNow:retryWaitTime]; - // TODO: remove logging - NSLog(@"ERROR: Feed download failed: %@ (errorCount: %d)", self.url, n); } - (void)setSucessfulWithResponse:(NSHTTPURLResponse*)response { diff --git a/baRSS/FeedDownload.m b/baRSS/FeedDownload.m index 93f2adf..2e59661 100644 --- a/baRSS/FeedDownload.m +++ b/baRSS/FeedDownload.m @@ -161,31 +161,43 @@ static BOOL _nextUpdateIsForced = NO; /// @return New request with no caching policy and timeout interval of 30 seconds. + (NSMutableURLRequest*)newRequestURL:(NSString*)urlStr { - NSMutableURLRequest *req = [NSMutableURLRequest requestWithURL:[self fixURL:urlStr]]; - req.cachePolicy = NSURLRequestReloadIgnoringLocalCacheData; - req.HTTPShouldHandleCookies = NO; -// req.timeoutInterval = 30; - return req; + return [NSMutableURLRequest requestWithURL:[self fixURL:urlStr]]; } /// @return New request with etag and modified headers set (or not, if @c flag @c == @c YES ). + (NSURLRequest*)newRequest:(FeedMeta*)meta ignoreCache:(BOOL)flag { NSMutableURLRequest *req = [self newRequestURL:meta.url]; if (!flag) { - NSString* etag = [meta.etag stringByReplacingOccurrencesOfString:@"-gzip" withString:@""]; - if (meta.modified.length > 0) + if (meta.etag.length > 0) + [req setValue:meta.etag forHTTPHeaderField:@"If-None-Match"]; // ETag + else if (meta.modified.length > 0) [req setValue:meta.modified forHTTPHeaderField:@"If-Modified-Since"]; - if (etag.length > 0) - [req setValue:etag forHTTPHeaderField:@"If-None-Match"]; // ETag } if (!_nextUpdateIsForced) // any request that is not forced, is a background update req.networkServiceType = NSURLNetworkServiceTypeBackground; return req; } ++ (NSURLSession*)nonCachingSession { + static NSURLSession *session = nil; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSURLSessionConfiguration *conf = [NSURLSessionConfiguration defaultSessionConfiguration]; + conf.HTTPCookieAcceptPolicy = NSHTTPCookieAcceptPolicyNever; + conf.HTTPShouldSetCookies = NO; + conf.HTTPCookieStorage = nil; // disables '~/Library/Cookies/' + conf.requestCachePolicy = NSURLRequestReloadIgnoringLocalCacheData; + conf.URLCache = nil; // disables '~/Library/Caches/de.relikd.baRSS/' + conf.HTTPAdditionalHeaders = @{ @"User-Agent": @"baRSS (macOS)", + @"Accept-Encoding": @"gzip" }; + session = [NSURLSession sessionWithConfiguration:conf]; + }); + return session; // [NSURLSession sharedSession]; +} + /// Helper method to start new @c NSURLSession. If @c (http.statusCode==304) then set @c data @c = @c nil. + (void)asyncRequest:(NSURLRequest*)request block:(nonnull void(^)(NSData * _Nullable data, NSError * _Nullable error, NSHTTPURLResponse *response))block { - [[[NSURLSession sharedSession] dataTaskWithRequest:request completionHandler:^(NSData * _Nullable data, NSURLResponse * _Nullable response, NSError * _Nullable error) { + [[[self nonCachingSession] dataTaskWithRequest:request completionHandler:^(NSData * _Nullable data, NSURLResponse * _Nullable response, NSError * _Nullable error) { NSHTTPURLResponse* httpResponse = (NSHTTPURLResponse*)response; if (error || [httpResponse statusCode] == 304) data = nil; @@ -285,7 +297,6 @@ static BOOL _nextUpdateIsForced = NO; alertPopup.informativeText = [NSString stringWithFormat:@"Error loading source: %@", reqURL]; [alertPopup runModal]; } - // TODO: don't increase error count on forced update [f.meta setErrorAndPostponeSchedule]; } else { success = YES; @@ -397,12 +408,67 @@ static BOOL _nextUpdateIsForced = NO; /// Download favicon located at http://.../ @c favicon.ico. Callback @c block will be called on main thread. + (void)downloadFavicon:(NSString*)urlStr finished:(void(^)(NSImage * _Nullable img))block { - dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - NSURL *favURL = [[self hostURL:urlStr] URLByAppendingPathComponent:@"favicon.ico"]; - // TODO: fix anonymous session. initWithContentsOfURL: will set cookie in ~/Library/Cookies/ - // TODO: check ~/Library/Caches/de.relikd.baRSS/fsCachedData/ - // TODO: fix missing favicon by parsing html - NSImage *img = [[NSImage alloc] initWithContentsOfURL:favURL]; + NSURL *host = [self hostURL:urlStr]; + NSString *hostURL = host.absoluteString; + NSString *favURL = [host URLByAppendingPathComponent:@"favicon.ico"].absoluteString; + [self downloadImage:favURL finished:^(NSImage * _Nullable img) { + if (img) { + block(img); // is on main already (from downloadImage:) + } else { + [self downloadFaviconByParsingHTML:hostURL finished:block]; + } + }]; +} + +/// Download html page and parse all icon urls. Starting a successive request on the url of the smallest icon. ++ (void)downloadFaviconByParsingHTML:(NSString*)hostURL finished:(void(^)(NSImage * _Nullable img))block { + [self asyncRequest:[self newRequestURL:hostURL] block:^(NSData * _Nullable htmlData, NSError * _Nullable error, NSHTTPURLResponse *response) { + if (htmlData) { + // TODO: use session delegate to stop downloading after + RSXMLData *xml = [[RSXMLData alloc] initWithData:htmlData urlString:hostURL]; + RSHTMLMetadataParser *parser = [RSHTMLMetadataParser parserWithXMLData:xml]; + RSHTMLMetadata *meta = [parser parseSync:&error]; + if (error) meta = nil; + NSString *iconURL = [self faviconUrlForMetadata:meta]; + if (iconURL) { + // if everything went well we can finally start a request on the url we found. + [self downloadImage:iconURL finished:block]; + return; + } + } + dispatch_async(dispatch_get_main_queue(), ^{ block(nil); }); // on failure + }]; +} + +/// Extract favicon URL from parsed HTML metadata. ++ (NSString*)faviconUrlForMetadata:(RSHTMLMetadata*)meta { + if (meta) { + if (meta.faviconLink.length > 0) { + return meta.faviconLink; + } + else if (meta.iconLinks.count > 0) { + // at least any url (even if all items in list have size 0) + NSString *iconURL = meta.iconLinks.firstObject.link; + // we dont need much, lets find the smallest icon ... + int smallest = 9001; + for (RSHTMLMetadataIconLink *icon in meta.iconLinks) { + int size = (int)[icon getSize].width; + if (size > 0 && size < smallest) { + smallest = size; + iconURL = icon.link; + } + } + if (iconURL && iconURL.length > 0) + return iconURL; + } + } + return nil; +} + +/// Download image in a background thread and notify once finished. ++ (void)downloadImage:(NSString*)url finished:(void(^)(NSImage * _Nullable img))block { + [self asyncRequest:[self newRequestURL:url] block:^(NSData * _Nullable data, NSError * _Nullable e, NSHTTPURLResponse *r) { + NSImage *img = [[NSImage alloc] initWithData:data]; if (!img || ![img isValid]) img = nil; // if (img.size.width > 16 || img.size.height > 16) { @@ -413,10 +479,8 @@ static BOOL _nextUpdateIsForced = NO; // if (img.TIFFRepresentation.length > smallImage.TIFFRepresentation.length) // img = smallImage; // } - dispatch_async(dispatch_get_main_queue(), ^{ - block(img); - }); - }); + dispatch_async(dispatch_get_main_queue(), ^{ block(img); }); + }]; }