From 37d3a461d6f3e4d560f3d31109a076d69d5ba09b Mon Sep 17 00:00:00 2001 From: relikd Date: Wed, 18 Sep 2019 17:21:37 +0200 Subject: [PATCH] Parser for YouTube URLs (channel, user, playlist) --- CHANGELOG.md | 1 + baRSS.xcodeproj/project.pbxproj | 6 + baRSS/Constants.h | 2 + baRSS/Core Data/FeedGroup+Ext.m | 2 - baRSS/Feed Import/Download3rdParty.h | 30 +++++ baRSS/Feed Import/Download3rdParty.m | 84 ++++++++++++++ baRSS/Feed Import/FeedDownload.m | 103 ++++++++++-------- baRSS/Helper/NSError+Ext.h | 3 + baRSS/Helper/NSError+Ext.m | 18 ++- baRSS/Info.plist | 2 +- .../Preferences/Feeds Tab/SettingsFeedsView.m | 1 - 11 files changed, 203 insertions(+), 49 deletions(-) create mode 100644 baRSS/Feed Import/Download3rdParty.h create mode 100644 baRSS/Feed Import/Download3rdParty.m diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cfb25f..3a93618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project does adhere to [Semantic Versioning](https://semver.org/spec/v2 - Quick Look preview for OPML files - *Adding feed:* 5xx server errors have a reload button which will initiate a new download with the same URL - *Adding feed:* Empty feed title will automatically reuse title from xml file (even if xml title changes) +- *Adding feed:* Parser for YouTube channel, user, and playlist URLs - *Adding feed:* `⌘R` will reload the same URL - *Settings, Feeds:* `⌘R` will reload the data source - *Settings, Feeds:* Refresh interval string localizations diff --git a/baRSS.xcodeproj/project.pbxproj b/baRSS.xcodeproj/project.pbxproj index dedb9b9..bf3eafe 100644 --- a/baRSS.xcodeproj/project.pbxproj +++ b/baRSS.xcodeproj/project.pbxproj @@ -26,6 +26,7 @@ 5477D34E21233C62002BA27F /* FeedGroup+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 5477D34D21233C62002BA27F /* FeedGroup+Ext.m */; }; 5478DF04225A7AE200D30C64 /* SettingsFeedsView.m in Sources */ = {isa = PBXBuildFile; fileRef = 5478DF03225A7AE200D30C64 /* SettingsFeedsView.m */; }; 548C6D0A230C33DE003A1AAF /* NSURL+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 548C6D09230C33DE003A1AAF /* NSURL+Ext.m */; }; + 5491005D2331435E00858AE2 /* Download3rdParty.m in Sources */ = {isa = PBXBuildFile; fileRef = 5491005C2331435E00858AE2 /* Download3rdParty.m */; }; 5496B511214D6275003ED4ED /* UserPrefs.m in Sources */ = {isa = PBXBuildFile; fileRef = 5496B510214D6275003ED4ED /* UserPrefs.m */; }; 54A07A7F220E04CF00082C51 /* NSFetchRequest+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 54A07A7E220E04CF00082C51 /* NSFetchRequest+Ext.m */; }; 54A07A82220E723D00082C51 /* MapUnreadTotal.m in Sources */ = {isa = PBXBuildFile; fileRef = 54A07A81220E723D00082C51 /* MapUnreadTotal.m */; }; @@ -135,6 +136,8 @@ 54892F1D2235285700271CBA /* CHANGELOG.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = CHANGELOG.md; sourceTree = ""; }; 548C6D08230C33DE003A1AAF /* NSURL+Ext.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NSURL+Ext.h"; sourceTree = ""; }; 548C6D09230C33DE003A1AAF /* NSURL+Ext.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "NSURL+Ext.m"; sourceTree = ""; }; + 5491005B2331435E00858AE2 /* Download3rdParty.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Download3rdParty.h; sourceTree = ""; }; + 5491005C2331435E00858AE2 /* Download3rdParty.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Download3rdParty.m; sourceTree = ""; }; 5496B50F214D6275003ED4ED /* UserPrefs.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = UserPrefs.h; sourceTree = ""; }; 5496B510214D6275003ED4ED /* UserPrefs.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = UserPrefs.m; sourceTree = ""; }; 54A07A7D220E04CF00082C51 /* NSFetchRequest+Ext.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NSFetchRequest+Ext.h"; sourceTree = ""; }; @@ -334,6 +337,8 @@ children = ( 54ACC29321061E270020715F /* UpdateScheduler.h */, 54ACC29421061E270020715F /* UpdateScheduler.m */, + 5491005B2331435E00858AE2 /* Download3rdParty.h */, + 5491005C2331435E00858AE2 /* Download3rdParty.m */, 5450100E230E9C8600F0B165 /* FeedDownload.h */, 5450100F230E9C8600F0B165 /* FeedDownload.m */, 54B6F148231551B3002C94C9 /* FaviconDownload.h */, @@ -586,6 +591,7 @@ 546A6A2F22C585580034E806 /* SettingsAboutView.m in Sources */, 54B517072270E990006C1B29 /* NSView+Ext.m in Sources */, 54D857CE227C5785001BA1C8 /* RefreshStatisticsView.m in Sources */, + 5491005D2331435E00858AE2 /* Download3rdParty.m in Sources */, 541C67C32255470B004D2CE6 /* SettingsAppearance.m in Sources */, 54B749DA2204A85C0022CC6D /* BarStatusItem.m in Sources */, 546FC43D21188AD5007CC3A3 /* SettingsFeeds.m in Sources */, diff --git a/baRSS/Constants.h b/baRSS/Constants.h index 41c502c..915e4b4 100644 --- a/baRSS/Constants.h +++ b/baRSS/Constants.h @@ -28,6 +28,8 @@ // TODO: Add support for media player? image feed? // // TODO: Disable 'update all' menu item during update? +// TODO: HTML to Feed Generator. https://github.com/RSS-Bridge/rss-bridge +// TODO: SQlite instead of CoreData? https://www.objc.io/issues/4-core-data/SQLite-instead-of-core-data/ /// UTI type used for opml files diff --git a/baRSS/Core Data/FeedGroup+Ext.m b/baRSS/Core Data/FeedGroup+Ext.m index f833806..67224d7 100644 --- a/baRSS/Core Data/FeedGroup+Ext.m +++ b/baRSS/Core Data/FeedGroup+Ext.m @@ -22,9 +22,7 @@ #import "FeedGroup+Ext.h" #import "Feed+Ext.h" -#import "FeedMeta+Ext.h" #import "StoreCoordinator.h" -#import "NSDate+Ext.h" @implementation FeedGroup (Ext) diff --git a/baRSS/Feed Import/Download3rdParty.h b/baRSS/Feed Import/Download3rdParty.h new file mode 100644 index 0000000..06746a2 --- /dev/null +++ b/baRSS/Feed Import/Download3rdParty.h @@ -0,0 +1,30 @@ +// +// The MIT License (MIT) +// Copyright (c) 2019 Oleg Geier +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +@import Cocoa; + +// TODO: Make plugins extensible? community extensions. +@interface YouTubePlugin : NSObject ++ (NSString*)feedURL:(NSURL*)url; ++ (NSString*)videoImage:(NSString*)videoid; ++ (NSString*)videoImageHQ:(NSString*)videoid; +@end diff --git a/baRSS/Feed Import/Download3rdParty.m b/baRSS/Feed Import/Download3rdParty.m new file mode 100644 index 0000000..81820a7 --- /dev/null +++ b/baRSS/Feed Import/Download3rdParty.m @@ -0,0 +1,84 @@ +// +// The MIT License (MIT) +// Copyright (c) 2019 Oleg Geier +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "Download3rdParty.h" + +@implementation YouTubePlugin + +/** + Transforms YouTube URL to XML feed URL. @c https://www.youtube.com/{channel|user|playlist]}/{id} + + @note + Some YouTube HTML pages contain the 'alternate' tag, others don't. + This method will only be executed, if no other feed url was found. + + @return @c nil if @c url is not properly formatted, YouTube feed URL otherwise. + */ ++ (NSString*)feedURL:(NSURL*)url { + if (![url.host hasSuffix:@"youtube.com"]) // 'youtu.be' & 'youtube-nocookie.com' will redirect + return nil; + // https://www.youtube.com/channel/[channel-id] + // https://www.youtube.com/user/[user-name] + // https://www.youtube.com/playlist?list=[playlist-id] +#ifdef DEBUG + printf("resolving YouTube url:\n"); + printf(" ↳ %s\n", url.absoluteString.UTF8String); +#endif + NSString *found = nil; + NSArray *parts = url.pathComponents; + if (parts.count > 1) { // first path component is always '/' + static NSString* const ytBase = @"https://www.youtube.com/feeds/videos.xml"; + NSString *type = parts[1]; + if ([type isEqualToString:@"channel"]) { + if (parts.count > 2) + found = [ytBase stringByAppendingFormat:@"?channel_id=%@", parts[2]]; + } else if ([type isEqualToString:@"user"]) { + if (parts.count > 2) + found = [ytBase stringByAppendingFormat:@"?user=%@", parts[2]]; + } else if ([type isEqualToString:@"playlist"]) { + NSURLComponents *uc = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO]; + for (NSURLQueryItem *q in uc.queryItems) { + if ([q.name isEqualToString:@"list"]) { + found = [ytBase stringByAppendingFormat:@"?playlist_id=%@", q.value]; + break; + } + } + } + } +#ifdef DEBUG + printf(" ↳ %s\n", found ? found.UTF8String : "could not resolve!"); +#endif + return found; // may be nil +} + +/// @return @c http://i.ytimg.com/vi//default.jpg ++ (NSString*)videoImage:(NSString*)videoid { + return [NSString stringWithFormat:@"http://i.ytimg.com/vi/%@/default.jpg", videoid]; +} + +/// @return @c http://i.ytimg.com/vi//hqdefault.jpg ++ (NSString*)videoImageHQ:(NSString*)videoid { + return [NSString stringWithFormat:@"http://i.ytimg.com/vi/%@/hqdefault.jpg", videoid]; +} + +@end + diff --git a/baRSS/Feed Import/FeedDownload.m b/baRSS/Feed Import/FeedDownload.m index aac7018..64aeb86 100644 --- a/baRSS/Feed Import/FeedDownload.m +++ b/baRSS/Feed Import/FeedDownload.m @@ -23,8 +23,10 @@ @import RSXML2; #import "FeedDownload.h" #import "FaviconDownload.h" +#import "Download3rdParty.h" #import "Feed+Ext.h" #import "FeedMeta+Ext.h" +#import "NSError+Ext.h" #import "NSURLRequest+Ext.h" @interface FeedDownload() @@ -122,6 +124,33 @@ [self.currentDownload cancel]; } +/** + Persist in memory object by copying all attributes to permanent core data storage. + + @param flag If @c YES then @c FeedGroup won't increase the error count for the feed. + Feed will be scheduled as soon as the user reconnects to the internet. + @return @c YES if downloaded feed contains at least one article. ( @c 304 returns @c NO ) + */ +- (BOOL)copyValuesTo:(nonnull Feed*)feed ignoreError:(BOOL)flag { + if (!flag && self.error) // Increase error count and schedule next update. + [feed.meta setErrorAndPostponeSchedule]; + else if (self.response) // Update Etag & Last modified and schedule next update. + [feed.meta setSucessfulWithResponse:self.response]; + else // Update URL but keep schedule (e.g., error while adding feed should auto-try once reconnected) + [feed.meta setUrlIfChanged:self.request.URL.absoluteString]; + + // If feed is broken indicate that feed will not be updated + if (!self.xmlfeed || self.xmlfeed.articles.count == 0) + return NO; + // Else: Update stored articles and indicate that feed was updated + [feed updateWithRSS:self.xmlfeed postUnreadCountChange:YES]; + return YES; +} + +// --------------------------------------------------------------- +// | MARK: - HTML Source Handling +// --------------------------------------------------------------- + /// Take the @c urlStr and run a download @c dataTask: on it. Auto-detect if data is HTML or feed. - (void)downloadSource:(NSURLRequest*)request { self.currentDownload = [request dataTask:^(NSData * _Nullable data, NSError * _Nullable error, NSHTTPURLResponse *response) { @@ -133,9 +162,9 @@ } RSXMLData *xml = [[RSXMLData alloc] initWithData:data url:response.URL]; if (!self.assertIsFeedURL && [xml.parserClass isHTMLParser]) - [self processXMLDataHTML:xml]; + [self processXMLDataHTML:xml]; // HTML source handling else - [self processXMLDataFeed:xml]; + [self processXMLDataFeed:xml]; // XML source handling }]; } @@ -143,31 +172,40 @@ - (void)processXMLDataHTML:(RSXMLData*)xml { RSHTMLMetadataParser *parser = [RSHTMLMetadataParser parserWithXMLData:xml]; [parser parseAsync:^(RSHTMLMetadata * _Nullable meta, NSError * _Nullable error) { + NSString *feedURL = nil; if (error) { self.error = error; - } else if (!meta || meta.feedLinks.count == 0) { - self.error = RSXMLMakeErrorWrongParser(RSXMLErrorExpectingFeed, RSXMLErrorExpectingHTML, xml.url); - } else { - self.faviconURL = [FaviconDownload urlForMetadata:meta]; // we can re-use favicon url if we find one - NSString *chosenURL = meta.feedLinks.firstObject.link; - if (self.respondToSelectFeed && meta.feedLinks.count > 1) - chosenURL = [self.delegate feedDownload:self selectFeedFromList:meta.feedLinks]; - - if (chosenURL.length > 0) { - self.assertIsFeedURL = YES; - // Feeds like https://news.ycombinator.com/ return 503 if URLs are requested too rapidly - //CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.0, false); // Non-blocking sleep (1s) - [self downloadSource:[NSURLRequest withURL:chosenURL]]; - return; - } else { // User canceled operation, show appropriate error message - NSDictionary *info = @{ NSLocalizedDescriptionKey: NSLocalizedString(@"Operation canceled.", nil) }; - self.error = [NSError errorWithDomain:NSURLErrorDomain code:NSURLErrorCancelled userInfo:info]; - } } - [self finishAndNotify]; + else if (!meta || meta.feedLinks.count == 0) { + if ([xml.url.host hasSuffix:@"youtube.com"]) + feedURL = [YouTubePlugin feedURL:xml.url]; + if (feedURL.length == 0) + self.error = [NSError feedURLNotFound:xml.url]; + } + else { + feedURL = meta.feedLinks.firstObject.link; + if (meta.feedLinks.count > 1 && self.respondToSelectFeed) + feedURL = [self.delegate feedDownload:self selectFeedFromList:meta.feedLinks]; + if (!feedURL) + self.error = [NSError canceledByUser]; + } + // finalize HTML parsing + if (self.error) { + [self finishAndNotify]; + } else { + self.assertIsFeedURL = YES; + self.faviconURL = [FaviconDownload urlForMetadata:meta]; // re-use favicon url (if present) + // Feeds like https://news.ycombinator.com/ return 503 if URLs are requested too rapidly + //CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.0, false); // Non-blocking sleep (1s) + [self downloadSource:[NSURLRequest withURL:feedURL]]; + } }]; } +// --------------------------------------------------------------- +// | MARK: - XML Source Handling +// --------------------------------------------------------------- + /// The downloaded source seems to be proper feed data, lets parse it with @c RSXML @c RSFeedParser - (void)processXMLDataFeed:(RSXMLData*)xml { RSFeedParser *parser = [RSFeedParser parserWithXMLData:xml]; @@ -197,27 +235,4 @@ if (self.block) { self.block(self); self.block = nil; } } -/** - Persist in memory object by copying all attributes to permanent core data storage. - - @param flag If @c YES then @c FeedGroup won't increase the error count for the feed. - Feed will be scheduled as soon as the user reconnects to the internet. - @return @c YES if downloaded feed contains at least one article. ( @c 304 returns @c NO ) - */ -- (BOOL)copyValuesTo:(nonnull Feed*)feed ignoreError:(BOOL)flag { - if (!flag && self.error) // Increase error count and schedule next update. - [feed.meta setErrorAndPostponeSchedule]; - else if (self.response) // Update Etag & Last modified and schedule next update. - [feed.meta setSucessfulWithResponse:self.response]; - else // Update URL but keep schedule (e.g., error while adding feed should auto-try once reconnected) - [feed.meta setUrlIfChanged:self.request.URL.absoluteString]; - - // If feed is broken indicate that feed will not be updated - if (!self.xmlfeed || self.xmlfeed.articles.count == 0) - return NO; - // Else: Update stored articles and indicate that feed was updated - [feed updateWithRSS:self.xmlfeed postUnreadCountChange:YES]; - return YES; -} - @end diff --git a/baRSS/Helper/NSError+Ext.h b/baRSS/Helper/NSError+Ext.h index 2b0f5ac..abe6cb7 100644 --- a/baRSS/Helper/NSError+Ext.h +++ b/baRSS/Helper/NSError+Ext.h @@ -23,5 +23,8 @@ @import Cocoa; @interface NSError (Ext) +// Generators + (instancetype)statusCode:(NSInteger)code reason:(nullable NSString*)reason; ++ (instancetype)canceledByUser; ++ (instancetype)feedURLNotFound:(NSURL*)url; @end diff --git a/baRSS/Helper/NSError+Ext.m b/baRSS/Helper/NSError+Ext.m index 549fe58..85ca6ed 100644 --- a/baRSS/Helper/NSError+Ext.m +++ b/baRSS/Helper/NSError+Ext.m @@ -20,6 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +@import RSXML2.RSXMLError; #import "NSError+Ext.h" @implementation NSError (Ext) @@ -98,6 +99,10 @@ static const char* CodeDescription(NSInteger code) { return "Unknown"; } +// --------------------------------------------------------------- +// | MARK: - Generators +// --------------------------------------------------------------- + /// Generate @c NSError from HTTP status code. E.g., @c code @c = @c 404 will return "404 Not Found". + (instancetype)statusCode:(NSInteger)code reason:(nullable NSString*)reason { NSMutableDictionary *info = [NSMutableDictionary dictionaryWithCapacity:2]; @@ -107,7 +112,18 @@ static const char* CodeDescription(NSInteger code) { NSInteger errCode = NSURLErrorUnknown; if (code < 500) { if (code >= 400) errCode = NSURLErrorResourceUnavailable; } else if (code < 600) errCode = NSURLErrorBadServerResponse; - return [NSError errorWithDomain:NSURLErrorDomain code:errCode userInfo:info]; + return [self errorWithDomain:NSURLErrorDomain code:errCode userInfo:info]; +} + +/// Generate @c NSError for user canceled operation. With title "Operation canceled.". ++ (instancetype)canceledByUser { + NSDictionary *info = @{ NSLocalizedDescriptionKey: NSLocalizedString(@"Operation canceled.", nil) }; + return [self errorWithDomain:NSURLErrorDomain code:NSURLErrorCancelled userInfo:info]; +} + +/// Generate @c NSError for webpages that don't contain feed urls. ++ (instancetype)feedURLNotFound:(NSURL*)url { + return RSXMLMakeErrorWrongParser(RSXMLErrorExpectingFeed, RSXMLErrorExpectingHTML, url); } @end diff --git a/baRSS/Info.plist b/baRSS/Info.plist index c29a122..b75b610 100644 --- a/baRSS/Info.plist +++ b/baRSS/Info.plist @@ -70,7 +70,7 @@ CFBundleVersion - 13207 + 13404 LSApplicationCategoryType public.app-category.news LSMinimumSystemVersion diff --git a/baRSS/Preferences/Feeds Tab/SettingsFeedsView.m b/baRSS/Preferences/Feeds Tab/SettingsFeedsView.m index 1ed99d7..a8b97ec 100644 --- a/baRSS/Preferences/Feeds Tab/SettingsFeedsView.m +++ b/baRSS/Preferences/Feeds Tab/SettingsFeedsView.m @@ -23,7 +23,6 @@ #import "SettingsFeedsView.h" #import "StoreCoordinator.h" #import "FeedGroup+Ext.h" -#import "FeedMeta+Ext.h" #import "DrawImage.h" #import "SettingsFeeds.h" #import "NSDate+Ext.h"