Parser for YouTube URLs (channel, user, playlist)

This commit is contained in:
relikd
2019-09-18 17:21:37 +02:00
parent 1d9275e0df
commit 37d3a461d6
11 changed files with 203 additions and 49 deletions

View File

@@ -13,6 +13,7 @@ and this project does adhere to [Semantic Versioning](https://semver.org/spec/v2
- Quick Look preview for OPML files
- *Adding feed:* 5xx server errors have a reload button which will initiate a new download with the same URL
- *Adding feed:* Empty feed title will automatically reuse title from xml file (even if xml title changes)
- *Adding feed:* Parser for YouTube channel, user, and playlist URLs
- *Adding feed:* `⌘R` will reload the same URL
- *Settings, Feeds:* `⌘R` will reload the data source
- *Settings, Feeds:* Refresh interval string localizations

View File

@@ -26,6 +26,7 @@
5477D34E21233C62002BA27F /* FeedGroup+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 5477D34D21233C62002BA27F /* FeedGroup+Ext.m */; };
5478DF04225A7AE200D30C64 /* SettingsFeedsView.m in Sources */ = {isa = PBXBuildFile; fileRef = 5478DF03225A7AE200D30C64 /* SettingsFeedsView.m */; };
548C6D0A230C33DE003A1AAF /* NSURL+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 548C6D09230C33DE003A1AAF /* NSURL+Ext.m */; };
5491005D2331435E00858AE2 /* Download3rdParty.m in Sources */ = {isa = PBXBuildFile; fileRef = 5491005C2331435E00858AE2 /* Download3rdParty.m */; };
5496B511214D6275003ED4ED /* UserPrefs.m in Sources */ = {isa = PBXBuildFile; fileRef = 5496B510214D6275003ED4ED /* UserPrefs.m */; };
54A07A7F220E04CF00082C51 /* NSFetchRequest+Ext.m in Sources */ = {isa = PBXBuildFile; fileRef = 54A07A7E220E04CF00082C51 /* NSFetchRequest+Ext.m */; };
54A07A82220E723D00082C51 /* MapUnreadTotal.m in Sources */ = {isa = PBXBuildFile; fileRef = 54A07A81220E723D00082C51 /* MapUnreadTotal.m */; };
@@ -135,6 +136,8 @@
54892F1D2235285700271CBA /* CHANGELOG.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = CHANGELOG.md; sourceTree = "<group>"; };
548C6D08230C33DE003A1AAF /* NSURL+Ext.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NSURL+Ext.h"; sourceTree = "<group>"; };
548C6D09230C33DE003A1AAF /* NSURL+Ext.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = "NSURL+Ext.m"; sourceTree = "<group>"; };
5491005B2331435E00858AE2 /* Download3rdParty.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Download3rdParty.h; sourceTree = "<group>"; };
5491005C2331435E00858AE2 /* Download3rdParty.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Download3rdParty.m; sourceTree = "<group>"; };
5496B50F214D6275003ED4ED /* UserPrefs.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = UserPrefs.h; sourceTree = "<group>"; };
5496B510214D6275003ED4ED /* UserPrefs.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = UserPrefs.m; sourceTree = "<group>"; };
54A07A7D220E04CF00082C51 /* NSFetchRequest+Ext.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "NSFetchRequest+Ext.h"; sourceTree = "<group>"; };
@@ -334,6 +337,8 @@
children = (
54ACC29321061E270020715F /* UpdateScheduler.h */,
54ACC29421061E270020715F /* UpdateScheduler.m */,
5491005B2331435E00858AE2 /* Download3rdParty.h */,
5491005C2331435E00858AE2 /* Download3rdParty.m */,
5450100E230E9C8600F0B165 /* FeedDownload.h */,
5450100F230E9C8600F0B165 /* FeedDownload.m */,
54B6F148231551B3002C94C9 /* FaviconDownload.h */,
@@ -586,6 +591,7 @@
546A6A2F22C585580034E806 /* SettingsAboutView.m in Sources */,
54B517072270E990006C1B29 /* NSView+Ext.m in Sources */,
54D857CE227C5785001BA1C8 /* RefreshStatisticsView.m in Sources */,
5491005D2331435E00858AE2 /* Download3rdParty.m in Sources */,
541C67C32255470B004D2CE6 /* SettingsAppearance.m in Sources */,
54B749DA2204A85C0022CC6D /* BarStatusItem.m in Sources */,
546FC43D21188AD5007CC3A3 /* SettingsFeeds.m in Sources */,

View File

@@ -28,6 +28,8 @@
// TODO: Add support for media player? image feed?
// <enclosure url="https://url.mp3" length="63274022" type="audio/mpeg" />
// TODO: Disable 'update all' menu item during update?
// TODO: HTML to Feed Generator. https://github.com/RSS-Bridge/rss-bridge
// TODO: SQlite instead of CoreData? https://www.objc.io/issues/4-core-data/SQLite-instead-of-core-data/
/// UTI type used for opml files

View File

@@ -22,9 +22,7 @@
#import "FeedGroup+Ext.h"
#import "Feed+Ext.h"
#import "FeedMeta+Ext.h"
#import "StoreCoordinator.h"
#import "NSDate+Ext.h"
@implementation FeedGroup (Ext)

View File

@@ -0,0 +1,30 @@
//
// The MIT License (MIT)
// Copyright (c) 2019 Oleg Geier
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
@import Cocoa;
// TODO: Make plugins extensible? community extensions.
@interface YouTubePlugin : NSObject
+ (NSString*)feedURL:(NSURL*)url;
+ (NSString*)videoImage:(NSString*)videoid;
+ (NSString*)videoImageHQ:(NSString*)videoid;
@end

View File

@@ -0,0 +1,84 @@
//
// The MIT License (MIT)
// Copyright (c) 2019 Oleg Geier
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
// of the Software, and to permit persons to whom the Software is furnished to do
// so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "Download3rdParty.h"
@implementation YouTubePlugin
/**
Transforms YouTube URL to XML feed URL. @c https://www.youtube.com/{channel|user|playlist]}/{id}
@note
Some YouTube HTML pages contain the 'alternate' tag, others don't.
This method will only be executed, if no other feed url was found.
@return @c nil if @c url is not properly formatted, YouTube feed URL otherwise.
*/
+ (NSString*)feedURL:(NSURL*)url {
if (![url.host hasSuffix:@"youtube.com"]) // 'youtu.be' & 'youtube-nocookie.com' will redirect
return nil;
// https://www.youtube.com/channel/[channel-id]
// https://www.youtube.com/user/[user-name]
// https://www.youtube.com/playlist?list=[playlist-id]
#ifdef DEBUG
printf("resolving YouTube url:\n");
printf(" ↳ %s\n", url.absoluteString.UTF8String);
#endif
NSString *found = nil;
NSArray<NSString*> *parts = url.pathComponents;
if (parts.count > 1) { // first path component is always '/'
static NSString* const ytBase = @"https://www.youtube.com/feeds/videos.xml";
NSString *type = parts[1];
if ([type isEqualToString:@"channel"]) {
if (parts.count > 2)
found = [ytBase stringByAppendingFormat:@"?channel_id=%@", parts[2]];
} else if ([type isEqualToString:@"user"]) {
if (parts.count > 2)
found = [ytBase stringByAppendingFormat:@"?user=%@", parts[2]];
} else if ([type isEqualToString:@"playlist"]) {
NSURLComponents *uc = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO];
for (NSURLQueryItem *q in uc.queryItems) {
if ([q.name isEqualToString:@"list"]) {
found = [ytBase stringByAppendingFormat:@"?playlist_id=%@", q.value];
break;
}
}
}
}
#ifdef DEBUG
printf(" ↳ %s\n", found ? found.UTF8String : "could not resolve!");
#endif
return found; // may be nil
}
/// @return @c http://i.ytimg.com/vi/<videoid>/default.jpg
+ (NSString*)videoImage:(NSString*)videoid {
return [NSString stringWithFormat:@"http://i.ytimg.com/vi/%@/default.jpg", videoid];
}
/// @return @c http://i.ytimg.com/vi/<videoid>/hqdefault.jpg
+ (NSString*)videoImageHQ:(NSString*)videoid {
return [NSString stringWithFormat:@"http://i.ytimg.com/vi/%@/hqdefault.jpg", videoid];
}
@end

View File

@@ -23,8 +23,10 @@
@import RSXML2;
#import "FeedDownload.h"
#import "FaviconDownload.h"
#import "Download3rdParty.h"
#import "Feed+Ext.h"
#import "FeedMeta+Ext.h"
#import "NSError+Ext.h"
#import "NSURLRequest+Ext.h"
@interface FeedDownload()
@@ -122,6 +124,33 @@
[self.currentDownload cancel];
}
/**
Persist in memory object by copying all attributes to permanent core data storage.
@param flag If @c YES then @c FeedGroup won't increase the error count for the feed.
Feed will be scheduled as soon as the user reconnects to the internet.
@return @c YES if downloaded feed contains at least one article. ( @c 304 returns @c NO )
*/
- (BOOL)copyValuesTo:(nonnull Feed*)feed ignoreError:(BOOL)flag {
if (!flag && self.error) // Increase error count and schedule next update.
[feed.meta setErrorAndPostponeSchedule];
else if (self.response) // Update Etag & Last modified and schedule next update.
[feed.meta setSucessfulWithResponse:self.response];
else // Update URL but keep schedule (e.g., error while adding feed should auto-try once reconnected)
[feed.meta setUrlIfChanged:self.request.URL.absoluteString];
// If feed is broken indicate that feed will not be updated
if (!self.xmlfeed || self.xmlfeed.articles.count == 0)
return NO;
// Else: Update stored articles and indicate that feed was updated
[feed updateWithRSS:self.xmlfeed postUnreadCountChange:YES];
return YES;
}
// ---------------------------------------------------------------
// | MARK: - HTML Source Handling
// ---------------------------------------------------------------
/// Take the @c urlStr and run a download @c dataTask: on it. Auto-detect if data is HTML or feed.
- (void)downloadSource:(NSURLRequest*)request {
self.currentDownload = [request dataTask:^(NSData * _Nullable data, NSError * _Nullable error, NSHTTPURLResponse *response) {
@@ -133,9 +162,9 @@
}
RSXMLData *xml = [[RSXMLData alloc] initWithData:data url:response.URL];
if (!self.assertIsFeedURL && [xml.parserClass isHTMLParser])
[self processXMLDataHTML:xml];
[self processXMLDataHTML:xml]; // HTML source handling
else
[self processXMLDataFeed:xml];
[self processXMLDataFeed:xml]; // XML source handling
}];
}
@@ -143,31 +172,40 @@
- (void)processXMLDataHTML:(RSXMLData*)xml {
RSHTMLMetadataParser *parser = [RSHTMLMetadataParser parserWithXMLData:xml];
[parser parseAsync:^(RSHTMLMetadata * _Nullable meta, NSError * _Nullable error) {
NSString *feedURL = nil;
if (error) {
self.error = error;
} else if (!meta || meta.feedLinks.count == 0) {
self.error = RSXMLMakeErrorWrongParser(RSXMLErrorExpectingFeed, RSXMLErrorExpectingHTML, xml.url);
}
else if (!meta || meta.feedLinks.count == 0) {
if ([xml.url.host hasSuffix:@"youtube.com"])
feedURL = [YouTubePlugin feedURL:xml.url];
if (feedURL.length == 0)
self.error = [NSError feedURLNotFound:xml.url];
}
else {
feedURL = meta.feedLinks.firstObject.link;
if (meta.feedLinks.count > 1 && self.respondToSelectFeed)
feedURL = [self.delegate feedDownload:self selectFeedFromList:meta.feedLinks];
if (!feedURL)
self.error = [NSError canceledByUser];
}
// finalize HTML parsing
if (self.error) {
[self finishAndNotify];
} else {
self.faviconURL = [FaviconDownload urlForMetadata:meta]; // we can re-use favicon url if we find one
NSString *chosenURL = meta.feedLinks.firstObject.link;
if (self.respondToSelectFeed && meta.feedLinks.count > 1)
chosenURL = [self.delegate feedDownload:self selectFeedFromList:meta.feedLinks];
if (chosenURL.length > 0) {
self.assertIsFeedURL = YES;
self.faviconURL = [FaviconDownload urlForMetadata:meta]; // re-use favicon url (if present)
// Feeds like https://news.ycombinator.com/ return 503 if URLs are requested too rapidly
//CFRunLoopRunInMode(kCFRunLoopDefaultMode, 1.0, false); // Non-blocking sleep (1s)
[self downloadSource:[NSURLRequest withURL:chosenURL]];
return;
} else { // User canceled operation, show appropriate error message
NSDictionary *info = @{ NSLocalizedDescriptionKey: NSLocalizedString(@"Operation canceled.", nil) };
self.error = [NSError errorWithDomain:NSURLErrorDomain code:NSURLErrorCancelled userInfo:info];
[self downloadSource:[NSURLRequest withURL:feedURL]];
}
}
[self finishAndNotify];
}];
}
// ---------------------------------------------------------------
// | MARK: - XML Source Handling
// ---------------------------------------------------------------
/// The downloaded source seems to be proper feed data, lets parse it with @c RSXML @c RSFeedParser
- (void)processXMLDataFeed:(RSXMLData*)xml {
RSFeedParser *parser = [RSFeedParser parserWithXMLData:xml];
@@ -197,27 +235,4 @@
if (self.block) { self.block(self); self.block = nil; }
}
/**
Persist in memory object by copying all attributes to permanent core data storage.
@param flag If @c YES then @c FeedGroup won't increase the error count for the feed.
Feed will be scheduled as soon as the user reconnects to the internet.
@return @c YES if downloaded feed contains at least one article. ( @c 304 returns @c NO )
*/
- (BOOL)copyValuesTo:(nonnull Feed*)feed ignoreError:(BOOL)flag {
if (!flag && self.error) // Increase error count and schedule next update.
[feed.meta setErrorAndPostponeSchedule];
else if (self.response) // Update Etag & Last modified and schedule next update.
[feed.meta setSucessfulWithResponse:self.response];
else // Update URL but keep schedule (e.g., error while adding feed should auto-try once reconnected)
[feed.meta setUrlIfChanged:self.request.URL.absoluteString];
// If feed is broken indicate that feed will not be updated
if (!self.xmlfeed || self.xmlfeed.articles.count == 0)
return NO;
// Else: Update stored articles and indicate that feed was updated
[feed updateWithRSS:self.xmlfeed postUnreadCountChange:YES];
return YES;
}
@end

View File

@@ -23,5 +23,8 @@
@import Cocoa;
@interface NSError (Ext)
// Generators
+ (instancetype)statusCode:(NSInteger)code reason:(nullable NSString*)reason;
+ (instancetype)canceledByUser;
+ (instancetype)feedURLNotFound:(NSURL*)url;
@end

View File

@@ -20,6 +20,7 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
@import RSXML2.RSXMLError;
#import "NSError+Ext.h"
@implementation NSError (Ext)
@@ -98,6 +99,10 @@ static const char* CodeDescription(NSInteger code) {
return "Unknown";
}
// ---------------------------------------------------------------
// | MARK: - Generators
// ---------------------------------------------------------------
/// Generate @c NSError from HTTP status code. E.g., @c code @c = @c 404 will return "404 Not Found".
+ (instancetype)statusCode:(NSInteger)code reason:(nullable NSString*)reason {
NSMutableDictionary *info = [NSMutableDictionary dictionaryWithCapacity:2];
@@ -107,7 +112,18 @@ static const char* CodeDescription(NSInteger code) {
NSInteger errCode = NSURLErrorUnknown;
if (code < 500) { if (code >= 400) errCode = NSURLErrorResourceUnavailable; }
else if (code < 600) errCode = NSURLErrorBadServerResponse;
return [NSError errorWithDomain:NSURLErrorDomain code:errCode userInfo:info];
return [self errorWithDomain:NSURLErrorDomain code:errCode userInfo:info];
}
/// Generate @c NSError for user canceled operation. With title "Operation canceled.".
+ (instancetype)canceledByUser {
NSDictionary *info = @{ NSLocalizedDescriptionKey: NSLocalizedString(@"Operation canceled.", nil) };
return [self errorWithDomain:NSURLErrorDomain code:NSURLErrorCancelled userInfo:info];
}
/// Generate @c NSError for webpages that don't contain feed urls.
+ (instancetype)feedURLNotFound:(NSURL*)url {
return RSXMLMakeErrorWrongParser(RSXMLErrorExpectingFeed, RSXMLErrorExpectingHTML, url);
}
@end

View File

@@ -70,7 +70,7 @@
</dict>
</array>
<key>CFBundleVersion</key>
<string>13207</string>
<string>13404</string>
<key>LSApplicationCategoryType</key>
<string>public.app-category.news</string>
<key>LSMinimumSystemVersion</key>

View File

@@ -23,7 +23,6 @@
#import "SettingsFeedsView.h"
#import "StoreCoordinator.h"
#import "FeedGroup+Ext.h"
#import "FeedMeta+Ext.h"
#import "DrawImage.h"
#import "SettingsFeeds.h"
#import "NSDate+Ext.h"