From 6e6e0d95b32029b0d14843ffacc18305555e2230 Mon Sep 17 00:00:00 2001 From: shirt <2660574+shirt-dev@users.noreply.github.com> Date: Mon, 9 Aug 2021 16:24:50 -0400 Subject: [PATCH] [paramountplus] Separate extractor and fix some titles (#652) Co-authored-by: shirt, pukkandan --- yt_dlp/extractor/cbs.py | 185 ++++++++++-------------------- yt_dlp/extractor/extractors.py | 9 +- yt_dlp/extractor/paramountplus.py | 145 +++++++++++++++++++++++ 3 files changed, 208 insertions(+), 131 deletions(-) create mode 100644 yt_dlp/extractor/paramountplus.py diff --git a/yt_dlp/extractor/cbs.py b/yt_dlp/extractor/cbs.py index fbbbe5545..ed5dc84a7 100644 --- a/yt_dlp/extractor/cbs.py +++ b/yt_dlp/extractor/cbs.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from .common import InfoExtractor from .theplatform import ThePlatformFeedIE from ..utils import ( ExtractorError, @@ -9,6 +8,7 @@ from ..utils import ( xpath_element, xpath_text, update_url_query, + url_or_none, ) @@ -26,16 +26,62 @@ class CBSBaseIE(ThePlatformFeedIE): }) return subtitles + def _extract_common_video_info(self, content_id, asset_types, mpx_acc, extra_info): + tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id) + tp_release_url = f'https://link.theplatform.com/s/{tp_path}' + info = self._extract_theplatform_metadata(tp_path, content_id) + + formats, subtitles = [], {} + last_e = None + for asset_type, query in asset_types.items(): + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + update_url_query(tp_release_url, query), content_id, + 'Downloading %s SMIL data' % asset_type) + except ExtractorError as e: + last_e = e + if asset_type != 'fallback': + continue + query['formats'] = '' # blank query to check if expired + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + update_url_query(tp_release_url, query), content_id, + 'Downloading %s SMIL data, trying again with another format' % asset_type) + except ExtractorError as e: + last_e = e + continue + formats.extend(tp_formats) + subtitles = self._merge_subtitles(subtitles, tp_subtitles) + if last_e and not formats: + self.raise_no_formats(last_e, True, content_id) + self._sort_formats(formats) + + extra_info.update({ + 'id': content_id, + 'formats': formats, + 'subtitles': subtitles, + }) + info.update({k: v for k, v in extra_info.items() if v is not None}) + return info + + def _extract_video_info(self, *args, **kwargs): + # Extract assets + metadata and call _extract_common_video_info + raise NotImplementedError('This method must be implemented by subclasses') + + def _real_extract(self, url): + return self._extract_video_info(self._match_id(url)) + class CBSIE(CBSBaseIE): _VALID_URL = r'''(?x) (?: cbs:| https?://(?:www\.)?(?: - (?:cbs|paramountplus)\.com/(?:shows/[^/]+/video|movies/[^/]+)/| + cbs\.com/(?:shows/[^/]+/video|movies/[^/]+)/| colbertlateshow\.com/(?:video|podcasts)/) )(?P[\w-]+)''' + # All tests are blocked outside US _TESTS = [{ 'url': 'https://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', 'info_dict': { @@ -52,39 +98,6 @@ class CBSIE(CBSBaseIE): # m3u8 download 'skip_download': True, }, - '_skip': 'Blocked outside the US', - }, { - 'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/', - 'info_dict': { - 'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k', - 'ext': 'mp4', - 'title': 'CatDog - Climb Every CatDog/The Canine Mutiny', - 'description': 'md5:7ac835000645a69933df226940e3c859', - 'duration': 1418, - 'timestamp': 920264400, - 'upload_date': '19990301', - 'uploader': 'CBSI-NEW', - }, - 'params': { - 'skip_download': 'm3u8', - }, - '_skip': 'Blocked outside the US', - }, { - 'url': 'https://www.paramountplus.com/shows/tooning-out-the-news/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/7-23-21-week-in-review-rep-jahana-hayes-howard-fineman-sen-michael-bennet-sheera-frenkel-cecilia-kang-/', - 'info_dict': { - 'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd', - 'ext': 'mp4', - 'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)', - 'description': 'md5:f4adcea3e8b106192022e121f1565bae', - 'duration': 2506, - 'timestamp': 1627063200, - 'upload_date': '20210723', - 'uploader': 'CBSI-NEW', - }, - 'params': { - 'skip_download': 'm3u8', - }, - '_skip': 'Blocked outside the US', }, { 'url': 'https://www.cbs.com/shows/the-late-show-with-stephen-colbert/video/60icOhMb9NcjbcWnF_gub9XXHdeBcNk2/the-late-show-6-23-21-christine-baranski-joy-oladokun-', 'info_dict': { @@ -107,12 +120,6 @@ class CBSIE(CBSBaseIE): }, { 'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/', 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/', - 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/movies/million-dollar-american-princesses-meghan-and-harry/C0LpgNwXYeB8txxycdWdR9TjxpJOsdCq', - 'only_matching': True, }] def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517): @@ -121,14 +128,8 @@ class CBSIE(CBSBaseIE): content_id, query={'partner': site, 'contentId': content_id}) video_data = xpath_element(items_data, './/item') title = xpath_text(video_data, 'videoTitle', 'title') or xpath_text(video_data, 'videotitle', 'title') - tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id) - tp_release_url = 'https://link.theplatform.com/s/' + tp_path - asset_types = [] - subtitles = {} - formats = [] - useXMLmetadata = True - last_e = None + asset_types = {} for item in items_data.findall('.//item'): asset_type = xpath_text(item, 'assetType') query = { @@ -137,94 +138,24 @@ class CBSIE(CBSBaseIE): } if not asset_type: # fallback for content_ids that videoPlayerService doesn't return anything for - useXMLmetadata = False asset_type = 'fallback' query['formats'] = 'M3U+none,MPEG4,M3U+appleHlsEncryption,MP3' del query['assetTypes'] - elif asset_type in asset_types: + if asset_type in asset_types: continue elif any(excluded in asset_type for excluded in ('HLS_FPS', 'DASH_CENC', 'OnceURL')): continue - asset_types.append(asset_type) if asset_type.startswith('HLS') or 'StreamPack' in asset_type: query['formats'] = 'MPEG4,M3U' elif asset_type in ('RTMP', 'WIFI', '3G'): query['formats'] = 'MPEG4,FLV' - try: - tp_formats, tp_subtitles = self._extract_theplatform_smil( - update_url_query(tp_release_url, query), content_id, - 'Downloading %s SMIL data' % asset_type) - except ExtractorError as e: - last_e = e - if useXMLmetadata: - continue - query['formats'] = '' # blank query to check if expired - try: - tp_formats, tp_subtitles = self._extract_theplatform_smil( - update_url_query(tp_release_url, query), content_id, - 'Downloading %s SMIL data, trying again with another format' % asset_type) - except ExtractorError as e: - last_e = e - continue - formats.extend(tp_formats) - subtitles = self._merge_subtitles(subtitles, tp_subtitles) - if last_e and not formats: - self.raise_no_formats(last_e, True, content_id) - self._sort_formats(formats) + asset_types[asset_type] = query - info = self._extract_theplatform_metadata(tp_path, content_id) - info.update({ - 'formats': formats, - 'subtitles': subtitles, - 'id': content_id + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ + 'title': title, + 'series': xpath_text(video_data, 'seriesTitle'), + 'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')), + 'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')), + 'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000), + 'thumbnail': url_or_none(xpath_text(video_data, 'previewImageURL')), }) - if useXMLmetadata: - info.update({ - 'title': title, - 'series': xpath_text(video_data, 'seriesTitle'), - 'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')), - 'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')), - 'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000), - 'thumbnail': xpath_text(video_data, 'previewImageURL') - }) - return info - - def _real_extract(self, url): - content_id = self._match_id(url) - return self._extract_video_info(content_id) - - -class ParamountPlusSeriesIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P[a-zA-Z0-9-_]+)/?(?:[#?]|$)' - _TESTS = [{ - 'url': 'https://www.paramountplus.com/shows/drake-josh', - 'playlist_mincount': 50, - 'info_dict': { - 'id': 'drake-josh', - } - }, { - 'url': 'https://www.paramountplus.com/shows/hawaii_five_0/', - 'playlist_mincount': 240, - 'info_dict': { - 'id': 'hawaii_five_0', - } - }, { - 'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/', - 'playlist_mincount': 248, - 'info_dict': { - 'id': 'spongebob-squarepants', - } - }] - _API_URL = 'https://www.paramountplus.com/shows/{}/xhr/episodes/page/0/size/100000/xs/0/season/0/' - - def _entries(self, show_name): - show_json = self._download_json(self._API_URL.format(show_name), video_id=show_name) - if show_json.get('success'): - for episode in show_json['result']['data']: - yield self.url_result( - 'https://www.paramountplus.com%s' % episode['url'], - ie=CBSIE.ie_key(), video_id=episode['content_id']) - - def _real_extract(self, url): - show_name = self._match_id(url) - return self.playlist_result(self._entries(show_name), playlist_id=show_name) diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py index f803c6765..df0d83359 100644 --- a/yt_dlp/extractor/extractors.py +++ b/yt_dlp/extractor/extractors.py @@ -202,10 +202,7 @@ from .cbc import ( CBCWatchIE, CBCOlympicsIE, ) -from .cbs import ( - CBSIE, - ParamountPlusSeriesIE, -) +from .cbs import CBSIE from .cbslocal import ( CBSLocalIE, CBSLocalArticleIE, @@ -969,6 +966,10 @@ from .palcomp3 import ( PalcoMP3VideoIE, ) from .pandoratv import PandoraTVIE +from .paramountplus import ( + ParamountPlusIE, + ParamountPlusSeriesIE, +) from .parliamentliveuk import ParliamentLiveUKIE from .parlview import ParlviewIE from .patreon import PatreonIE diff --git a/yt_dlp/extractor/paramountplus.py b/yt_dlp/extractor/paramountplus.py new file mode 100644 index 000000000..e1d479588 --- /dev/null +++ b/yt_dlp/extractor/paramountplus.py @@ -0,0 +1,145 @@ +from __future__ import unicode_literals + +from .common import InfoExtractor +from .cbs import CBSBaseIE +from ..utils import ( + int_or_none, + url_or_none, +) + + +class ParamountPlusIE(CBSBaseIE): + _VALID_URL = r'''(?x) + (?: + paramountplus:| + https?://(?:www\.)?(?: + paramountplus\.com/(?:shows/[^/]+/video|movies/[^/]+)/ + )(?P[\w-]+))''' + + # All tests are blocked outside US + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/', + 'info_dict': { + 'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k', + 'ext': 'mp4', + 'title': 'CatDog - Climb Every CatDog/The Canine Mutiny', + 'description': 'md5:7ac835000645a69933df226940e3c859', + 'duration': 1418, + 'timestamp': 920264400, + 'upload_date': '19990301', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/shows/tooning-out-the-news/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/7-23-21-week-in-review-rep-jahana-hayes-howard-fineman-sen-michael-bennet-sheera-frenkel-cecilia-kang-/', + 'info_dict': { + 'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd', + 'ext': 'mp4', + 'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)', + 'description': 'md5:f4adcea3e8b106192022e121f1565bae', + 'duration': 2506, + 'timestamp': 1627063200, + 'upload_date': '20210723', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/movies/daddys-home/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', + 'info_dict': { + 'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', + 'ext': 'mp4', + 'title': 'Daddy\'s Home', + 'upload_date': '20151225', + 'description': 'md5:a0beaf24e8d3b0e81b2ee41d47c06f33', + 'uploader': 'CBSI-NEW', + 'timestamp': 1451030400, + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], # TODO: Investigate this + }, { + 'url': 'https://www.paramountplus.com/movies/sonic-the-hedgehog/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', + 'info_dict': { + 'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', + 'ext': 'mp4', + 'uploader': 'CBSI-NEW', + 'description': 'md5:bc7b6fea84ba631ef77a9bda9f2ff911', + 'timestamp': 1577865600, + 'title': 'Sonic the Hedgehog', + 'upload_date': '20200101', + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], + }, { + 'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/', + 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com/movies/million-dollar-american-princesses-meghan-and-harry/C0LpgNwXYeB8txxycdWdR9TjxpJOsdCq', + 'only_matching': True, + }] + + def _extract_video_info(self, content_id, mpx_acc=2198311517): + items_data = self._download_json( + 'https://www.paramountplus.com/apps-api/v2.0/androidtv/video/cid/%s.json' % content_id, + content_id, query={'locale': 'en-us', 'at': 'ABCqWNNSwhIqINWIIAG+DFzcFUvF8/vcN6cNyXFFfNzWAIvXuoVgX+fK4naOC7V8MLI='}) + + asset_types = { + item.get('assetType'): { + 'format': 'SMIL', + 'formats': 'MPEG4,M3U', + } for item in items_data['itemList'] + } + item = items_data['itemList'][-1] + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ + 'title': item.get('title'), + 'series': item.get('seriesTitle'), + 'season_number': int_or_none(item.get('seasonNum')), + 'episode_number': int_or_none(item.get('episodeNum')), + 'duration': int_or_none(item.get('duration')), + 'thumbnail': url_or_none(item.get('thumbnail')), + }) + + +class ParamountPlusSeriesIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P[a-zA-Z0-9-_]+)/?(?:[#?]|$)' + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/drake-josh', + 'playlist_mincount': 50, + 'info_dict': { + 'id': 'drake-josh', + } + }, { + 'url': 'https://www.paramountplus.com/shows/hawaii_five_0/', + 'playlist_mincount': 240, + 'info_dict': { + 'id': 'hawaii_five_0', + } + }, { + 'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/', + 'playlist_mincount': 248, + 'info_dict': { + 'id': 'spongebob-squarepants', + } + }] + _API_URL = 'https://www.paramountplus.com/shows/{}/xhr/episodes/page/0/size/100000/xs/0/season/0/' + + def _entries(self, show_name): + show_json = self._download_json(self._API_URL.format(show_name), video_id=show_name) + if show_json.get('success'): + for episode in show_json['result']['data']: + yield self.url_result( + 'https://www.paramountplus.com%s' % episode['url'], + ie=ParamountPlusIE.ie_key(), video_id=episode['content_id']) + + def _real_extract(self, url): + show_name = self._match_id(url) + return self.playlist_result(self._entries(show_name), playlist_id=show_name)