From 6b903389329b99150302e6a22ce85d4b7408ae76 Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 13:13:57 -0400 Subject: [PATCH 1/8] [ie/crunchyroll] Make aditional audio languages available --- yt_dlp/extractor/crunchyroll.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index 385a3c2d3..6784609de 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -175,17 +175,41 @@ class CrunchyrollBaseIE(InfoExtractor): display_id = identifier self._update_auth() + stream_url = 'https://cr-play-service.prd.crunchyrollsvc.com/v1/{}/console/switch/play' stream_response = self._download_json( - f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play', + stream_url.format(identifier), display_id, note='Downloading stream info', errnote='Failed to download stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS) + default_audio_locale = traverse_obj(stream_response, ('audioLocale', {str})) + + available_formats = {} + stream_variants = {} + + variant_obj_check = lambda _, v: v['guid'] and v['audio_locale'] + for audio_locale_variant in traverse_obj(stream_response, ('versions', variant_obj_check)): + locale = audio_locale_variant['audio_locale'] + is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default = False) + + stream_variants[locale] = self._download_json( + stream_url.format(audio_locale_variant['guid']), + display_id, note=f'Downloading {locale} stream info', errnote=f'Failed to download {locale} stream info', + headers=CrunchyrollBaseIE._AUTH_HEADERS) + + if is_original_stream: + stream_variants['original'] = stream_variants[locale] + + available_formats[f'audio_{locale}'] = (f'audio-{locale}', '', locale, stream_variants['original']['url']) + + if not stream_variants: + stream_variants['original'] = stream_response + available_formats['original'] = ('original', '', default_audio_locale, stream_variants['original']['url']) + - available_formats = {'': ('', '', stream_response['url'])} for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): - available_formats[hardsub_lang] = (f'hardsub-{hardsub_lang}', hardsub_lang, stream['url']) + available_formats[f'hardsub_{hardsub_lang}'] = (f'hardsub-{hardsub_lang}', hardsub_lang, default_audio_locale, stream['url']) requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] - hardsub_langs = [lang for lang in available_formats if lang] + hardsub_langs = [lang.replace('hardsub_', '') for lang in available_formats if lang.startswith('hardsub_')] if hardsub_langs and 'all' not in requested_hardsubs: full_format_langs = set(requested_hardsubs) self.to_screen(f'Available hardsub languages: {", ".join(hardsub_langs)}') @@ -197,14 +221,13 @@ class CrunchyrollBaseIE(InfoExtractor): else: full_format_langs = set(map(str.lower, available_formats)) - audio_locale = traverse_obj(stream_response, ('audioLocale', {str})) hardsub_preference = qualities(requested_hardsubs[::-1]) formats, subtitles = [], {} - for format_id, hardsub_lang, stream_url in available_formats.values(): + for format_id, hardsub_lang, audio_locale, stream_url in available_formats.values(): if hardsub_lang.lower() in full_format_langs: adaptive_formats, dash_subs = self._extract_mpd_formats_and_subtitles( stream_url, display_id, mpd_id=format_id, headers=CrunchyrollBaseIE._AUTH_HEADERS, - fatal=False, note=f'Downloading {f"{format_id} " if hardsub_lang else ""}MPD manifest') + fatal=False, note=f'Downloading {format_id} MPD manifest') self._merge_subtitles(dash_subs, target=subtitles) else: continue # XXX: Update this if/when meta mpd formats are working From 1657011c9bbf5ec10bc45a712f2e6dbf8d065bd3 Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 13:42:01 -0400 Subject: [PATCH 2/8] [ie/cruncyroll] Remove unused key --- yt_dlp/extractor/crunchyroll.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index 29aa446b8..c50686a04 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -188,21 +188,16 @@ class CrunchyrollBaseIE(InfoExtractor): variant_obj_check = lambda _, v: v['guid'] and v['audio_locale'] for audio_locale_variant in traverse_obj(stream_response, ('versions', variant_obj_check)): locale = audio_locale_variant['audio_locale'] - is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default = False) stream_variants[locale] = self._download_json( stream_url.format(audio_locale_variant['guid']), display_id, note=f'Downloading {locale} stream info', errnote=f'Failed to download {locale} stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS) - if is_original_stream: - stream_variants['original'] = stream_variants[locale] - - available_formats[f'audio_{locale}'] = (f'audio-{locale}', '', locale, stream_variants['original']['url']) + available_formats[f'audio_{locale}'] = (f'audio-{locale}', '', locale, stream_variants[locale]['url']) if not stream_variants: - stream_variants['original'] = stream_response - available_formats['original'] = ('original', '', default_audio_locale, stream_variants['original']['url']) + available_formats['default'] = ('default', '', default_audio_locale, stream_response) for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): From 5ed8cd7b11a037821883f34aa7defbf75220987a Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 14:17:50 -0400 Subject: [PATCH 3/8] [ie/crunchyroll] Don't use 'default' as the default stream format key It's incompatible with the hardsub code --- yt_dlp/extractor/crunchyroll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index c50686a04..be97e11d5 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -197,7 +197,7 @@ class CrunchyrollBaseIE(InfoExtractor): available_formats[f'audio_{locale}'] = (f'audio-{locale}', '', locale, stream_variants[locale]['url']) if not stream_variants: - available_formats['default'] = ('default', '', default_audio_locale, stream_response) + available_formats[''] = ('', '', default_audio_locale, stream_response['url']) for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): From ab06f632af36f032e289af19c5181adcf2039dbe Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 15:16:48 -0400 Subject: [PATCH 4/8] [ie/crunchyroll] Skip video on audio format variants --- yt_dlp/extractor/crunchyroll.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index be97e11d5..b13a823c5 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -188,20 +188,21 @@ class CrunchyrollBaseIE(InfoExtractor): variant_obj_check = lambda _, v: v['guid'] and v['audio_locale'] for audio_locale_variant in traverse_obj(stream_response, ('versions', variant_obj_check)): locale = audio_locale_variant['audio_locale'] + is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default = False) stream_variants[locale] = self._download_json( stream_url.format(audio_locale_variant['guid']), display_id, note=f'Downloading {locale} stream info', errnote=f'Failed to download {locale} stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS) - available_formats[f'audio_{locale}'] = (f'audio-{locale}', '', locale, stream_variants[locale]['url']) + available_formats[f'audio_{locale}'] = (f'{locale}', '', locale, stream_variants[locale]['url'], is_original_stream) if not stream_variants: - available_formats[''] = ('', '', default_audio_locale, stream_response['url']) + available_formats[''] = ('', '', default_audio_locale, stream_response['url'], True) for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): - available_formats[f'hardsub_{hardsub_lang}'] = (f'hardsub-{hardsub_lang}', hardsub_lang, default_audio_locale, stream['url']) + available_formats[f'hardsub_{hardsub_lang}'] = (f'hardsub-{hardsub_lang}', hardsub_lang, default_audio_locale, stream['url'], True) requested_hardsubs = [('' if val == 'none' else val) for val in (self._configuration_arg('hardsub') or ['none'])] hardsub_langs = [lang.replace('hardsub_', '') for lang in available_formats if lang.startswith('hardsub_')] @@ -218,7 +219,7 @@ class CrunchyrollBaseIE(InfoExtractor): hardsub_preference = qualities(requested_hardsubs[::-1]) formats, subtitles = [], {} - for format_id, hardsub_lang, audio_locale, stream_url in available_formats.values(): + for format_id, hardsub_lang, audio_locale, stream_url, is_original in available_formats.values(): if hardsub_lang.lower() in full_format_langs: adaptive_formats, dash_subs = self._extract_mpd_formats_and_subtitles( stream_url, display_id, mpd_id=format_id, headers=CrunchyrollBaseIE._AUTH_HEADERS, @@ -229,8 +230,10 @@ class CrunchyrollBaseIE(InfoExtractor): for f in adaptive_formats: if f.get('acodec') != 'none': f['language'] = audio_locale + elif not is_original: + continue f['quality'] = hardsub_preference(hardsub_lang.lower()) - formats.extend(adaptive_formats) + formats.append(f) for locale, subtitle in traverse_obj(stream_response, (('subtitles', 'captions'), {dict.items}, ...)): subtitles.setdefault(locale, []).append(traverse_obj(subtitle, {'url': 'url', 'ext': 'format'})) From 4df78bbfde33ac019c309208547667575cfc4ea6 Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 15:47:30 -0400 Subject: [PATCH 5/8] [ie/crunchyroll] Remove unused dict --- yt_dlp/extractor/crunchyroll.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index b13a823c5..53393adf6 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -183,22 +183,21 @@ class CrunchyrollBaseIE(InfoExtractor): default_audio_locale = traverse_obj(stream_response, ('audioLocale', {str})) available_formats = {} - stream_variants = {} variant_obj_check = lambda _, v: v['guid'] and v['audio_locale'] for audio_locale_variant in traverse_obj(stream_response, ('versions', variant_obj_check)): locale = audio_locale_variant['audio_locale'] is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default = False) - stream_variants[locale] = self._download_json( + stream_variant = self._download_json( stream_url.format(audio_locale_variant['guid']), display_id, note=f'Downloading {locale} stream info', errnote=f'Failed to download {locale} stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS) - available_formats[f'audio_{locale}'] = (f'{locale}', '', locale, stream_variants[locale]['url'], is_original_stream) + available_formats[f'audio_{locale}'] = (f'{locale}', '', locale, stream_variant.get('url'), is_original_stream) - if not stream_variants: - available_formats[''] = ('', '', default_audio_locale, stream_response['url'], True) + if not available_formats: + available_formats[''] = ('', '', default_audio_locale, stream_response.get('url'), True) for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): From 51ccd78ead24197a08e2621ffbba1b4b13c1750d Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 15:50:50 -0400 Subject: [PATCH 6/8] [ie/crunchyroll] Use flake8 --- yt_dlp/extractor/crunchyroll.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index 53393adf6..6730355a0 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -187,19 +187,18 @@ class CrunchyrollBaseIE(InfoExtractor): variant_obj_check = lambda _, v: v['guid'] and v['audio_locale'] for audio_locale_variant in traverse_obj(stream_response, ('versions', variant_obj_check)): locale = audio_locale_variant['audio_locale'] - is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default = False) - + is_original_stream = traverse_obj(audio_locale_variant, ('original', {bool}), default=False) + stream_variant = self._download_json( stream_url.format(audio_locale_variant['guid']), display_id, note=f'Downloading {locale} stream info', errnote=f'Failed to download {locale} stream info', headers=CrunchyrollBaseIE._AUTH_HEADERS) - + available_formats[f'audio_{locale}'] = (f'{locale}', '', locale, stream_variant.get('url'), is_original_stream) if not available_formats: available_formats[''] = ('', '', default_audio_locale, stream_response.get('url'), True) - for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])): available_formats[f'hardsub_{hardsub_lang}'] = (f'hardsub-{hardsub_lang}', hardsub_lang, default_audio_locale, stream['url'], True) From 698191c04517ed06a96768b325a97d6c44f25a02 Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 20:03:47 -0400 Subject: [PATCH 7/8] [ie/crunchyroll] Set original language as default --- yt_dlp/extractor/crunchyroll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index 6730355a0..61a721590 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -230,7 +230,7 @@ class CrunchyrollBaseIE(InfoExtractor): f['language'] = audio_locale elif not is_original: continue - f['quality'] = hardsub_preference(hardsub_lang.lower()) + f['quality'] = hardsub_preference(hardsub_lang.lower()) + 1 if is_original else 0 formats.append(f) for locale, subtitle in traverse_obj(stream_response, (('subtitles', 'captions'), {dict.items}, ...)): From 342f2e44ac1834c090e7287ff0ad59e4d1e323c4 Mon Sep 17 00:00:00 2001 From: Oliver Hattshire Date: Sun, 5 May 2024 20:07:01 -0400 Subject: [PATCH 8/8] [ie/crunchyroll] Use hardsub_lang instead of format_id hardsub format filtering --- yt_dlp/extractor/crunchyroll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py index 61a721590..c08c05742 100644 --- a/yt_dlp/extractor/crunchyroll.py +++ b/yt_dlp/extractor/crunchyroll.py @@ -213,7 +213,7 @@ class CrunchyrollBaseIE(InfoExtractor): 'See https://github.com/yt-dlp/yt-dlp#crunchyrollbeta-crunchyroll for more info', only_once=True) else: - full_format_langs = set(map(str.lower, available_formats)) + full_format_langs = set(map(lambda v: str.lower(v[1]), available_formats.values())) hardsub_preference = qualities(requested_hardsubs[::-1]) formats, subtitles = [], {}