2013-06-23 13:58:33 -04:00
# coding: utf-8
2014-09-13 01:51:06 -04:00
from __future__ import unicode_literals
2021-04-07 07:37:43 -04:00
import calendar
2021-06-29 18:07:49 -04:00
import copy
2021-03-03 05:32:40 -05:00
import hashlib
2013-09-22 04:37:23 -04:00
import itertools
2013-06-23 13:58:33 -04:00
import json
2013-09-21 18:35:03 -04:00
import os . path
2016-02-29 14:01:33 -05:00
import random
2013-06-23 13:58:33 -04:00
import re
2021-02-04 09:57:26 -05:00
import time
2013-09-21 08:19:30 -04:00
import traceback
2013-06-23 13:58:33 -04:00
2013-06-23 14:28:15 -04:00
from . common import InfoExtractor , SearchInfoExtractor
2014-12-11 04:08:17 -05:00
from . . compat import (
2013-09-22 04:30:02 -04:00
compat_chr ,
2021-01-01 07:26:37 -05:00
compat_HTTPError ,
2013-06-23 13:58:33 -04:00
compat_parse_qs ,
2021-02-04 09:37:17 -05:00
compat_str ,
2015-07-17 13:51:57 -04:00
compat_urllib_parse_unquote_plus ,
2016-03-25 15:46:57 -04:00
compat_urllib_parse_urlencode ,
2015-07-20 15:10:28 -04:00
compat_urllib_parse_urlparse ,
2013-10-01 11:58:13 -04:00
compat_urlparse ,
2014-12-11 04:08:17 -05:00
)
2021-02-04 09:37:17 -05:00
from . . jsinterp import JSInterpreter
2014-12-11 04:08:17 -05:00
from . . utils import (
2021-03-21 17:23:34 -04:00
bool_or_none ,
2013-06-23 13:58:33 -04:00
clean_html ,
2021-03-07 00:27:45 -05:00
dict_get ,
2021-04-07 07:37:43 -04:00
datetime_from_str ,
2021-04-16 18:39:08 -04:00
error_to_compat_str ,
2013-06-23 13:58:33 -04:00
ExtractorError ,
2021-02-02 11:21:32 -05:00
format_field ,
2015-02-11 12:39:31 -05:00
float_or_none ,
2014-01-18 23:47:20 -05:00
int_or_none ,
2016-01-24 12:02:19 -05:00
mimetype2ext ,
2017-02-12 06:09:53 -05:00
parse_codecs ,
2015-07-20 15:10:28 -04:00
parse_duration ,
2021-02-18 13:12:56 -05:00
qualities ,
2017-05-06 17:19:11 -04:00
remove_start ,
2015-07-25 11:30:34 -04:00
smuggle_url ,
2018-11-02 19:26:16 -04:00
str_or_none ,
2015-06-28 14:48:06 -04:00
str_to_int ,
2017-01-26 09:43:14 -05:00
try_get ,
2013-06-23 13:58:33 -04:00
unescapeHTML ,
unified_strdate ,
2015-07-25 11:30:34 -04:00
unsmuggle_url ,
2020-11-19 14:22:59 -05:00
update_url_query ,
2018-12-16 07:35:48 -05:00
url_or_none ,
2016-03-25 16:19:24 -04:00
urlencode_postdata ,
2021-04-07 07:37:43 -04:00
urljoin
2013-06-23 13:58:33 -04:00
)
2014-11-23 14:41:03 -05:00
2021-04-16 23:02:33 -04:00
def parse_qs ( url ) :
return compat_urlparse . parse_qs ( compat_urlparse . urlparse ( url ) . query )
2013-09-11 09:48:23 -04:00
class YoutubeBaseInfoExtractor ( InfoExtractor ) :
2013-07-24 14:40:12 -04:00
""" Provide base functions for Youtube extractors """
_LOGIN_URL = ' https://accounts.google.com/ServiceLogin '
2015-08-13 23:11:11 -04:00
_TWOFACTOR_URL = ' https://accounts.google.com/signin/challenge '
2017-05-06 12:58:47 -04:00
_LOOKUP_URL = ' https://accounts.google.com/_/signin/sl/lookup '
2017-05-06 17:19:11 -04:00
_CHALLENGE_URL = ' https://accounts.google.com/_/signin/sl/challenge '
_TFA_URL = ' https://accounts.google.com/_/signin/challenge?hl=en&TL= {0} '
2017-05-06 12:58:47 -04:00
2020-11-21 17:21:09 -05:00
_RESERVED_NAMES = (
2021-06-05 08:39:51 -04:00
r ' channel|c|user|browse|playlist|watch|w|v|embed|e|watch_popup|shorts| '
2021-05-28 14:19:26 -04:00
r ' movies|results|shared|hashtag|trending|feed|feeds|oembed|get_video_info| '
2021-03-04 13:05:26 -05:00
r ' storefront|oops|index|account|reporthistory|t/terms|about|upload|signin|logout ' )
2020-11-21 17:21:09 -05:00
2013-07-24 14:40:12 -04:00
_NETRC_MACHINE = ' youtube '
# If True it will raise an error if no login info is provided
_LOGIN_REQUIRED = False
2020-11-23 16:47:42 -05:00
_PLAYLIST_ID_RE = r ' (?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_] { 10,}|RDMM|WL|LL|LM) '
2017-03-24 14:17:17 -04:00
2013-07-24 14:40:12 -04:00
def _login ( self ) :
2014-08-16 17:28:41 -04:00
"""
Attempt to log in to YouTube .
True is returned if successful or skipped .
False is returned if login failed .
If _LOGIN_REQUIRED is set and no authentication was provided , an error is raised .
"""
2021-05-19 09:41:44 -04:00
def warn ( message ) :
self . report_warning ( message )
# username+password login is broken
if self . _LOGIN_REQUIRED and self . get_param ( ' cookiefile ' ) is None :
self . raise_login_required (
' Login details are needed to download this content ' , method = ' cookies ' )
2018-05-26 11:12:44 -04:00
username , password = self . _get_login_info ( )
2021-05-19 09:41:44 -04:00
if username :
warn ( ' Logging in using username and password is broken. %s ' % self . _LOGIN_HINTS [ ' cookies ' ] )
return
# Everything below this is broken!
2013-07-24 14:40:12 -04:00
# No authentication to be performed
if username is None :
2021-05-17 08:23:08 -04:00
if self . _LOGIN_REQUIRED and self . get_param ( ' cookiefile ' ) is None :
2014-09-24 03:51:45 -04:00
raise ExtractorError ( ' No login info available, needed for using %s . ' % self . IE_NAME , expected = True )
2021-05-17 08:23:08 -04:00
# if self.get_param('cookiefile'): # TODO remove 'and False' later - too many people using outdated cookies and open issues, remind them.
2021-02-04 09:37:17 -05:00
# self.to_screen('[Cookies] Reminder - Make sure to always use up to date cookies!')
2014-08-16 17:28:41 -04:00
return True
2013-07-24 14:40:12 -04:00
2013-12-08 19:49:01 -05:00
login_page = self . _download_webpage (
self . _LOGIN_URL , None ,
2014-09-24 03:51:45 -04:00
note = ' Downloading login page ' ,
errnote = ' unable to fetch login page ' , fatal = False )
2013-12-08 19:49:01 -05:00
if login_page is False :
return
2013-07-24 14:40:12 -04:00
2016-08-23 13:25:21 -04:00
login_form = self . _hidden_inputs ( login_page )
2013-06-23 13:58:33 -04:00
2017-05-06 12:58:47 -04:00
def req ( url , f_req , note , errnote ) :
data = login_form . copy ( )
data . update ( {
' pstMsg ' : 1 ,
' checkConnection ' : ' youtube ' ,
' checkedDomains ' : ' youtube ' ,
' hl ' : ' en ' ,
' deviceinfo ' : ' [null,null,null,[],null, " US " ,null,null,[], " GlifWebSignIn " ,null,[null,null,[]]] ' ,
2017-05-06 17:19:11 -04:00
' f.req ' : json . dumps ( f_req ) ,
2017-05-06 12:58:47 -04:00
' flowName ' : ' GlifWebSignIn ' ,
' flowEntry ' : ' ServiceLogin ' ,
2019-07-11 15:26:05 -04:00
# TODO: reverse actual botguard identifier generation algo
' bgRequest ' : ' [ " identifier " , " " ] ' ,
2015-08-15 12:03:43 -04:00
} )
2017-05-06 12:58:47 -04:00
return self . _download_json (
url , None , note = note , errnote = errnote ,
transform_source = lambda s : re . sub ( r ' ^[^[]* ' , ' ' , s ) ,
fatal = False ,
data = urlencode_postdata ( data ) , headers = {
' Content-Type ' : ' application/x-www-form-urlencoded;charset=utf-8 ' ,
' Google-Accounts-XSRF ' : 1 ,
} )
2017-05-06 17:19:11 -04:00
lookup_req = [
username ,
None , [ ] , None , ' US ' , None , None , 2 , False , True ,
[
None , None ,
[ 2 , 1 , None , 1 ,
' https://accounts.google.com/ServiceLogin?passive=true&continue=https % 3A %2F %2F www.youtube.com %2F signin %3F next % 3D %252F %26a ction_handle_signin % 3Dtrue % 26hl % 3Den %26a pp % 3Ddesktop %26f eature % 3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath= %2F ServiceLogin&Page=PasswordSeparationSignIn ' ,
None , [ ] , 4 ] ,
1 , [ None , None , [ ] ] , None , None , None , True
] ,
username ,
]
2017-05-06 12:58:47 -04:00
lookup_results = req (
2017-05-06 17:19:11 -04:00
self . _LOOKUP_URL , lookup_req ,
2017-05-06 12:58:47 -04:00
' Looking up account info ' , ' Unable to look up account info ' )
if lookup_results is False :
return False
2015-08-15 12:03:43 -04:00
2017-05-06 17:19:11 -04:00
user_hash = try_get ( lookup_results , lambda x : x [ 0 ] [ 2 ] , compat_str )
if not user_hash :
warn ( ' Unable to extract user hash ' )
return False
challenge_req = [
user_hash ,
None , 1 , None , [ 1 , None , None , None , [ password , None , True ] ] ,
[
None , None , [ 2 , 1 , None , 1 , ' https://accounts.google.com/ServiceLogin?passive=true&continue=https % 3A %2F %2F www.youtube.com %2F signin %3F next % 3D %252F %26a ction_handle_signin % 3Dtrue % 26hl % 3Den %26a pp % 3Ddesktop %26f eature % 3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath= %2F ServiceLogin&Page=PasswordSeparationSignIn ' , None , [ ] , 4 ] ,
1 , [ None , None , [ ] ] , None , None , None , True
] ]
2014-08-16 17:28:41 -04:00
2017-05-06 17:19:11 -04:00
challenge_results = req (
self . _CHALLENGE_URL , challenge_req ,
' Logging in ' , ' Unable to log in ' )
2014-08-16 17:28:41 -04:00
2017-05-06 17:19:11 -04:00
if challenge_results is False :
2017-05-06 12:58:47 -04:00
return
2014-08-16 17:28:41 -04:00
2017-05-06 17:19:11 -04:00
login_res = try_get ( challenge_results , lambda x : x [ 0 ] [ 5 ] , list )
if login_res :
login_msg = try_get ( login_res , lambda x : x [ 5 ] , compat_str )
warn (
' Unable to login: %s ' % ' Invalid password '
if login_msg == ' INCORRECT_ANSWER_ENTERED ' else login_msg )
return False
res = try_get ( challenge_results , lambda x : x [ 0 ] [ - 1 ] , list )
if not res :
warn ( ' Unable to extract result entry ' )
return False
2018-07-04 13:36:35 -04:00
login_challenge = try_get ( res , lambda x : x [ 0 ] [ 0 ] , list )
if login_challenge :
challenge_str = try_get ( login_challenge , lambda x : x [ 2 ] , compat_str )
if challenge_str == ' TWO_STEP_VERIFICATION ' :
2017-05-06 17:19:11 -04:00
# SEND_SUCCESS - TFA code has been successfully sent to phone
# QUOTA_EXCEEDED - reached the limit of TFA codes
2018-07-04 13:36:35 -04:00
status = try_get ( login_challenge , lambda x : x [ 5 ] , compat_str )
2017-05-06 17:19:11 -04:00
if status == ' QUOTA_EXCEEDED ' :
warn ( ' Exceeded the limit of TFA codes, try later ' )
return False
tl = try_get ( challenge_results , lambda x : x [ 1 ] [ 2 ] , compat_str )
if not tl :
warn ( ' Unable to extract TL ' )
return False
tfa_code = self . _get_tfa_info ( ' 2-step verification code ' )
if not tfa_code :
warn (
' Two-factor authentication required. Provide it either interactively or with --twofactor <code> '
' (Note that only TOTP (Google Authenticator App) codes work at this time.) ' )
return False
tfa_code = remove_start ( tfa_code , ' G- ' )
tfa_req = [
user_hash , None , 2 , None ,
[
9 , None , None , None , None , None , None , None ,
[ None , tfa_code , True , 2 ]
] ]
tfa_results = req (
self . _TFA_URL . format ( tl ) , tfa_req ,
' Submitting TFA code ' , ' Unable to submit TFA code ' )
if tfa_results is False :
return False
tfa_res = try_get ( tfa_results , lambda x : x [ 0 ] [ 5 ] , list )
if tfa_res :
tfa_msg = try_get ( tfa_res , lambda x : x [ 5 ] , compat_str )
warn (
' Unable to finish TFA: %s ' % ' Invalid TFA code '
if tfa_msg == ' INCORRECT_ANSWER_ENTERED ' else tfa_msg )
return False
check_cookie_url = try_get (
tfa_results , lambda x : x [ 0 ] [ - 1 ] [ 2 ] , compat_str )
2018-07-04 13:36:35 -04:00
else :
CHALLENGES = {
' LOGIN_CHALLENGE ' : " This device isn ' t recognized. For your security, Google wants to make sure it ' s really you. " ,
' USERNAME_RECOVERY ' : ' Please provide additional information to aid in the recovery process. ' ,
' REAUTH ' : " There is something unusual about your activity. For your security, Google wants to make sure it ' s really you. " ,
}
challenge = CHALLENGES . get (
challenge_str ,
' %s returned error %s . ' % ( self . IE_NAME , challenge_str ) )
warn ( ' %s \n Go to https://accounts.google.com/, login and solve a challenge. ' % challenge )
return False
2017-05-06 17:19:11 -04:00
else :
check_cookie_url = try_get ( res , lambda x : x [ 2 ] , compat_str )
if not check_cookie_url :
warn ( ' Unable to extract CheckCookie URL ' )
return False
2017-05-06 12:58:47 -04:00
check_cookie_results = self . _download_webpage (
2017-05-06 17:19:11 -04:00
check_cookie_url , None , ' Checking cookie ' , fatal = False )
if check_cookie_results is False :
return False
2017-05-06 12:58:47 -04:00
2017-05-06 17:19:11 -04:00
if ' https://myaccount.google.com/ ' not in check_cookie_results :
warn ( ' Unable to log in ' )
2013-07-24 14:40:12 -04:00
return False
2017-05-06 12:58:47 -04:00
2013-07-24 14:40:12 -04:00
return True
2021-04-01 04:28:33 -04:00
def _initialize_consent ( self ) :
cookies = self . _get_cookies ( ' https://www.youtube.com/ ' )
if cookies . get ( ' __Secure-3PSID ' ) :
return
consent_id = None
consent = cookies . get ( ' CONSENT ' )
if consent :
if ' YES ' in consent . value :
return
consent_id = self . _search_regex (
r ' PENDING \ +( \ d+) ' , consent . value , ' consent ' , default = None )
if not consent_id :
consent_id = random . randint ( 100 , 999 )
self . _set_cookie ( ' .youtube.com ' , ' CONSENT ' , ' YES+cb.20210328-17-p0.en+FX+ %s ' % consent_id )
2017-09-01 13:57:14 -04:00
2013-07-24 14:40:12 -04:00
def _real_initialize ( self ) :
2021-04-01 04:28:33 -04:00
self . _initialize_consent ( )
2013-07-24 14:40:12 -04:00
if self . _downloader is None :
return
if not self . _login ( ) :
return
2013-06-23 13:58:33 -04:00
2020-11-21 09:50:42 -05:00
_YT_INITIAL_DATA_RE = r ' (?:window \ s* \ [ \ s*[ " \' ]ytInitialData[ " \' ] \ s* \ ]|ytInitialData) \ s*= \ s*( { .+?}) \ s*; '
2021-01-01 07:26:37 -05:00
_YT_INITIAL_PLAYER_RESPONSE_RE = r ' ytInitialPlayerResponse \ s*= \ s*( { .+?}) \ s*; '
_YT_INITIAL_BOUNDARY_RE = r ' (?:var \ s+meta|</script| \ n) '
2020-11-21 09:50:42 -05:00
2021-06-29 18:07:49 -04:00
_YT_DEFAULT_YTCFGS = {
' WEB ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' WEB ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 2.20210622.10.00 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' WEB ' ,
' clientVersion ' : ' 2.20210622.10.00 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : 1
} ,
' WEB_REMIX ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' WEB_REMIX ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 1.20210621.00.00 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' WEB_REMIX ' ,
' clientVersion ' : ' 1.20210621.00.00 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : 67
} ,
' WEB_EMBEDDED_PLAYER ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' WEB_EMBEDDED_PLAYER ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 1.20210620.0.1 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' WEB_EMBEDDED_PLAYER ' ,
' clientVersion ' : ' 1.20210620.0.1 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : 56
} ,
' ANDROID ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' ANDROID ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 16.20 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' ANDROID ' ,
' clientVersion ' : ' 16.20 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : ' ANDROID '
} ,
' ANDROID_EMBEDDED_PLAYER ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' ANDROID_EMBEDDED_PLAYER ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 16.20 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' ANDROID_EMBEDDED_PLAYER ' ,
' clientVersion ' : ' 16.20 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : ' ANDROID_EMBEDDED_PLAYER '
} ,
' ANDROID_MUSIC ' : {
' INNERTUBE_API_VERSION ' : ' v1 ' ,
' INNERTUBE_CLIENT_NAME ' : ' ANDROID_MUSIC ' ,
' INNERTUBE_CLIENT_VERSION ' : ' 4.32 ' ,
' INNERTUBE_API_KEY ' : ' AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30 ' ,
' INNERTUBE_CONTEXT ' : {
' client ' : {
' clientName ' : ' ANDROID_MUSIC ' ,
' clientVersion ' : ' 4.32 ' ,
' hl ' : ' en ' ,
}
} ,
' INNERTUBE_CONTEXT_CLIENT_NAME ' : ' ANDROID_MUSIC '
}
}
_YT_DEFAULT_INNERTUBE_HOSTS = {
' DIRECT ' : ' youtubei.googleapis.com ' ,
' WEB ' : ' www.youtube.com ' ,
' WEB_REMIX ' : ' music.youtube.com ' ,
' ANDROID_MUSIC ' : ' music.youtube.com '
}
def _get_default_ytcfg ( self , client = ' WEB ' ) :
if client in self . _YT_DEFAULT_YTCFGS :
return copy . deepcopy ( self . _YT_DEFAULT_YTCFGS [ client ] )
self . write_debug ( f ' INNERTUBE default client { client } does not exist - falling back to WEB client. ' )
return copy . deepcopy ( self . _YT_DEFAULT_YTCFGS [ ' WEB ' ] )
def _get_innertube_host ( self , client = ' WEB ' ) :
return dict_get ( self . _YT_DEFAULT_INNERTUBE_HOSTS , ( client , ' WEB ' ) )
def _ytcfg_get_safe ( self , ytcfg , getter , expected_type = None , default_client = ' WEB ' ) :
# try_get but with fallback to default ytcfg client values when present
_func = lambda y : try_get ( y , getter , expected_type )
return _func ( ytcfg ) or _func ( self . _get_default_ytcfg ( default_client ) )
def _extract_client_name ( self , ytcfg , default_client = ' WEB ' ) :
return self . _ytcfg_get_safe ( ytcfg , lambda x : x [ ' INNERTUBE_CLIENT_NAME ' ] , compat_str , default_client )
def _extract_client_version ( self , ytcfg , default_client = ' WEB ' ) :
return self . _ytcfg_get_safe ( ytcfg , lambda x : x [ ' INNERTUBE_CLIENT_VERSION ' ] , compat_str , default_client )
def _extract_api_key ( self , ytcfg = None , default_client = ' WEB ' ) :
return self . _ytcfg_get_safe ( ytcfg , lambda x : x [ ' INNERTUBE_API_KEY ' ] , compat_str , default_client )
def _extract_context ( self , ytcfg = None , default_client = ' WEB ' ) :
_get_context = lambda y : try_get ( y , lambda x : x [ ' INNERTUBE_CONTEXT ' ] , dict )
context = _get_context ( ytcfg )
if context :
return context
context = _get_context ( self . _get_default_ytcfg ( default_client ) )
if not ytcfg :
return context
# Recreate the client context (required)
context [ ' client ' ] . update ( {
' clientVersion ' : self . _extract_client_version ( ytcfg , default_client ) ,
' clientName ' : self . _extract_client_name ( ytcfg , default_client ) ,
} )
visitor_data = try_get ( ytcfg , lambda x : x [ ' VISITOR_DATA ' ] , compat_str )
if visitor_data :
context [ ' client ' ] [ ' visitorData ' ] = visitor_data
return context
def _generate_sapisidhash_header ( self , origin = ' https://www.youtube.com ' ) :
2021-06-10 17:02:57 -04:00
# Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
# See: https://github.com/yt-dlp/yt-dlp/issues/393
yt_cookies = self . _get_cookies ( ' https://www.youtube.com ' )
sapisid_cookie = dict_get (
yt_cookies , ( ' __Secure-3PAPISID ' , ' SAPISID ' ) )
2021-03-03 05:32:40 -05:00
if sapisid_cookie is None :
return
time_now = round ( time . time ( ) )
2021-06-10 17:02:57 -04:00
# SAPISID cookie is required if not already present
if not yt_cookies . get ( ' SAPISID ' ) :
self . _set_cookie (
' .youtube.com ' , ' SAPISID ' , sapisid_cookie . value , secure = True , expire_time = time_now + 3600 )
# SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
sapisidhash = hashlib . sha1 (
2021-06-29 18:07:49 -04:00
f ' { time_now } { sapisid_cookie . value } { origin } ' . encode ( ' utf-8 ' ) ) . hexdigest ( )
2021-06-10 17:02:57 -04:00
return f ' SAPISIDHASH { time_now } _ { sapisidhash } '
2021-03-03 05:32:40 -05:00
def _call_api ( self , ep , query , video_id , fatal = True , headers = None ,
2021-04-11 16:23:04 -04:00
note = ' Downloading API JSON ' , errnote = ' Unable to download API page ' ,
2021-06-29 18:07:49 -04:00
context = None , api_key = None , api_hostname = None , default_client = ' WEB ' ) :
2021-04-11 16:23:04 -04:00
2021-06-29 18:07:49 -04:00
data = { ' context ' : context } if context else { ' context ' : self . _extract_context ( default_client = default_client ) }
2020-11-19 14:22:59 -05:00
data . update ( query )
2021-06-29 18:07:49 -04:00
real_headers = self . _generate_api_headers ( client = default_client )
2021-04-11 16:23:04 -04:00
real_headers . update ( { ' content-type ' : ' application/json ' } )
if headers :
real_headers . update ( headers )
2021-02-04 09:37:17 -05:00
return self . _download_json (
2021-06-29 18:07:49 -04:00
' https:// %s /youtubei/v1/ %s ' % ( api_hostname or self . _get_innertube_host ( default_client ) , ep ) ,
2021-03-03 05:32:40 -05:00
video_id = video_id , fatal = fatal , note = note , errnote = errnote ,
2021-04-11 16:23:04 -04:00
data = json . dumps ( data ) . encode ( ' utf8 ' ) , headers = real_headers ,
query = { ' key ' : api_key or self . _extract_api_key ( ) } )
2020-11-19 14:22:59 -05:00
def _extract_yt_initial_data ( self , video_id , webpage ) :
return self . _parse_json (
self . _search_regex (
2021-01-01 07:26:37 -05:00
( r ' %s \ s* %s ' % ( self . _YT_INITIAL_DATA_RE , self . _YT_INITIAL_BOUNDARY_RE ) ,
2020-11-21 09:50:42 -05:00
self . _YT_INITIAL_DATA_RE ) , webpage , ' yt initial data ' ) ,
2020-11-19 14:22:59 -05:00
video_id )
2015-11-21 17:17:07 -05:00
2021-03-14 18:41:11 -04:00
def _extract_identity_token ( self , webpage , item_id ) :
ytcfg = self . _extract_ytcfg ( item_id , webpage )
if ytcfg :
token = try_get ( ytcfg , lambda x : x [ ' ID_TOKEN ' ] , compat_str )
if token :
return token
return self . _search_regex (
r ' \ bID_TOKEN[ " \' ] \ s*: \ s*[ " \' ](.+?)[ " \' ] ' , webpage ,
' identity token ' , default = None )
@staticmethod
def _extract_account_syncid ( data ) :
2021-04-14 01:07:03 -04:00
"""
Extract syncId required to download private playlists of secondary channels
@param data Either response or ytcfg
"""
sync_ids = ( try_get (
data , ( lambda x : x [ ' responseContext ' ] [ ' mainAppWebResponseContext ' ] [ ' datasyncId ' ] ,
lambda x : x [ ' DATASYNC_ID ' ] ) , compat_str ) or ' ' ) . split ( " || " )
2021-03-14 18:41:11 -04:00
if len ( sync_ids ) > = 2 and sync_ids [ 1 ] :
# datasyncid is of the form "channel_syncid||user_syncid" for secondary channel
# and just "user_syncid||" for primary channel. We only want the channel_syncid
return sync_ids [ 0 ]
2021-04-14 01:07:03 -04:00
# ytcfg includes channel_syncid if on secondary channel
return data . get ( ' DELEGATED_SESSION_ID ' )
2021-03-14 18:41:11 -04:00
2021-01-01 07:26:37 -05:00
def _extract_ytcfg ( self , video_id , webpage ) :
2021-04-21 01:07:24 -04:00
if not webpage :
return { }
2021-01-01 07:26:37 -05:00
return self . _parse_json (
self . _search_regex (
r ' ytcfg \ .set \ s* \ ( \ s*( { .+?}) \ s* \ ) \ s*; ' , webpage , ' ytcfg ' ,
2021-04-11 16:23:04 -04:00
default = ' {} ' ) , video_id , fatal = False ) or { }
2021-06-29 18:07:49 -04:00
def _generate_api_headers ( self , ytcfg = None , identity_token = None , account_syncid = None ,
visitor_data = None , api_hostname = None , client = ' WEB ' ) :
origin = ' https:// ' + ( api_hostname if api_hostname else self . _get_innertube_host ( client ) )
2021-04-11 16:23:04 -04:00
headers = {
2021-06-29 18:07:49 -04:00
' X-YouTube-Client-Name ' : compat_str (
self . _ytcfg_get_safe ( ytcfg , lambda x : x [ ' INNERTUBE_CONTEXT_CLIENT_NAME ' ] , default_client = client ) ) ,
' X-YouTube-Client-Version ' : self . _extract_client_version ( ytcfg , client ) ,
' Origin ' : origin
2021-04-11 16:23:04 -04:00
}
if identity_token :
2021-06-29 18:07:49 -04:00
headers [ ' X-Youtube-Identity-Token ' ] = identity_token
2021-04-11 16:23:04 -04:00
if account_syncid :
headers [ ' X-Goog-PageId ' ] = account_syncid
headers [ ' X-Goog-AuthUser ' ] = 0
if visitor_data :
2021-06-29 18:07:49 -04:00
headers [ ' X-Goog-Visitor-Id ' ] = visitor_data
auth = self . _generate_sapisidhash_header ( origin )
2021-04-11 16:23:04 -04:00
if auth is not None :
headers [ ' Authorization ' ] = auth
2021-06-29 18:07:49 -04:00
headers [ ' X-Origin ' ] = origin
2021-04-11 16:23:04 -04:00
return headers
2021-01-01 07:26:37 -05:00
2021-06-29 18:07:49 -04:00
@staticmethod
def _extract_alerts ( data ) :
for alert_dict in try_get ( data , lambda x : x [ ' alerts ' ] , list ) or [ ] :
if not isinstance ( alert_dict , dict ) :
continue
for alert in alert_dict . values ( ) :
alert_type = alert . get ( ' type ' )
if not alert_type :
continue
message = try_get ( alert , lambda x : x [ ' text ' ] [ ' simpleText ' ] , compat_str ) or ' '
if message :
yield alert_type , message
for run in try_get ( alert , lambda x : x [ ' text ' ] [ ' runs ' ] , list ) or [ ] :
message + = try_get ( run , lambda x : x [ ' text ' ] , compat_str )
if message :
yield alert_type , message
def _report_alerts ( self , alerts , expected = True ) :
errors = [ ]
warnings = [ ]
for alert_type , alert_message in alerts :
if alert_type . lower ( ) == ' error ' :
errors . append ( [ alert_type , alert_message ] )
else :
warnings . append ( [ alert_type , alert_message ] )
for alert_type , alert_message in ( warnings + errors [ : - 1 ] ) :
self . report_warning ( ' YouTube said: %s - %s ' % ( alert_type , alert_message ) )
if errors :
raise ExtractorError ( ' YouTube said: %s ' % errors [ - 1 ] [ 1 ] , expected = expected )
def _extract_and_report_alerts ( self , data , * args , * * kwargs ) :
return self . _report_alerts ( self . _extract_alerts ( data ) , * args , * * kwargs )
def _extract_response ( self , item_id , query , note = ' Downloading API JSON ' , headers = None ,
ytcfg = None , check_get_keys = None , ep = ' browse ' , fatal = True , api_hostname = None ,
default_client = ' WEB ' ) :
response = None
last_error = None
count = - 1
retries = self . get_param ( ' extractor_retries ' , 3 )
if check_get_keys is None :
check_get_keys = [ ]
while count < retries :
count + = 1
if last_error :
self . report_warning ( ' %s . Retrying ... ' % last_error )
try :
response = self . _call_api (
ep = ep , fatal = True , headers = headers ,
video_id = item_id , query = query ,
context = self . _extract_context ( ytcfg , default_client ) ,
api_key = self . _extract_api_key ( ytcfg , default_client ) ,
api_hostname = api_hostname , default_client = default_client ,
note = ' %s %s ' % ( note , ' (retry # %d ) ' % count if count else ' ' ) )
except ExtractorError as e :
if isinstance ( e . cause , compat_HTTPError ) and e . cause . code in ( 500 , 503 , 404 ) :
# Downloading page may result in intermittent 5xx HTTP error
# Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
last_error = ' HTTP Error %s ' % e . cause . code
if count < retries :
continue
if fatal :
raise
else :
self . report_warning ( error_to_compat_str ( e ) )
return
else :
# Youtube may send alerts if there was an issue with the continuation page
try :
self . _extract_and_report_alerts ( response , expected = False )
except ExtractorError as e :
if fatal :
raise
self . report_warning ( error_to_compat_str ( e ) )
return
if not check_get_keys or dict_get ( response , check_get_keys ) :
break
# Youtube sometimes sends incomplete data
# See: https://github.com/ytdl-org/youtube-dl/issues/28194
last_error = ' Incomplete data received '
if count > = retries :
if fatal :
raise ExtractorError ( last_error )
else :
self . report_warning ( last_error )
return
return response
2021-05-15 10:38:47 -04:00
@staticmethod
def is_music_url ( url ) :
return re . match ( r ' https?://music \ .youtube \ .com/ ' , url ) is not None
2021-01-16 07:40:15 -05:00
def _extract_video ( self , renderer ) :
video_id = renderer . get ( ' videoId ' )
title = try_get (
renderer ,
( lambda x : x [ ' title ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ,
lambda x : x [ ' title ' ] [ ' simpleText ' ] ) , compat_str )
description = try_get (
renderer , lambda x : x [ ' descriptionSnippet ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ,
compat_str )
duration = parse_duration ( try_get (
renderer , lambda x : x [ ' lengthText ' ] [ ' simpleText ' ] , compat_str ) )
view_count_text = try_get (
renderer , lambda x : x [ ' viewCountText ' ] [ ' simpleText ' ] , compat_str ) or ' '
view_count = str_to_int ( self . _search_regex (
r ' ^([ \ d,]+) ' , re . sub ( r ' \ s ' , ' ' , view_count_text ) ,
' view count ' , default = None ) )
uploader = try_get (
2021-02-19 15:44:36 -05:00
renderer ,
( lambda x : x [ ' ownerText ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ,
lambda x : x [ ' shortBylineText ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ) , compat_str )
2021-01-16 07:40:15 -05:00
return {
2021-04-03 18:05:17 -04:00
' _type ' : ' url ' ,
2021-01-16 07:40:15 -05:00
' ie_key ' : YoutubeIE . ie_key ( ) ,
' id ' : video_id ,
' url ' : video_id ,
' title ' : title ,
' description ' : description ,
' duration ' : duration ,
' view_count ' : view_count ,
' uploader ' : uploader ,
}
2015-11-21 17:17:07 -05:00
2015-02-16 15:44:17 -05:00
class YoutubeIE ( YoutubeBaseInfoExtractor ) :
2014-09-13 01:51:06 -04:00
IE_DESC = ' YouTube.com '
2021-02-19 15:44:36 -05:00
_INVIDIOUS_SITES = (
# invidious-redirect websites
r ' (?:www \ .)?redirect \ .invidious \ .io ' ,
r ' (?:(?:www|dev) \ .)?invidio \ .us ' ,
# Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
r ' (?:www \ .)?invidious \ .pussthecat \ .org ' ,
r ' (?:www \ .)?invidious \ .zee \ .li ' ,
r ' (?:www \ .)?invidious \ .ethibox \ .fr ' ,
r ' (?:www \ .)?invidious \ .3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd \ .onion ' ,
# youtube-dl invidious instances list
r ' (?:(?:www|no) \ .)?invidiou \ .sh ' ,
r ' (?:(?:www|fi) \ .)?invidious \ .snopyta \ .org ' ,
r ' (?:www \ .)?invidious \ .kabi \ .tk ' ,
r ' (?:www \ .)?invidious \ .mastodon \ .host ' ,
r ' (?:www \ .)?invidious \ .zapashcanon \ .fr ' ,
2021-06-22 21:26:37 -04:00
r ' (?:www \ .)?(?:invidious(?:-us)?|piped) \ .kavin \ .rocks ' ,
2021-04-16 23:02:33 -04:00
r ' (?:www \ .)?invidious \ .tinfoil-hat \ .net ' ,
r ' (?:www \ .)?invidious \ .himiko \ .cloud ' ,
r ' (?:www \ .)?invidious \ .reallyancient \ .tech ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?invidious \ .tube ' ,
r ' (?:www \ .)?invidiou \ .site ' ,
r ' (?:www \ .)?invidious \ .site ' ,
r ' (?:www \ .)?invidious \ .xyz ' ,
r ' (?:www \ .)?invidious \ .nixnet \ .xyz ' ,
2021-04-16 23:02:33 -04:00
r ' (?:www \ .)?invidious \ .048596 \ .xyz ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?invidious \ .drycat \ .fr ' ,
2021-04-16 23:02:33 -04:00
r ' (?:www \ .)?inv \ .skyn3t \ .in ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?tube \ .poal \ .co ' ,
r ' (?:www \ .)?tube \ .connect \ .cafe ' ,
r ' (?:www \ .)?vid \ .wxzm \ .sx ' ,
r ' (?:www \ .)?vid \ .mint \ .lgbt ' ,
2021-04-16 23:02:33 -04:00
r ' (?:www \ .)?vid \ .puffyan \ .us ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?yewtu \ .be ' ,
r ' (?:www \ .)?yt \ .elukerio \ .org ' ,
r ' (?:www \ .)?yt \ .lelux \ .fi ' ,
r ' (?:www \ .)?invidious \ .ggc-project \ .de ' ,
r ' (?:www \ .)?yt \ .maisputain \ .ovh ' ,
2021-04-16 23:02:33 -04:00
r ' (?:www \ .)?ytprivate \ .com ' ,
r ' (?:www \ .)?invidious \ .13ad \ .de ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?invidious \ .toot \ .koeln ' ,
r ' (?:www \ .)?invidious \ .fdn \ .fr ' ,
r ' (?:www \ .)?watch \ .nettohikari \ .com ' ,
2021-06-22 21:26:37 -04:00
r ' (?:www \ .)?invidious \ .namazso \ .eu ' ,
r ' (?:www \ .)?invidious \ .silkky \ .cloud ' ,
r ' (?:www \ .)?invidious \ .exonip \ .de ' ,
r ' (?:www \ .)?invidious \ .riverside \ .rocks ' ,
r ' (?:www \ .)?invidious \ .blamefran \ .net ' ,
r ' (?:www \ .)?invidious \ .moomoo \ .de ' ,
r ' (?:www \ .)?ytb \ .trom \ .tf ' ,
r ' (?:www \ .)?yt \ .cyberhost \ .uk ' ,
2021-02-19 15:44:36 -05:00
r ' (?:www \ .)?kgg2m7yk5aybusll \ .onion ' ,
r ' (?:www \ .)?qklhadlycap4cnod \ .onion ' ,
r ' (?:www \ .)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid \ .onion ' ,
r ' (?:www \ .)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid \ .onion ' ,
r ' (?:www \ .)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad \ .onion ' ,
r ' (?:www \ .)?invidious \ .l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd \ .onion ' ,
r ' (?:www \ .)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya \ .b32 \ .i2p ' ,
r ' (?:www \ .)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd \ .onion ' ,
2021-06-22 21:26:37 -04:00
r ' (?:www \ .)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd \ .onion ' ,
r ' (?:www \ .)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad \ .onion ' ,
r ' (?:www \ .)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad \ .onion ' ,
r ' (?:www \ .)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid \ .onion ' ,
2021-02-19 15:44:36 -05:00
)
2013-11-18 10:42:35 -05:00
_VALID_URL = r """ (?x)^
2013-06-23 13:58:33 -04:00
(
2014-09-11 15:47:25 -04:00
( ? : https ? : / / | / / ) # http(s):// or protocol-independent URL
2021-02-19 15:44:36 -05:00
( ? : ( ? : ( ? : ( ? : \w + \. ) ? [ yY ] [ oO ] [ uU ] [ tT ] [ uU ] [ bB ] [ eE ] ( ? : - nocookie | kids ) ? \. com |
( ? : www \. ) ? deturl \. com / www \. youtube \. com |
( ? : www \. ) ? pwnyoutube \. com |
( ? : www \. ) ? hooktube \. com |
( ? : www \. ) ? yourepeat \. com |
tube \. majestyc \. net |
% ( invidious ) s |
youtube \. googleapis \. com ) / # the various hostnames, with wildcard subdomains
2013-06-23 13:58:33 -04:00
( ? : . * ? \#/)? # handle anchor (#/) redirect urls
( ? : # the various things that can precede the ID:
2014-09-24 04:34:29 -04:00
( ? : ( ? : v | embed | e ) / ( ? ! videoseries ) ) # v/ or embed/ or e/
2013-06-23 13:58:33 -04:00
| ( ? : # or the v= param in all its forms
2014-02-18 14:00:54 -05:00
( ? : ( ? : watch | movie ) ( ? : _popup ) ? ( ? : \. php ) ? / ? ) ? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
2013-06-23 13:58:33 -04:00
( ? : \? | \#!?) # the params delimiter ? or # or #!
2015-11-29 10:01:59 -05:00
( ? : . * ? [ & ; ] ) ? ? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&v=V36LpHqtcDY)
2013-06-23 13:58:33 -04:00
v =
)
2013-09-05 16:38:23 -04:00
) )
2015-08-16 16:04:13 -04:00
| ( ? :
youtu \. be | # just youtu.be/xxxx
2016-04-03 16:26:20 -04:00
vid \. plus | # or vid.plus/xxxx
zwearz \. com / watch | # or zwearz.com/watch/xxxx
2021-02-19 15:44:36 -05:00
% ( invidious ) s
2015-08-16 16:04:13 -04:00
) /
2014-09-11 15:47:25 -04:00
| ( ? : www \. ) ? cleanvideosearch \. com / media / action / yt / watch \? videoId =
2013-09-05 16:38:23 -04:00
)
2013-06-23 13:58:33 -04:00
) ? # all until now is optional -> you can pass the naked ID
2021-04-16 23:02:33 -04:00
( ? P < id > [ 0 - 9 A - Za - z_ - ] { 11 } ) # here is it! the YouTube video ID
2013-06-23 13:58:33 -04:00
( ? ( 1 ) . + ) ? # if we found the ID, everything can follow
2021-05-15 10:38:47 -04:00
( ? : \#|$)""" % {
2021-02-19 15:44:36 -05:00
' invidious ' : ' | ' . join ( _INVIDIOUS_SITES ) ,
}
2020-05-01 20:18:08 -04:00
_PLAYER_INFO_RE = (
2021-02-10 16:22:55 -05:00
r ' /s/player/(?P<id>[a-zA-Z0-9_-] { 8,})/player ' ,
r ' /(?P<id>[a-zA-Z0-9_-] { 8,})/player(?:_ias \ .vflset(?:/[a-zA-Z] { 2,3}_[a-zA-Z] { 2,3})?|-plasma-ias-(?:phone|tablet)-[a-z] {2} _[A-Z] {2} \ .vflset)/base \ .js$ ' ,
2021-02-04 09:37:17 -05:00
r ' \ b(?P<id>vfl[a-zA-Z0-9_-]+) \ b.*? \ .js$ ' ,
2020-05-01 20:18:08 -04:00
)
2013-12-24 06:34:09 -05:00
_formats = {
2016-03-02 11:35:04 -05:00
' 5 ' : { ' ext ' : ' flv ' , ' width ' : 400 , ' height ' : 240 , ' acodec ' : ' mp3 ' , ' abr ' : 64 , ' vcodec ' : ' h263 ' } ,
' 6 ' : { ' ext ' : ' flv ' , ' width ' : 450 , ' height ' : 270 , ' acodec ' : ' mp3 ' , ' abr ' : 64 , ' vcodec ' : ' h263 ' } ,
' 13 ' : { ' ext ' : ' 3gp ' , ' acodec ' : ' aac ' , ' vcodec ' : ' mp4v ' } ,
' 17 ' : { ' ext ' : ' 3gp ' , ' width ' : 176 , ' height ' : 144 , ' acodec ' : ' aac ' , ' abr ' : 24 , ' vcodec ' : ' mp4v ' } ,
' 18 ' : { ' ext ' : ' mp4 ' , ' width ' : 640 , ' height ' : 360 , ' acodec ' : ' aac ' , ' abr ' : 96 , ' vcodec ' : ' h264 ' } ,
' 22 ' : { ' ext ' : ' mp4 ' , ' width ' : 1280 , ' height ' : 720 , ' acodec ' : ' aac ' , ' abr ' : 192 , ' vcodec ' : ' h264 ' } ,
' 34 ' : { ' ext ' : ' flv ' , ' width ' : 640 , ' height ' : 360 , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' } ,
' 35 ' : { ' ext ' : ' flv ' , ' width ' : 854 , ' height ' : 480 , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' } ,
2016-02-07 14:30:57 -05:00
# itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
2016-03-02 11:35:04 -05:00
' 36 ' : { ' ext ' : ' 3gp ' , ' width ' : 320 , ' acodec ' : ' aac ' , ' vcodec ' : ' mp4v ' } ,
' 37 ' : { ' ext ' : ' mp4 ' , ' width ' : 1920 , ' height ' : 1080 , ' acodec ' : ' aac ' , ' abr ' : 192 , ' vcodec ' : ' h264 ' } ,
' 38 ' : { ' ext ' : ' mp4 ' , ' width ' : 4096 , ' height ' : 3072 , ' acodec ' : ' aac ' , ' abr ' : 192 , ' vcodec ' : ' h264 ' } ,
' 43 ' : { ' ext ' : ' webm ' , ' width ' : 640 , ' height ' : 360 , ' acodec ' : ' vorbis ' , ' abr ' : 128 , ' vcodec ' : ' vp8 ' } ,
' 44 ' : { ' ext ' : ' webm ' , ' width ' : 854 , ' height ' : 480 , ' acodec ' : ' vorbis ' , ' abr ' : 128 , ' vcodec ' : ' vp8 ' } ,
' 45 ' : { ' ext ' : ' webm ' , ' width ' : 1280 , ' height ' : 720 , ' acodec ' : ' vorbis ' , ' abr ' : 192 , ' vcodec ' : ' vp8 ' } ,
2016-01-02 22:11:19 -05:00
' 46 ' : { ' ext ' : ' webm ' , ' width ' : 1920 , ' height ' : 1080 , ' acodec ' : ' vorbis ' , ' abr ' : 192 , ' vcodec ' : ' vp8 ' } ,
2016-03-02 11:35:04 -05:00
' 59 ' : { ' ext ' : ' mp4 ' , ' width ' : 854 , ' height ' : 480 , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' } ,
' 78 ' : { ' ext ' : ' mp4 ' , ' width ' : 854 , ' height ' : 480 , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' } ,
2016-01-02 22:11:19 -05:00
# 3D videos
2016-03-02 11:35:04 -05:00
' 82 ' : { ' ext ' : ' mp4 ' , ' height ' : 360 , ' format_note ' : ' 3D ' , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' , ' preference ' : - 20 } ,
' 83 ' : { ' ext ' : ' mp4 ' , ' height ' : 480 , ' format_note ' : ' 3D ' , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' , ' preference ' : - 20 } ,
' 84 ' : { ' ext ' : ' mp4 ' , ' height ' : 720 , ' format_note ' : ' 3D ' , ' acodec ' : ' aac ' , ' abr ' : 192 , ' vcodec ' : ' h264 ' , ' preference ' : - 20 } ,
' 85 ' : { ' ext ' : ' mp4 ' , ' height ' : 1080 , ' format_note ' : ' 3D ' , ' acodec ' : ' aac ' , ' abr ' : 192 , ' vcodec ' : ' h264 ' , ' preference ' : - 20 } ,
2016-01-02 22:11:19 -05:00
' 100 ' : { ' ext ' : ' webm ' , ' height ' : 360 , ' format_note ' : ' 3D ' , ' acodec ' : ' vorbis ' , ' abr ' : 128 , ' vcodec ' : ' vp8 ' , ' preference ' : - 20 } ,
' 101 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' format_note ' : ' 3D ' , ' acodec ' : ' vorbis ' , ' abr ' : 192 , ' vcodec ' : ' vp8 ' , ' preference ' : - 20 } ,
' 102 ' : { ' ext ' : ' webm ' , ' height ' : 720 , ' format_note ' : ' 3D ' , ' acodec ' : ' vorbis ' , ' abr ' : 192 , ' vcodec ' : ' vp8 ' , ' preference ' : - 20 } ,
2013-08-19 21:22:25 -04:00
2013-09-03 21:49:35 -04:00
# Apple HTTP Live Streaming
2016-03-17 07:25:37 -04:00
' 91 ' : { ' ext ' : ' mp4 ' , ' height ' : 144 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 48 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
2016-03-02 11:35:04 -05:00
' 92 ' : { ' ext ' : ' mp4 ' , ' height ' : 240 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 48 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
' 93 ' : { ' ext ' : ' mp4 ' , ' height ' : 360 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
' 94 ' : { ' ext ' : ' mp4 ' , ' height ' : 480 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 128 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
' 95 ' : { ' ext ' : ' mp4 ' , ' height ' : 720 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 256 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
' 96 ' : { ' ext ' : ' mp4 ' , ' height ' : 1080 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 256 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
2016-01-02 22:11:19 -05:00
' 132 ' : { ' ext ' : ' mp4 ' , ' height ' : 240 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 48 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
' 151 ' : { ' ext ' : ' mp4 ' , ' height ' : 72 , ' format_note ' : ' HLS ' , ' acodec ' : ' aac ' , ' abr ' : 24 , ' vcodec ' : ' h264 ' , ' preference ' : - 10 } ,
2013-12-24 06:34:09 -05:00
# DASH mp4 video
2017-04-11 11:41:48 -04:00
' 133 ' : { ' ext ' : ' mp4 ' , ' height ' : 240 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 134 ' : { ' ext ' : ' mp4 ' , ' height ' : 360 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 135 ' : { ' ext ' : ' mp4 ' , ' height ' : 480 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 136 ' : { ' ext ' : ' mp4 ' , ' height ' : 720 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 137 ' : { ' ext ' : ' mp4 ' , ' height ' : 1080 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
2019-03-09 07:14:41 -05:00
' 138 ' : { ' ext ' : ' mp4 ' , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } , # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
2017-04-11 11:41:48 -04:00
' 160 ' : { ' ext ' : ' mp4 ' , ' height ' : 144 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 212 ' : { ' ext ' : ' mp4 ' , ' height ' : 480 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 264 ' : { ' ext ' : ' mp4 ' , ' height ' : 1440 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
' 298 ' : { ' ext ' : ' mp4 ' , ' height ' : 720 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' , ' fps ' : 60 } ,
' 299 ' : { ' ext ' : ' mp4 ' , ' height ' : 1080 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' , ' fps ' : 60 } ,
' 266 ' : { ' ext ' : ' mp4 ' , ' height ' : 2160 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' h264 ' } ,
2013-08-19 21:22:25 -04:00
2013-10-18 17:53:00 -04:00
# Dash mp4 audio
2017-04-11 11:41:48 -04:00
' 139 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' aac ' , ' abr ' : 48 , ' container ' : ' m4a_dash ' } ,
' 140 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' aac ' , ' abr ' : 128 , ' container ' : ' m4a_dash ' } ,
' 141 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' aac ' , ' abr ' : 256 , ' container ' : ' m4a_dash ' } ,
' 256 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' aac ' , ' container ' : ' m4a_dash ' } ,
' 258 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' aac ' , ' container ' : ' m4a_dash ' } ,
' 325 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' dtse ' , ' container ' : ' m4a_dash ' } ,
' 328 ' : { ' ext ' : ' m4a ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' ec-3 ' , ' container ' : ' m4a_dash ' } ,
2013-08-19 21:22:25 -04:00
# Dash webm
2017-04-11 11:41:48 -04:00
' 167 ' : { ' ext ' : ' webm ' , ' height ' : 360 , ' width ' : 640 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 168 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' width ' : 854 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 169 ' : { ' ext ' : ' webm ' , ' height ' : 720 , ' width ' : 1280 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 170 ' : { ' ext ' : ' webm ' , ' height ' : 1080 , ' width ' : 1920 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 218 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' width ' : 854 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 219 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' width ' : 854 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp8 ' } ,
' 278 ' : { ' ext ' : ' webm ' , ' height ' : 144 , ' format_note ' : ' DASH video ' , ' container ' : ' webm ' , ' vcodec ' : ' vp9 ' } ,
' 242 ' : { ' ext ' : ' webm ' , ' height ' : 240 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 243 ' : { ' ext ' : ' webm ' , ' height ' : 360 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 244 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 245 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 246 ' : { ' ext ' : ' webm ' , ' height ' : 480 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 247 ' : { ' ext ' : ' webm ' , ' height ' : 720 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 248 ' : { ' ext ' : ' webm ' , ' height ' : 1080 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 271 ' : { ' ext ' : ' webm ' , ' height ' : 1440 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
2015-11-30 09:42:05 -05:00
# itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
2017-04-11 11:41:48 -04:00
' 272 ' : { ' ext ' : ' webm ' , ' height ' : 2160 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 302 ' : { ' ext ' : ' webm ' , ' height ' : 720 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' , ' fps ' : 60 } ,
' 303 ' : { ' ext ' : ' webm ' , ' height ' : 1080 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' , ' fps ' : 60 } ,
' 308 ' : { ' ext ' : ' webm ' , ' height ' : 1440 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' , ' fps ' : 60 } ,
' 313 ' : { ' ext ' : ' webm ' , ' height ' : 2160 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' } ,
' 315 ' : { ' ext ' : ' webm ' , ' height ' : 2160 , ' format_note ' : ' DASH video ' , ' vcodec ' : ' vp9 ' , ' fps ' : 60 } ,
2013-12-24 06:34:09 -05:00
# Dash webm audio
2017-04-11 11:41:48 -04:00
' 171 ' : { ' ext ' : ' webm ' , ' acodec ' : ' vorbis ' , ' format_note ' : ' DASH audio ' , ' abr ' : 128 } ,
' 172 ' : { ' ext ' : ' webm ' , ' acodec ' : ' vorbis ' , ' format_note ' : ' DASH audio ' , ' abr ' : 256 } ,
2014-01-08 20:38:50 -05:00
2014-11-18 05:06:09 -05:00
# Dash webm audio with opus inside
2017-04-11 11:41:48 -04:00
' 249 ' : { ' ext ' : ' webm ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' opus ' , ' abr ' : 50 } ,
' 250 ' : { ' ext ' : ' webm ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' opus ' , ' abr ' : 70 } ,
' 251 ' : { ' ext ' : ' webm ' , ' format_note ' : ' DASH audio ' , ' acodec ' : ' opus ' , ' abr ' : 160 } ,
2014-11-18 05:06:09 -05:00
2014-01-08 20:38:50 -05:00
# RTMP (unnamed)
' _rtmp ' : { ' protocol ' : ' rtmp ' } ,
2019-06-13 14:59:05 -04:00
# av01 video only formats sometimes served with "unknown" codecs
' 394 ' : { ' acodec ' : ' none ' , ' vcodec ' : ' av01.0.05M.08 ' } ,
' 395 ' : { ' acodec ' : ' none ' , ' vcodec ' : ' av01.0.05M.08 ' } ,
' 396 ' : { ' acodec ' : ' none ' , ' vcodec ' : ' av01.0.05M.08 ' } ,
' 397 ' : { ' acodec ' : ' none ' , ' vcodec ' : ' av01.0.05M.08 ' } ,
2013-06-23 13:58:33 -04:00
}
2021-01-01 07:26:37 -05:00
_SUBTITLE_FORMATS = ( ' json3 ' , ' srv1 ' , ' srv2 ' , ' srv3 ' , ' ttml ' , ' vtt ' )
2013-08-19 21:22:25 -04:00
2021-06-29 18:07:49 -04:00
_AGE_GATE_REASONS = (
' Sign in to confirm your age ' ,
' This video may be inappropriate for some users. ' ,
' Sorry, this content is age-restricted. ' )
2017-02-26 04:51:21 -05:00
_GEO_BYPASS = False
2014-09-13 01:51:06 -04:00
IE_NAME = ' youtube '
2013-06-27 13:13:11 -04:00
_TESTS = [
{
2016-09-17 10:48:20 -04:00
' url ' : ' https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9 ' ,
2014-09-24 03:49:53 -04:00
' info_dict ' : {
' id ' : ' BaW_jenozKc ' ,
' ext ' : ' mp4 ' ,
2020-09-02 16:37:35 -04:00
' title ' : ' youtube-dl test video " \' / \\ ä↭𝕐 ' ,
2014-09-24 03:49:53 -04:00
' uploader ' : ' Philipp Hagemeister ' ,
' uploader_id ' : ' phihag ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/phihag ' ,
2018-09-14 14:24:26 -04:00
' channel_id ' : ' UCLqxVugv74EIW3VWh2NOa3Q ' ,
' channel_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCLqxVugv74EIW3VWh2NOa3Q ' ,
2014-09-24 03:49:53 -04:00
' upload_date ' : ' 20121002 ' ,
2020-09-02 16:37:35 -04:00
' description ' : ' test chars: " \' / \\ ä↭𝕐 \n test URL: https://github.com/rg3/youtube-dl/issues/1892 \n \n This is a test video for youtube-dl. \n \n For more information, contact phihag@phihag.de . ' ,
2014-09-24 03:49:53 -04:00
' categories ' : [ ' Science & Technology ' ] ,
2020-09-02 16:37:35 -04:00
' tags ' : [ ' youtube-dl ' ] ,
2017-01-26 09:43:14 -05:00
' duration ' : 10 ,
2018-11-02 19:26:16 -04:00
' view_count ' : int ,
2014-08-31 12:10:05 -04:00
' like_count ' : int ,
' dislike_count ' : int ,
2015-07-20 15:10:28 -04:00
' start_time ' : 1 ,
2015-07-23 07:20:21 -04:00
' end_time ' : 9 ,
2013-06-27 13:13:11 -04:00
}
2013-06-27 13:55:39 -04:00
} ,
2013-11-18 07:05:18 -05:00
{
2014-09-24 03:49:53 -04:00
' url ' : ' //www.YouTube.com/watch?v=yZIXLfi8CZQ ' ,
' note ' : ' Embed-only video (#1746) ' ,
' info_dict ' : {
' id ' : ' yZIXLfi8CZQ ' ,
' ext ' : ' mp4 ' ,
' upload_date ' : ' 20120608 ' ,
' title ' : ' Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012 ' ,
' description ' : ' md5:09b78bd971f1e3e289601dfba15ca4f7 ' ,
' uploader ' : ' SET India ' ,
2015-11-23 10:35:23 -05:00
' uploader_id ' : ' setindia ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/setindia ' ,
2015-11-23 10:35:23 -05:00
' age_limit ' : 18 ,
2021-02-04 09:37:17 -05:00
} ,
' skip ' : ' Private video ' ,
2013-11-18 07:05:18 -05:00
} ,
2015-08-10 14:52:38 -04:00
{
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ ' ,
2015-08-10 14:52:38 -04:00
' note ' : ' Use the first video ID in the URL ' ,
' info_dict ' : {
' id ' : ' BaW_jenozKc ' ,
' ext ' : ' mp4 ' ,
2020-09-02 16:37:35 -04:00
' title ' : ' youtube-dl test video " \' / \\ ä↭𝕐 ' ,
2015-08-10 14:52:38 -04:00
' uploader ' : ' Philipp Hagemeister ' ,
' uploader_id ' : ' phihag ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/phihag ' ,
2015-08-10 14:52:38 -04:00
' upload_date ' : ' 20121002 ' ,
2020-09-02 16:37:35 -04:00
' description ' : ' test chars: " \' / \\ ä↭𝕐 \n test URL: https://github.com/rg3/youtube-dl/issues/1892 \n \n This is a test video for youtube-dl. \n \n For more information, contact phihag@phihag.de . ' ,
2015-08-10 14:52:38 -04:00
' categories ' : [ ' Science & Technology ' ] ,
2020-09-02 16:37:35 -04:00
' tags ' : [ ' youtube-dl ' ] ,
2017-01-26 09:43:14 -05:00
' duration ' : 10 ,
2018-11-02 19:26:16 -04:00
' view_count ' : int ,
2015-08-10 14:52:38 -04:00
' like_count ' : int ,
' dislike_count ' : int ,
2015-08-10 15:22:06 -04:00
} ,
' params ' : {
' skip_download ' : True ,
} ,
2015-08-10 14:52:38 -04:00
} ,
2014-01-18 23:47:20 -05:00
{
2016-09-17 10:48:20 -04:00
' url ' : ' https://www.youtube.com/watch?v=a9LDPn-MO4I ' ,
2014-09-24 03:49:53 -04:00
' note ' : ' 256k DASH audio (format 141) via DASH manifest ' ,
' info_dict ' : {
' id ' : ' a9LDPn-MO4I ' ,
' ext ' : ' m4a ' ,
' upload_date ' : ' 20121002 ' ,
' uploader_id ' : ' 8KVIDEO ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/8KVIDEO ' ,
2014-09-24 03:49:53 -04:00
' description ' : ' ' ,
' uploader ' : ' 8KVIDEO ' ,
' title ' : ' UHDTV TEST 8K VIDEO.mp4 '
2014-01-22 15:56:37 -05:00
} ,
2014-09-24 03:49:53 -04:00
' params ' : {
' youtube_include_dash_manifest ' : True ,
' format ' : ' 141 ' ,
2014-01-22 15:56:37 -05:00
} ,
2016-06-24 11:27:55 -04:00
' skip ' : ' format 141 not served anymore ' ,
2014-01-18 23:47:20 -05:00
} ,
2020-11-19 14:22:59 -05:00
# DASH manifest with encrypted signature
{
' url ' : ' https://www.youtube.com/watch?v=IB3lcPjvWLA ' ,
' info_dict ' : {
' id ' : ' IB3lcPjvWLA ' ,
' ext ' : ' m4a ' ,
' title ' : ' Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson ' ,
' description ' : ' md5:8f5e2b82460520b619ccac1f509d43bf ' ,
' duration ' : 244 ,
' uploader ' : ' AfrojackVEVO ' ,
' uploader_id ' : ' AfrojackVEVO ' ,
' upload_date ' : ' 20131011 ' ,
2021-02-10 16:22:55 -05:00
' abr ' : 129.495 ,
2020-11-19 14:22:59 -05:00
} ,
' params ' : {
' youtube_include_dash_manifest ' : True ,
' format ' : ' 141/bestaudio[ext=m4a] ' ,
} ,
} ,
2014-11-23 03:59:02 -05:00
# Controversy video
{
' url ' : ' https://www.youtube.com/watch?v=T4XJQO3qol8 ' ,
' info_dict ' : {
' id ' : ' T4XJQO3qol8 ' ,
' ext ' : ' mp4 ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 219 ,
2014-11-23 03:59:02 -05:00
' upload_date ' : ' 20100909 ' ,
2019-01-15 14:18:27 -05:00
' uploader ' : ' Amazing Atheist ' ,
2014-11-23 03:59:02 -05:00
' uploader_id ' : ' TheAmazingAtheist ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/TheAmazingAtheist ' ,
2014-11-23 03:59:02 -05:00
' title ' : ' Burning Everyone \' s Koran ' ,
2021-02-04 09:37:17 -05:00
' description ' : ' SUBSCRIBE: http://www.youtube.com/saturninefilms \r \n \r \n Even Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html ' ,
2014-11-23 03:59:02 -05:00
}
2014-11-30 15:45:49 -05:00
} ,
2020-10-23 23:09:22 -04:00
# Normal age-gate video (embed allowed)
2014-11-30 15:45:49 -05:00
{
2016-09-17 10:48:20 -04:00
' url ' : ' https://youtube.com/watch?v=HtVdAasjOgU ' ,
2014-11-30 15:45:49 -05:00
' info_dict ' : {
' id ' : ' HtVdAasjOgU ' ,
' ext ' : ' mp4 ' ,
' title ' : ' The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer ' ,
2017-01-02 07:08:07 -05:00
' description ' : r ' re:(?s). { 100,}About the Game \ n.*?The Witcher 3: Wild Hunt. { 100,} ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 142 ,
2014-11-30 15:45:49 -05:00
' uploader ' : ' The Witcher ' ,
' uploader_id ' : ' WitcherGame ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/WitcherGame ' ,
2014-11-30 15:45:49 -05:00
' upload_date ' : ' 20140605 ' ,
2015-08-10 15:24:53 -04:00
' age_limit ' : 18 ,
2014-11-30 15:45:49 -05:00
} ,
} ,
2020-11-19 14:22:59 -05:00
# video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
# YouTube Red ad is not captured for creator
{
' url ' : ' __2ABJjxzNo ' ,
' info_dict ' : {
' id ' : ' __2ABJjxzNo ' ,
' ext ' : ' mp4 ' ,
' duration ' : 266 ,
' upload_date ' : ' 20100430 ' ,
' uploader_id ' : ' deadmau5 ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/deadmau5 ' ,
2021-02-04 09:37:17 -05:00
' creator ' : ' deadmau5 ' ,
' description ' : ' md5:6cbcd3a92ce1bc676fc4d6ab4ace2336 ' ,
2020-11-19 14:22:59 -05:00
' uploader ' : ' deadmau5 ' ,
' title ' : ' Deadmau5 - Some Chords (HD) ' ,
2021-02-04 09:37:17 -05:00
' alt_title ' : ' Some Chords ' ,
2020-11-19 14:22:59 -05:00
} ,
' expected_warnings ' : [
' DASH manifest missing ' ,
]
} ,
2019-03-09 07:14:41 -05:00
# Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
2014-12-11 10:28:07 -05:00
{
' url ' : ' lqQg6PlCWgI ' ,
' info_dict ' : {
' id ' : ' lqQg6PlCWgI ' ,
' ext ' : ' mp4 ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 6085 ,
2015-11-23 10:37:21 -05:00
' upload_date ' : ' 20150827 ' ,
2014-12-11 10:34:37 -05:00
' uploader_id ' : ' olympic ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/olympic ' ,
2014-12-11 10:34:37 -05:00
' description ' : ' HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games ' ,
2016-06-24 11:47:19 -04:00
' uploader ' : ' Olympic ' ,
2014-12-11 10:34:37 -05:00
' title ' : ' Hockey - Women - GER-AUS - London 2012 Olympic Games ' ,
} ,
' params ' : {
' skip_download ' : ' requires avconv ' ,
2014-12-11 10:28:07 -05:00
}
2014-12-11 10:34:37 -05:00
} ,
2015-01-09 23:45:51 -05:00
# Non-square pixels
{
' url ' : ' https://www.youtube.com/watch?v=_b-2C3KPAM0 ' ,
' info_dict ' : {
' id ' : ' _b-2C3KPAM0 ' ,
' ext ' : ' mp4 ' ,
' stretched_ratio ' : 16 / 9. ,
2017-01-26 09:43:14 -05:00
' duration ' : 85 ,
2015-01-09 23:45:51 -05:00
' upload_date ' : ' 20110310 ' ,
' uploader_id ' : ' AllenMeow ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/AllenMeow ' ,
2015-01-09 23:45:51 -05:00
' description ' : ' made by Wacom from Korea | 字幕&加油添醋 by TY \' s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯 ' ,
2018-06-02 15:23:45 -04:00
' uploader ' : ' 孫ᄋᄅ ' ,
2015-01-09 23:45:51 -05:00
' title ' : ' [A-made] 變態妍字幕版 太妍 我就是這樣的人 ' ,
} ,
2015-04-05 14:35:55 -04:00
} ,
# url_encoded_fmt_stream_map is empty string
{
' url ' : ' qEJwOuvDf7I ' ,
' info_dict ' : {
' id ' : ' qEJwOuvDf7I ' ,
2015-08-12 11:27:58 -04:00
' ext ' : ' webm ' ,
2015-04-05 14:35:55 -04:00
' title ' : ' Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге ' ,
' description ' : ' ' ,
' upload_date ' : ' 20150404 ' ,
' uploader_id ' : ' spbelect ' ,
' uploader ' : ' Наблюдатели Петербурга ' ,
} ,
' params ' : {
' skip_download ' : ' requires avconv ' ,
2016-01-19 09:56:04 -05:00
} ,
' skip ' : ' This live event has ended. ' ,
2015-04-05 14:35:55 -04:00
} ,
2019-03-09 07:14:41 -05:00
# Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
2015-06-27 04:55:46 -04:00
{
' url ' : ' https://www.youtube.com/watch?v=FIl7x6_3R5Y ' ,
' info_dict ' : {
' id ' : ' FIl7x6_3R5Y ' ,
2018-06-02 15:23:45 -04:00
' ext ' : ' webm ' ,
2015-06-27 04:55:46 -04:00
' title ' : ' md5:7b81415841e02ecd4313668cde88737a ' ,
' description ' : ' md5:116377fd2963b81ec4ce64b542173306 ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 220 ,
2015-06-27 04:55:46 -04:00
' upload_date ' : ' 20150625 ' ,
' uploader_id ' : ' dorappi2000 ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/dorappi2000 ' ,
2015-06-27 04:55:46 -04:00
' uploader ' : ' dorappi2000 ' ,
2018-06-02 15:23:45 -04:00
' formats ' : ' mincount:31 ' ,
2015-06-27 04:55:46 -04:00
} ,
2018-06-02 15:23:45 -04:00
' skip ' : ' not actual anymore ' ,
2015-07-20 13:34:24 -04:00
} ,
2015-06-10 02:47:02 -04:00
# DASH manifest with segment_list
{
' url ' : ' https://www.youtube.com/embed/CsmdDsKjzN8 ' ,
' md5 ' : ' 8ce563a1d667b599d21064e982ab9e31 ' ,
' info_dict ' : {
' id ' : ' CsmdDsKjzN8 ' ,
' ext ' : ' mp4 ' ,
2015-07-20 13:48:50 -04:00
' upload_date ' : ' 20150501 ' , # According to '<meta itemprop="datePublished"', but in other places it's 20150510
2015-06-10 02:47:02 -04:00
' uploader ' : ' Airtek ' ,
' description ' : ' Retransmisión en directo de la XVIII media maratón de Zaragoza. ' ,
' uploader_id ' : ' UCzTzUmjXxxacNnL8I3m4LnQ ' ,
' title ' : ' Retransmisión XVIII Media maratón Zaragoza 2015 ' ,
} ,
' params ' : {
' youtube_include_dash_manifest ' : True ,
' format ' : ' 135 ' , # bestvideo
2016-06-24 11:47:19 -04:00
} ,
' skip ' : ' This live event has ended. ' ,
2015-07-20 13:34:24 -04:00
} ,
2015-07-25 11:30:34 -04:00
{
# Multifeed videos (multiple cameras), URL is for Main Camera
2021-02-04 09:37:17 -05:00
' url ' : ' https://www.youtube.com/watch?v=jvGDaLqkpTg ' ,
2015-07-25 11:30:34 -04:00
' info_dict ' : {
2021-02-04 09:37:17 -05:00
' id ' : ' jvGDaLqkpTg ' ,
' title ' : ' Tom Clancy Free Weekend Rainbow Whatever ' ,
' description ' : ' md5:e03b909557865076822aa169218d6a5d ' ,
2015-07-25 11:30:34 -04:00
} ,
' playlist ' : [ {
' info_dict ' : {
2021-02-04 09:37:17 -05:00
' id ' : ' jvGDaLqkpTg ' ,
2015-07-25 11:30:34 -04:00
' ext ' : ' mp4 ' ,
2021-02-04 09:37:17 -05:00
' title ' : ' Tom Clancy Free Weekend Rainbow Whatever (Main Camera) ' ,
' description ' : ' md5:e03b909557865076822aa169218d6a5d ' ,
' duration ' : 10643 ,
' upload_date ' : ' 20161111 ' ,
' uploader ' : ' Team PGP ' ,
' uploader_id ' : ' UChORY56LMMETTuGjXaJXvLg ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UChORY56LMMETTuGjXaJXvLg ' ,
2015-07-25 11:30:34 -04:00
} ,
} , {
' info_dict ' : {
2021-02-04 09:37:17 -05:00
' id ' : ' 3AKt1R1aDnw ' ,
2015-07-25 11:30:34 -04:00
' ext ' : ' mp4 ' ,
2021-02-04 09:37:17 -05:00
' title ' : ' Tom Clancy Free Weekend Rainbow Whatever (Camera 2) ' ,
' description ' : ' md5:e03b909557865076822aa169218d6a5d ' ,
' duration ' : 10991 ,
' upload_date ' : ' 20161111 ' ,
' uploader ' : ' Team PGP ' ,
' uploader_id ' : ' UChORY56LMMETTuGjXaJXvLg ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UChORY56LMMETTuGjXaJXvLg ' ,
2015-07-25 11:30:34 -04:00
} ,
} , {
' info_dict ' : {
2021-02-04 09:37:17 -05:00
' id ' : ' RtAMM00gpVc ' ,
2015-07-25 11:30:34 -04:00
' ext ' : ' mp4 ' ,
2021-02-04 09:37:17 -05:00
' title ' : ' Tom Clancy Free Weekend Rainbow Whatever (Camera 3) ' ,
' description ' : ' md5:e03b909557865076822aa169218d6a5d ' ,
' duration ' : 10995 ,
' upload_date ' : ' 20161111 ' ,
' uploader ' : ' Team PGP ' ,
' uploader_id ' : ' UChORY56LMMETTuGjXaJXvLg ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UChORY56LMMETTuGjXaJXvLg ' ,
2015-07-25 11:30:34 -04:00
} ,
} , {
' info_dict ' : {
2021-02-04 09:37:17 -05:00
' id ' : ' 6N2fdlP3C5U ' ,
2015-07-25 11:30:34 -04:00
' ext ' : ' mp4 ' ,
2021-02-04 09:37:17 -05:00
' title ' : ' Tom Clancy Free Weekend Rainbow Whatever (Camera 4) ' ,
' description ' : ' md5:e03b909557865076822aa169218d6a5d ' ,
' duration ' : 10990 ,
' upload_date ' : ' 20161111 ' ,
' uploader ' : ' Team PGP ' ,
' uploader_id ' : ' UChORY56LMMETTuGjXaJXvLg ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UChORY56LMMETTuGjXaJXvLg ' ,
2015-07-25 11:30:34 -04:00
} ,
} ] ,
' params ' : {
' skip_download ' : True ,
} ,
2015-08-16 16:04:13 -04:00
} ,
2016-02-12 18:18:58 -05:00
{
2019-03-09 07:14:41 -05:00
# Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
2016-02-12 18:18:58 -05:00
' url ' : ' https://www.youtube.com/watch?v=gVfLd0zydlo ' ,
' info_dict ' : {
' id ' : ' gVfLd0zydlo ' ,
' title ' : ' DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30 ' ,
} ,
' playlist_count ' : 2 ,
2016-06-24 11:47:19 -04:00
' skip ' : ' Not multifeed anymore ' ,
2016-02-12 18:18:58 -05:00
} ,
2015-08-16 16:04:13 -04:00
{
2016-09-17 10:48:20 -04:00
' url ' : ' https://vid.plus/FlRa-iH7PGw ' ,
2015-08-16 16:04:13 -04:00
' only_matching ' : True ,
2015-11-22 07:49:33 -05:00
} ,
2016-04-03 16:26:20 -04:00
{
2016-09-17 10:48:20 -04:00
' url ' : ' https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html ' ,
2016-04-03 16:26:20 -04:00
' only_matching ' : True ,
} ,
2015-11-22 07:49:33 -05:00
{
2019-03-09 07:14:41 -05:00
# Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
2016-01-18 12:19:38 -05:00
# Also tests cut-off URL expansion in video description (see
2019-03-09 07:14:41 -05:00
# https://github.com/ytdl-org/youtube-dl/issues/1892,
# https://github.com/ytdl-org/youtube-dl/issues/8164)
2015-11-22 07:49:33 -05:00
' url ' : ' https://www.youtube.com/watch?v=lsguqyKfVQg ' ,
' info_dict ' : {
' id ' : ' lsguqyKfVQg ' ,
' ext ' : ' mp4 ' ,
' title ' : ' { dark walk}; Loki/AC/Dishonored; collab w/Elflover21 ' ,
2018-06-02 15:23:45 -04:00
' alt_title ' : ' Dark Walk - Position Music ' ,
2015-11-22 07:49:33 -05:00
' description ' : ' md5:8085699c11dc3f597ce0410b0dcbb34a ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 133 ,
2015-11-22 07:49:33 -05:00
' upload_date ' : ' 20151119 ' ,
' uploader_id ' : ' IronSoulElf ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/IronSoulElf ' ,
2015-11-22 07:49:33 -05:00
' uploader ' : ' IronSoulElf ' ,
2018-06-02 15:23:45 -04:00
' creator ' : ' Todd Haberman, Daniel Law Heath and Aaron Kaplan ' ,
' track ' : ' Dark Walk - Position Music ' ,
' artist ' : ' Todd Haberman, Daniel Law Heath and Aaron Kaplan ' ,
2019-04-28 12:37:46 -04:00
' album ' : ' Position Music - Production Music Vol. 143 - Dark Walk ' ,
2015-11-22 07:49:33 -05:00
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2015-11-23 10:02:37 -05:00
{
2019-03-09 07:14:41 -05:00
# Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
2015-11-23 10:02:37 -05:00
' url ' : ' https://www.youtube.com/watch?v=Ms7iBXnlUO8 ' ,
' only_matching ' : True ,
} ,
2015-11-27 19:07:07 -05:00
{
# Video with yt:stretch=17:0
' url ' : ' https://www.youtube.com/watch?v=Q39EVAstoRM ' ,
' info_dict ' : {
' id ' : ' Q39EVAstoRM ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Clash Of Clans#14 Dicas De Ataque Para CV 4 ' ,
' description ' : ' md5:ee18a25c350637c8faff806845bddee9 ' ,
' upload_date ' : ' 20151107 ' ,
' uploader_id ' : ' UCCr7TALkRbo3EtFzETQF1LA ' ,
' uploader ' : ' CH GAMER DROID ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2016-06-24 11:47:19 -04:00
' skip ' : ' This video does not exist. ' ,
2015-11-27 19:07:07 -05:00
} ,
2021-04-16 23:02:33 -04:00
{
# Video with incomplete 'yt:stretch=16:'
' url ' : ' https://www.youtube.com/watch?v=FRhJzUSJbGI ' ,
' only_matching ' : True ,
} ,
2016-03-02 12:07:25 -05:00
{
# Video licensed under Creative Commons
' url ' : ' https://www.youtube.com/watch?v=M4gD1WSo5mA ' ,
' info_dict ' : {
' id ' : ' M4gD1WSo5mA ' ,
' ext ' : ' mp4 ' ,
' title ' : ' md5:e41008789470fc2533a3252216f1c1d1 ' ,
' description ' : ' md5:a677553cf0840649b731a3024aeff4cc ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 721 ,
2016-03-02 12:07:25 -05:00
' upload_date ' : ' 20150127 ' ,
' uploader_id ' : ' BerkmanCenter ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/BerkmanCenter ' ,
2017-01-26 09:43:14 -05:00
' uploader ' : ' The Berkman Klein Center for Internet & Society ' ,
2016-03-02 12:07:25 -05:00
' license ' : ' Creative Commons Attribution license (reuse allowed) ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2016-03-02 12:49:10 -05:00
{
# Channel-like uploader_url
' url ' : ' https://www.youtube.com/watch?v=eQcmzGIKrzg ' ,
' info_dict ' : {
' id ' : ' eQcmzGIKrzg ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Democratic Socialism and Foreign Policy | Bernie Sanders ' ,
2021-02-04 09:37:17 -05:00
' description ' : ' md5:13a2503d7b5904ef4b223aa101628f39 ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 4060 ,
2016-03-02 12:49:10 -05:00
' upload_date ' : ' 20151119 ' ,
2018-06-02 15:23:45 -04:00
' uploader ' : ' Bernie Sanders ' ,
2016-03-02 12:49:10 -05:00
' uploader_id ' : ' UCH1dpzjCEiGAt8CXkryhkZg ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCH1dpzjCEiGAt8CXkryhkZg ' ,
2016-03-02 12:49:10 -05:00
' license ' : ' Creative Commons Attribution license (reuse allowed) ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2015-11-29 10:01:59 -05:00
{
' url ' : ' https://www.youtube.com/watch?feature=player_embedded&amp;v=V36LpHqtcDY ' ,
' only_matching ' : True ,
2016-07-11 15:10:35 -04:00
} ,
{
2019-03-09 07:14:41 -05:00
# YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
2016-07-11 15:10:35 -04:00
' url ' : ' https://www.youtube.com/watch?v=i1Ko8UG-Tdo ' ,
' only_matching ' : True ,
2016-09-02 14:17:15 -04:00
} ,
{
# Rental video preview
' url ' : ' https://www.youtube.com/watch?v=yYr8q0y5Jfg ' ,
' info_dict ' : {
' id ' : ' uGpuVWrhIzE ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Piku - Trailer ' ,
' description ' : ' md5:c36bd60c3fd6f1954086c083c72092eb ' ,
' upload_date ' : ' 20150811 ' ,
' uploader ' : ' FlixMatrix ' ,
' uploader_id ' : ' FlixMatrixKaravan ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/FlixMatrixKaravan ' ,
2016-09-02 14:17:15 -04:00
' license ' : ' Standard YouTube License ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2018-06-02 15:23:45 -04:00
' skip ' : ' This video is not available. ' ,
2017-01-09 10:30:46 -05:00
} ,
2017-01-21 06:10:32 -05:00
{
# YouTube Red video with episode data
' url ' : ' https://www.youtube.com/watch?v=iqKdEhx-dD4 ' ,
' info_dict ' : {
' id ' : ' iqKdEhx-dD4 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Isolation - Mind Field (Ep 1) ' ,
2021-02-04 09:37:17 -05:00
' description ' : ' md5:f540112edec5d09fc8cc752d3d4ba3cd ' ,
2017-01-26 09:43:14 -05:00
' duration ' : 2085 ,
2017-01-21 06:10:32 -05:00
' upload_date ' : ' 20170118 ' ,
' uploader ' : ' Vsauce ' ,
' uploader_id ' : ' Vsauce ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/Vsauce ' ,
' series ' : ' Mind Field ' ,
' season_number ' : 1 ,
' episode_number ' : 1 ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
' expected_warnings ' : [
' Skipping DASH manifest ' ,
] ,
} ,
2017-08-26 04:38:38 -04:00
{
# The following content has been identified by the YouTube community
# as inappropriate or offensive to some audiences.
' url ' : ' https://www.youtube.com/watch?v=6SJNVb0GnPI ' ,
' info_dict ' : {
' id ' : ' 6SJNVb0GnPI ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Race Differences in Intelligence ' ,
' description ' : ' md5:5d161533167390427a1f8ee89a1fc6f1 ' ,
' duration ' : 965 ,
' upload_date ' : ' 20140124 ' ,
' uploader ' : ' New Century Foundation ' ,
' uploader_id ' : ' UCEJYpZGqgUob0zVVEaLhvVg ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCEJYpZGqgUob0zVVEaLhvVg ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2021-02-04 09:37:17 -05:00
' skip ' : ' This video has been removed for violating YouTube \' s policy on hate speech. ' ,
2017-08-26 04:38:38 -04:00
} ,
2017-01-09 10:30:46 -05:00
{
# itag 212
' url ' : ' 1t24XAntNCY ' ,
' only_matching ' : True ,
2017-02-26 04:51:21 -05:00
} ,
{
# geo restricted to JP
' url ' : ' sJL6WA-aGkQ ' ,
' only_matching ' : True ,
} ,
2018-09-23 13:14:49 -04:00
{
' url ' : ' https://invidio.us/watch?v=BaW_jenozKc ' ,
' only_matching ' : True ,
} ,
2021-02-19 15:44:36 -05:00
{
' url ' : ' https://redirect.invidious.io/watch?v=BaW_jenozKc ' ,
' only_matching ' : True ,
} ,
{
# from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
' url ' : ' https://redirect.invidious.io/Yh0AhrY9GjA ' ,
' only_matching ' : True ,
} ,
2018-12-26 03:30:48 -05:00
{
# DRM protected
' url ' : ' https://www.youtube.com/watch?v=s7_qI6_mIXc ' ,
' only_matching ' : True ,
2019-01-15 14:18:27 -05:00
} ,
{
# Video with unsupported adaptive stream type formats
' url ' : ' https://www.youtube.com/watch?v=Z4Vy8R84T1U ' ,
' info_dict ' : {
' id ' : ' Z4Vy8R84T1U ' ,
' ext ' : ' mp4 ' ,
' title ' : ' saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta ' ,
' description ' : ' md5:d41d8cd98f00b204e9800998ecf8427e ' ,
' duration ' : 433 ,
' upload_date ' : ' 20130923 ' ,
' uploader ' : ' Amelia Putri Harwita ' ,
' uploader_id ' : ' UCpOxM49HJxmC1qCalXyB3_Q ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCpOxM49HJxmC1qCalXyB3_Q ' ,
' formats ' : ' maxcount:10 ' ,
} ,
' params ' : {
' skip_download ' : True ,
' youtube_include_dash_manifest ' : False ,
} ,
2020-03-05 12:05:50 -05:00
' skip ' : ' not actual anymore ' ,
2019-04-22 00:26:48 -04:00
} ,
{
2019-04-27 04:16:17 -04:00
# Youtube Music Auto-generated description
2019-04-22 00:26:48 -04:00
' url ' : ' https://music.youtube.com/watch?v=MgNrAu2pzNs ' ,
' info_dict ' : {
' id ' : ' MgNrAu2pzNs ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Voyeur Girl ' ,
' description ' : ' md5:7ae382a65843d6df2685993e90a8628f ' ,
' upload_date ' : ' 20190312 ' ,
2020-03-05 12:05:50 -05:00
' uploader ' : ' Stephen - Topic ' ,
' uploader_id ' : ' UC-pWHpBjdGG69N9mM2auIAA ' ,
2019-04-22 00:26:48 -04:00
' artist ' : ' Stephen ' ,
' track ' : ' Voyeur Girl ' ,
' album ' : ' it \' s too much love to know my dear ' ,
' release_date ' : ' 20190313 ' ,
' release_year ' : 2019 ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2019-11-30 11:51:34 -05:00
{
' url ' : ' https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q ' ,
' only_matching ' : True ,
} ,
2020-04-30 13:40:38 -04:00
{
# invalid -> valid video id redirection
' url ' : ' DJztXj2GPfl ' ,
' info_dict ' : {
' id ' : ' DJztXj2GPfk ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack) ' ,
' description ' : ' md5:bf577a41da97918e94fa9798d9228825 ' ,
' upload_date ' : ' 20090125 ' ,
' uploader ' : ' Prochorowka ' ,
' uploader_id ' : ' Prochorowka ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/Prochorowka ' ,
' artist ' : ' Panjabi MC ' ,
' track ' : ' Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix ' ,
' album ' : ' Beware of the Boys (Mundian To Bach Ke) ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2021-02-04 09:37:17 -05:00
' skip ' : ' Video unavailable ' ,
2020-09-13 10:23:21 -04:00
} ,
{
# empty description results in an empty string
' url ' : ' https://www.youtube.com/watch?v=x41yOUIvK2k ' ,
' info_dict ' : {
' id ' : ' x41yOUIvK2k ' ,
' ext ' : ' mp4 ' ,
' title ' : ' IMG 3456 ' ,
' description ' : ' ' ,
' upload_date ' : ' 20170613 ' ,
' uploader_id ' : ' ElevageOrVert ' ,
' uploader ' : ' ElevageOrVert ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2020-11-21 09:50:42 -05:00
{
2021-01-01 07:26:37 -05:00
# with '};' inside yt initial data (see [1])
# see [2] for an example with '};' inside ytInitialPlayerResponse
# 1. https://github.com/ytdl-org/youtube-dl/issues/27093
# 2. https://github.com/ytdl-org/youtube-dl/issues/27216
2020-11-21 09:50:42 -05:00
' url ' : ' https://www.youtube.com/watch?v=CHqg6qOn4no ' ,
' info_dict ' : {
' id ' : ' CHqg6qOn4no ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Part 77 Sort a list of simple types in c# ' ,
' description ' : ' md5:b8746fa52e10cdbf47997903f13b20dc ' ,
' upload_date ' : ' 20130831 ' ,
' uploader_id ' : ' kudvenkat ' ,
' uploader ' : ' kudvenkat ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
} ,
2021-01-01 07:26:37 -05:00
{
# another example of '};' in ytInitialData
' url ' : ' https://www.youtube.com/watch?v=gVfgbahppCY ' ,
' only_matching ' : True ,
} ,
{
' url ' : ' https://www.youtube.com/watch_popup?v=63RmMXCd_bQ ' ,
' only_matching ' : True ,
} ,
2021-02-04 09:37:17 -05:00
{
2021-02-10 16:22:55 -05:00
# https://github.com/ytdl-org/youtube-dl/pull/28094
' url ' : ' OtqTfy26tG0 ' ,
' info_dict ' : {
' id ' : ' OtqTfy26tG0 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Burn Out ' ,
' description ' : ' md5:8d07b84dcbcbfb34bc12a56d968b6131 ' ,
' upload_date ' : ' 20141120 ' ,
' uploader ' : ' The Cinematic Orchestra - Topic ' ,
' uploader_id ' : ' UCIzsJBIyo8hhpFm1NK0uLgw ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCIzsJBIyo8hhpFm1NK0uLgw ' ,
' artist ' : ' The Cinematic Orchestra ' ,
' track ' : ' Burn Out ' ,
' album ' : ' Every Day ' ,
' release_data ' : None ,
' release_year ' : None ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2021-02-04 09:37:17 -05:00
} ,
2021-02-19 15:44:36 -05:00
{
# controversial video, only works with bpctr when authenticated with cookies
' url ' : ' https://www.youtube.com/watch?v=nGC3D_FkCmg ' ,
' only_matching ' : True ,
} ,
2021-04-10 12:47:11 -04:00
{
# restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
' url ' : ' cBvYw8_A0vQ ' ,
' info_dict ' : {
' id ' : ' cBvYw8_A0vQ ' ,
' ext ' : ' mp4 ' ,
' title ' : ' 4K Ueno Okachimachi Street Scenes 上野御徒町歩き ' ,
' description ' : ' md5:ea770e474b7cd6722b4c95b833c03630 ' ,
' upload_date ' : ' 20201120 ' ,
' uploader ' : ' Walk around Japan ' ,
' uploader_id ' : ' UC3o_t8PzBmXf5S9b7GLx1Mw ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw ' ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2021-05-14 03:01:53 -04:00
} , {
# Has multiple audio streams
' url ' : ' WaOKSUlf4TM ' ,
' only_matching ' : True
2021-05-15 10:38:47 -04:00
} , {
# Requires Premium: has format 141 when requested using YTM url
' url ' : ' https://music.youtube.com/watch?v=XclachpHxis ' ,
' only_matching ' : True
} , {
2021-05-12 15:20:02 -04:00
# multiple subtitles with same lang_code
' url ' : ' https://www.youtube.com/watch?v=wsQiKKfKxug ' ,
' only_matching ' : True ,
2021-06-29 18:07:49 -04:00
} , {
# Force use android client fallback
' url ' : ' https://www.youtube.com/watch?v=YOelRv7fMxY ' ,
' info_dict ' : {
' id ' : ' YOelRv7fMxY ' ,
' title ' : ' Digging a Secret Tunnel from my Workshop ' ,
' ext ' : ' 3gp ' ,
' upload_date ' : ' 20210624 ' ,
' channel_id ' : ' UCp68_FLety0O-n9QU6phsgw ' ,
' uploader ' : ' colinfurze ' ,
' channel_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCp68_FLety0O-n9QU6phsgw ' ,
' description ' : ' md5:ecb672623246d98c6c562eed6ae798c3 '
} ,
' params ' : {
' format ' : ' 17 ' , # 3gp format available on android
' extractor_args ' : { ' youtube ' : { ' player_client ' : [ ' android ' ] } } ,
} ,
2021-05-12 15:20:02 -04:00
} ,
2021-06-29 18:07:49 -04:00
{
# Skip download of additional client configs (remix client config in this case)
' url ' : ' https://music.youtube.com/watch?v=MgNrAu2pzNs ' ,
' only_matching ' : True ,
' params ' : {
' extractor_args ' : { ' youtube ' : { ' player_skip ' : [ ' configs ' ] } } ,
} ,
}
2013-06-27 13:13:11 -04:00
]
2021-04-16 23:02:33 -04:00
@classmethod
def suitable ( cls , url ) :
2021-04-22 05:02:54 -04:00
# Hack for lazy extractors until more generic solution is implemented
# (see #28780)
from . youtube import parse_qs
2021-04-16 23:02:33 -04:00
qs = parse_qs ( url )
if qs . get ( ' list ' , [ None ] ) [ 0 ] :
return False
return super ( YoutubeIE , cls ) . suitable ( url )
2013-09-21 08:19:30 -04:00
def __init__ ( self , * args , * * kwargs ) :
super ( YoutubeIE , self ) . __init__ ( * args , * * kwargs )
2021-02-04 09:37:17 -05:00
self . _code_cache = { }
2013-09-21 09:19:48 -04:00
self . _player_cache = { }
2013-09-21 08:19:30 -04:00
2021-06-29 18:07:49 -04:00
def _extract_player_url ( self , ytcfg = None , webpage = None ) :
player_url = try_get ( ytcfg , ( lambda x : x [ ' PLAYER_JS_URL ' ] ) , str )
if not player_url :
player_url = self . _search_regex (
r ' " (?:PLAYER_JS_URL|jsUrl) " \ s*: \ s* " ([^ " ]+) " ' ,
webpage , ' player URL ' , fatal = False )
if player_url . startswith ( ' // ' ) :
player_url = ' https: ' + player_url
elif not re . match ( r ' https?:// ' , player_url ) :
player_url = compat_urlparse . urljoin (
' https://www.youtube.com ' , player_url )
return player_url
2014-08-02 06:21:53 -04:00
def _signature_cache_id ( self , example_sig ) :
""" Return a string representation of a signature """
2014-09-13 01:51:06 -04:00
return ' . ' . join ( compat_str ( len ( part ) ) for part in example_sig . split ( ' . ' ) )
2014-08-02 06:21:53 -04:00
2020-05-01 20:18:08 -04:00
@classmethod
def _extract_player_info ( cls , player_url ) :
for player_re in cls . _PLAYER_INFO_RE :
id_m = re . search ( player_re , player_url )
if id_m :
break
else :
2014-07-22 20:19:33 -04:00
raise ExtractorError ( ' Cannot identify player %r ' % player_url )
2021-02-04 09:37:17 -05:00
return id_m . group ( ' id ' )
2020-05-01 20:18:08 -04:00
2021-06-29 18:07:49 -04:00
def _load_player ( self , video_id , player_url , fatal = True ) - > bool :
player_id = self . _extract_player_info ( player_url )
if player_id not in self . _code_cache :
self . _code_cache [ player_id ] = self . _download_webpage (
player_url , video_id , fatal = fatal ,
note = ' Downloading player ' + player_id ,
errnote = ' Download of %s failed ' % player_url )
return player_id in self . _code_cache
2020-05-01 20:18:08 -04:00
def _extract_signature_function ( self , video_id , player_url , example_sig ) :
2021-02-04 09:37:17 -05:00
player_id = self . _extract_player_info ( player_url )
2013-09-21 08:19:30 -04:00
2013-09-21 18:35:03 -04:00
# Read from filesystem cache
2021-02-04 09:37:17 -05:00
func_id = ' js_ %s _ %s ' % (
player_id , self . _signature_cache_id ( example_sig ) )
2013-09-21 18:35:03 -04:00
assert os . path . basename ( func_id ) == func_id
2014-09-03 06:41:05 -04:00
2014-09-24 03:51:45 -04:00
cache_spec = self . _downloader . cache . load ( ' youtube-sigfuncs ' , func_id )
2014-09-03 06:41:05 -04:00
if cache_spec is not None :
2014-09-13 01:51:06 -04:00
return lambda s : ' ' . join ( s [ i ] for i in cache_spec )
2013-09-21 09:19:48 -04:00
2021-06-29 18:07:49 -04:00
if self . _load_player ( video_id , player_url ) :
code = self . _code_cache [ player_id ]
res = self . _parse_sig_js ( code )
2013-09-21 08:19:30 -04:00
2021-06-29 18:07:49 -04:00
test_string = ' ' . join ( map ( compat_chr , range ( len ( example_sig ) ) ) )
cache_res = res ( test_string )
cache_spec = [ ord ( c ) for c in cache_res ]
2013-09-21 09:19:48 -04:00
2021-06-29 18:07:49 -04:00
self . _downloader . cache . store ( ' youtube-sigfuncs ' , func_id , cache_spec )
return res
2013-09-21 09:19:48 -04:00
2014-08-02 06:21:53 -04:00
def _print_sig_code ( self , func , example_sig ) :
2013-09-22 04:30:02 -04:00
def gen_sig_code ( idxs ) :
def _genslice ( start , end , step ) :
2014-09-13 01:51:06 -04:00
starts = ' ' if start == 0 else str ( start )
2014-11-23 15:20:46 -05:00
ends = ( ' : %d ' % ( end + step ) ) if end + step > = 0 else ' : '
2014-09-24 03:51:45 -04:00
steps = ' ' if step == 1 else ( ' : %d ' % step )
2014-09-13 01:51:06 -04:00
return ' s[ %s %s %s ] ' % ( starts , ends , steps )
2013-09-22 04:30:02 -04:00
step = None
2014-12-16 18:06:41 -05:00
# Quelch pyflakes warnings - start will be set when step is set
start = ' (Never used) '
2013-09-22 04:30:02 -04:00
for i , prev in zip ( idxs [ 1 : ] , idxs [ : - 1 ] ) :
if step is not None :
if i - prev == step :
continue
yield _genslice ( start , prev , step )
step = None
continue
if i - prev in [ - 1 , 1 ] :
step = i - prev
start = prev
continue
else :
2014-09-13 01:51:06 -04:00
yield ' s[ %d ] ' % prev
2013-09-22 04:30:02 -04:00
if step is None :
2014-09-13 01:51:06 -04:00
yield ' s[ %d ] ' % i
2013-09-22 04:30:02 -04:00
else :
yield _genslice ( start , i , step )
2014-09-13 01:51:06 -04:00
test_string = ' ' . join ( map ( compat_chr , range ( len ( example_sig ) ) ) )
2013-09-22 06:18:16 -04:00
cache_res = func ( test_string )
2013-09-22 04:30:02 -04:00
cache_spec = [ ord ( c ) for c in cache_res ]
2014-09-13 01:51:06 -04:00
expr_code = ' + ' . join ( gen_sig_code ( cache_spec ) )
2014-08-02 06:21:53 -04:00
signature_id_tuple = ' ( %s ) ' % (
' , ' . join ( compat_str ( len ( p ) ) for p in example_sig . split ( ' . ' ) ) )
2014-09-24 03:51:45 -04:00
code = ( ' if tuple(len(p) for p in s.split( \' . \' )) == %s : \n '
2014-09-13 01:51:06 -04:00
' return %s \n ' ) % ( signature_id_tuple , expr_code )
2014-09-24 03:51:45 -04:00
self . to_screen ( ' Extracted signature function: \n ' + code )
2013-09-22 04:30:02 -04:00
2013-09-21 08:19:30 -04:00
def _parse_sig_js ( self , jscode ) :
funcname = self . _search_regex (
2019-06-21 11:58:42 -04:00
( r ' \ b[cs] \ s*&& \ s*[adf] \ .set \ ([^,]+ \ s*, \ s*encodeURIComponent \ s* \ ( \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ b[a-zA-Z0-9]+ \ s*&& \ s*[a-zA-Z0-9]+ \ .set \ ([^,]+ \ s*, \ s*encodeURIComponent \ s* \ ( \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
2021-02-10 16:22:55 -05:00
r ' \ bm=(?P<sig>[a-zA-Z0-9$] {2} ) \ (decodeURIComponent \ (h \ .s \ ) \ ) ' ,
r ' \ bc&& \ (c=(?P<sig>[a-zA-Z0-9$] {2} ) \ (decodeURIComponent \ (c \ ) \ ) ' ,
r ' (?: \ b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$] {2} ) \ s*= \ s*function \ ( \ s*a \ s* \ ) \ s* { \ s*a \ s*= \ s*a \ .split \ ( \ s* " " \ s* \ );[a-zA-Z0-9$] {2} \ .[a-zA-Z0-9$] {2} \ (a, \ d+ \ ) ' ,
2020-07-27 18:04:50 -04:00
r ' (?: \ b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$] {2} ) \ s*= \ s*function \ ( \ s*a \ s* \ ) \ s* { \ s*a \ s*= \ s*a \ .split \ ( \ s* " " \ s* \ ) ' ,
2019-06-21 15:22:07 -04:00
r ' (?P<sig>[a-zA-Z0-9$]+) \ s*= \ s*function \ ( \ s*a \ s* \ ) \ s* { \ s*a \ s*= \ s*a \ .split \ ( \ s* " " \ s* \ ) ' ,
2019-06-21 11:58:42 -04:00
# Obsolete patterns
r ' ([ " \' ])signature \ 1 \ s*, \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
2018-09-07 16:36:10 -04:00
r ' \ .sig \ | \ |(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
2019-06-21 11:58:42 -04:00
r ' yt \ .akamaized \ .net/ \ ) \ s* \ | \ | \ s*.*? \ s*[cs] \ s*&& \ s*[adf] \ .set \ ([^,]+ \ s*, \ s*(?:encodeURIComponent \ s* \ ()? \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ b[cs] \ s*&& \ s*[adf] \ .set \ ([^,]+ \ s*, \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ b[a-zA-Z0-9]+ \ s*&& \ s*[a-zA-Z0-9]+ \ .set \ ([^,]+ \ s*, \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ bc \ s*&& \ s*a \ .set \ ([^,]+ \ s*, \ s* \ ([^)]* \ ) \ s* \ ( \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ bc \ s*&& \ s*[a-zA-Z0-9]+ \ .set \ ([^,]+ \ s*, \ s* \ ([^)]* \ ) \ s* \ ( \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ,
r ' \ bc \ s*&& \ s*[a-zA-Z0-9]+ \ .set \ ([^,]+ \ s*, \ s* \ ([^)]* \ ) \ s* \ ( \ s*(?P<sig>[a-zA-Z0-9$]+) \ ( ' ) ,
2017-01-31 10:19:29 -05:00
jscode , ' Initial JS player signature function name ' , group = ' sig ' )
2014-03-30 01:02:58 -04:00
jsi = JSInterpreter ( jscode )
initial_function = jsi . extract_function ( funcname )
2013-09-21 08:19:30 -04:00
return lambda s : initial_function ( [ s ] )
2021-02-04 09:37:17 -05:00
def _decrypt_signature ( self , s , video_id , player_url ) :
2013-06-27 16:20:50 -04:00
""" Turn the encrypted s field into a working signature """
2013-06-26 19:51:10 -04:00
2014-07-11 04:44:39 -04:00
if player_url is None :
2014-09-24 03:51:45 -04:00
raise ExtractorError ( ' Cannot decrypt signature without player_url ' )
2013-09-27 00:15:21 -04:00
2014-07-11 04:44:39 -04:00
try :
2014-08-02 06:23:18 -04:00
player_id = ( player_url , self . _signature_cache_id ( s ) )
2014-07-11 04:44:39 -04:00
if player_id not in self . _player_cache :
func = self . _extract_signature_function (
2014-08-02 06:21:53 -04:00
video_id , player_url , s
2014-07-11 04:44:39 -04:00
)
self . _player_cache [ player_id ] = func
func = self . _player_cache [ player_id ]
2021-05-17 08:23:08 -04:00
if self . get_param ( ' youtube_print_sig_code ' ) :
2014-08-02 06:21:53 -04:00
self . _print_sig_code ( func , s )
2014-07-11 04:44:39 -04:00
return func ( s )
except Exception as e :
tb = traceback . format_exc ( )
raise ExtractorError (
2014-09-13 01:51:06 -04:00
' Signature extraction failed: ' + tb , cause = e )
2013-09-21 08:19:30 -04:00
2021-06-29 18:07:49 -04:00
def _extract_signature_timestamp ( self , video_id , player_url , ytcfg = None , fatal = False ) :
"""
Extract signatureTimestamp ( sts )
Required to tell API what sig / player version is in use .
"""
sts = None
if isinstance ( ytcfg , dict ) :
sts = int_or_none ( ytcfg . get ( ' STS ' ) )
if not sts :
# Attempt to extract from player
if player_url is None :
error_msg = ' Cannot extract signature timestamp without player_url. '
if fatal :
raise ExtractorError ( error_msg )
self . report_warning ( error_msg )
return
if self . _load_player ( video_id , player_url , fatal = fatal ) :
player_id = self . _extract_player_info ( player_url )
code = self . _code_cache [ player_id ]
sts = int_or_none ( self . _search_regex (
r ' (?:signatureTimestamp|sts) \ s*: \ s*(?P<sts>[0-9] {5} ) ' , code ,
' JS player signature timestamp ' , group = ' sts ' , fatal = fatal ) )
return sts
2021-02-04 09:37:17 -05:00
def _mark_watched ( self , video_id , player_response ) :
2018-12-16 07:35:48 -05:00
playback_url = url_or_none ( try_get (
player_response ,
2021-02-04 09:37:17 -05:00
lambda x : x [ ' playbackTracking ' ] [ ' videostatsPlaybackUrl ' ] [ ' baseUrl ' ] ) )
2016-02-29 14:01:33 -05:00
if not playback_url :
return
parsed_playback_url = compat_urlparse . urlparse ( playback_url )
qs = compat_urlparse . parse_qs ( parsed_playback_url . query )
# cpn generation algorithm is reverse engineered from base.js.
# In fact it works even with dummy cpn.
CPN_ALPHABET = ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_ '
cpn = ' ' . join ( ( CPN_ALPHABET [ random . randint ( 0 , 256 ) & 63 ] for _ in range ( 0 , 16 ) ) )
qs . update ( {
' ver ' : [ ' 2 ' ] ,
' cpn ' : [ cpn ] ,
} )
playback_url = compat_urlparse . urlunparse (
2016-03-25 15:46:57 -04:00
parsed_playback_url . _replace ( query = compat_urllib_parse_urlencode ( qs , True ) ) )
2016-02-29 14:01:33 -05:00
self . _download_webpage (
playback_url , video_id , ' Marking watched ' ,
' Unable to mark watched ' , fatal = False )
2017-09-05 13:48:37 -04:00
@staticmethod
def _extract_urls ( webpage ) :
# Embedded YouTube player
entries = [
unescapeHTML ( mobj . group ( ' url ' ) )
for mobj in re . finditer ( r ''' (?x)
( ? :
< iframe [ ^ > ] + ? src = |
data - video - url = |
< embed [ ^ > ] + ? src = |
embedSWF \( ? : \s * |
< object [ ^ > ] + data = |
new \s + SWFObject \(
)
( [ " \' ])
( ? P < url > ( ? : https ? : ) ? / / ( ? : www \. ) ? youtube ( ? : - nocookie ) ? \. com /
2017-10-27 11:26:43 -04:00
( ? : embed | v | p ) / [ 0 - 9 A - Za - z_ - ] { 11 } . * ? )
2017-09-05 13:48:37 -04:00
\1 ''' , webpage)]
# lazyYT YouTube embed
entries . extend ( list ( map (
unescapeHTML ,
re . findall ( r ' class= " lazyYT " data-youtube-id= " ([^ " ]+) " ' , webpage ) ) ) )
# Wordpress "YouTube Video Importer" plugin
matches = re . findall ( r ''' (?x)<div[^>]+
class = ( ? P < q1 > [ \' " ])[^ \' " ]* \b yvii_single_video_player \b [^ \' " ]*(?P=q1)[^>]+
data - video_id = ( ? P < q2 > [ \' " ])([^ \' " ]+)(?P=q2) ' ' ' , webpage )
entries . extend ( m [ - 1 ] for m in matches )
return entries
@staticmethod
def _extract_url ( webpage ) :
urls = YoutubeIE . _extract_urls ( webpage )
return urls [ 0 ] if urls else None
2014-02-08 13:20:11 -05:00
@classmethod
def extract_id ( cls , url ) :
mobj = re . match ( cls . _VALID_URL , url , re . VERBOSE )
2013-06-23 13:58:33 -04:00
if mobj is None :
2014-09-24 03:51:45 -04:00
raise ExtractorError ( ' Invalid URL: %s ' % url )
2013-06-23 13:58:33 -04:00
video_id = mobj . group ( 2 )
return video_id
2021-02-04 09:37:17 -05:00
def _extract_chapters_from_json ( self , data , video_id , duration ) :
2020-06-05 17:16:31 -04:00
chapters_list = try_get (
2020-11-19 14:22:59 -05:00
data ,
2020-06-05 17:16:31 -04:00
lambda x : x [ ' playerOverlays ' ]
[ ' playerOverlayRenderer ' ]
[ ' decoratedPlayerBarRenderer ' ]
[ ' decoratedPlayerBarRenderer ' ]
[ ' playerBar ' ]
[ ' chapteredPlayerBarRenderer ' ]
[ ' chapters ' ] ,
list )
if not chapters_list :
return
def chapter_time ( chapter ) :
return float_or_none (
try_get (
chapter ,
lambda x : x [ ' chapterRenderer ' ] [ ' timeRangeStartMillis ' ] ,
int ) ,
scale = 1000 )
chapters = [ ]
for next_num , chapter in enumerate ( chapters_list , start = 1 ) :
start_time = chapter_time ( chapter )
if start_time is None :
continue
end_time = ( chapter_time ( chapters_list [ next_num ] )
if next_num < len ( chapters_list ) else duration )
if end_time is None :
continue
title = try_get (
chapter , lambda x : x [ ' chapterRenderer ' ] [ ' title ' ] [ ' simpleText ' ] ,
compat_str )
chapters . append ( {
' start_time ' : start_time ,
' end_time ' : end_time ,
' title ' : title ,
} )
return chapters
2021-02-04 09:37:17 -05:00
def _extract_yt_initial_variable ( self , webpage , regex , video_id , name ) :
return self . _parse_json ( self . _search_regex (
( r ' %s \ s* %s ' % ( regex , self . _YT_INITIAL_BOUNDARY_RE ) ,
regex ) , webpage , name , default = ' {} ' ) , video_id , fatal = False )
2020-06-05 17:16:31 -04:00
2021-04-07 07:37:43 -04:00
@staticmethod
def parse_time_text ( time_text ) :
"""
Parse the comment time text
time_text is in the format ' X units ago (edited) '
"""
time_text_split = time_text . split ( ' ' )
if len ( time_text_split ) > = 3 :
return datetime_from_str ( ' now- %s %s ' % ( time_text_split [ 0 ] , time_text_split [ 1 ] ) , precision = ' auto ' )
2021-03-14 18:41:11 -04:00
@staticmethod
def _join_text_entries ( runs ) :
text = None
for run in runs :
if not isinstance ( run , dict ) :
continue
sub_text = try_get ( run , lambda x : x [ ' text ' ] , compat_str )
if sub_text :
if not text :
text = sub_text
continue
text + = sub_text
return text
def _extract_comment ( self , comment_renderer , parent = None ) :
comment_id = comment_renderer . get ( ' commentId ' )
if not comment_id :
return
comment_text_runs = try_get ( comment_renderer , lambda x : x [ ' contentText ' ] [ ' runs ' ] ) or [ ]
text = self . _join_text_entries ( comment_text_runs ) or ' '
comment_time_text = try_get ( comment_renderer , lambda x : x [ ' publishedTimeText ' ] [ ' runs ' ] ) or [ ]
time_text = self . _join_text_entries ( comment_time_text )
2021-04-07 07:37:43 -04:00
timestamp = calendar . timegm ( self . parse_time_text ( time_text ) . timetuple ( ) )
2021-03-14 18:41:11 -04:00
author = try_get ( comment_renderer , lambda x : x [ ' authorText ' ] [ ' simpleText ' ] , compat_str )
author_id = try_get ( comment_renderer ,
lambda x : x [ ' authorEndpoint ' ] [ ' browseEndpoint ' ] [ ' browseId ' ] , compat_str )
votes = str_to_int ( try_get ( comment_renderer , ( lambda x : x [ ' voteCount ' ] [ ' simpleText ' ] ,
lambda x : x [ ' likeCount ' ] ) , compat_str ) ) or 0
author_thumbnail = try_get ( comment_renderer ,
lambda x : x [ ' authorThumbnail ' ] [ ' thumbnails ' ] [ - 1 ] [ ' url ' ] , compat_str )
author_is_uploader = try_get ( comment_renderer , lambda x : x [ ' authorIsChannelOwner ' ] , bool )
is_liked = try_get ( comment_renderer , lambda x : x [ ' isLiked ' ] , bool )
return {
' id ' : comment_id ,
' text ' : text ,
2021-04-07 07:37:43 -04:00
' timestamp ' : timestamp ,
2021-03-14 18:41:11 -04:00
' time_text ' : time_text ,
' like_count ' : votes ,
' is_favorited ' : is_liked ,
' author ' : author ,
' author_id ' : author_id ,
' author_thumbnail ' : author_thumbnail ,
' author_is_uploader ' : author_is_uploader ,
' parent ' : parent or ' root '
}
def _comment_entries ( self , root_continuation_data , identity_token , account_syncid ,
2021-04-11 16:23:04 -04:00
ytcfg , session_token_list , parent = None , comment_counts = None ) :
2021-03-14 18:41:11 -04:00
def extract_thread ( parent_renderer ) :
contents = try_get ( parent_renderer , lambda x : x [ ' contents ' ] , list ) or [ ]
if not parent :
comment_counts [ 2 ] = 0
for content in contents :
comment_thread_renderer = try_get ( content , lambda x : x [ ' commentThreadRenderer ' ] )
comment_renderer = try_get (
comment_thread_renderer , ( lambda x : x [ ' comment ' ] [ ' commentRenderer ' ] , dict ) ) or try_get (
content , ( lambda x : x [ ' commentRenderer ' ] , dict ) )
if not comment_renderer :
continue
comment = self . _extract_comment ( comment_renderer , parent )
if not comment :
continue
comment_counts [ 0 ] + = 1
yield comment
# Attempt to get the replies
comment_replies_renderer = try_get (
comment_thread_renderer , lambda x : x [ ' replies ' ] [ ' commentRepliesRenderer ' ] , dict )
if comment_replies_renderer :
comment_counts [ 2 ] + = 1
comment_entries_iter = self . _comment_entries (
2021-04-11 16:23:04 -04:00
comment_replies_renderer , identity_token , account_syncid , ytcfg ,
2021-03-14 18:41:11 -04:00
parent = comment . get ( ' id ' ) , session_token_list = session_token_list ,
comment_counts = comment_counts )
for reply_comment in comment_entries_iter :
yield reply_comment
if not comment_counts :
# comment so far, est. total comments, current comment thread #
comment_counts = [ 0 , 0 , 0 ]
# TODO: Generalize the download code with TabIE
2021-04-11 16:23:04 -04:00
context = self . _extract_context ( ytcfg )
visitor_data = try_get ( context , lambda x : x [ ' client ' ] [ ' visitorData ' ] , compat_str )
2021-03-14 18:41:11 -04:00
continuation = YoutubeTabIE . _extract_continuation ( root_continuation_data ) # TODO
first_continuation = False
if parent is None :
first_continuation = True
for page_num in itertools . count ( 0 ) :
if not continuation :
break
2021-04-11 16:23:04 -04:00
headers = self . _generate_api_headers ( ytcfg , identity_token , account_syncid , visitor_data )
2021-05-17 08:23:08 -04:00
retries = self . get_param ( ' extractor_retries ' , 3 )
2021-03-14 18:41:11 -04:00
count = - 1
last_error = None
while count < retries :
count + = 1
if last_error :
self . report_warning ( ' %s . Retrying ... ' % last_error )
try :
query = {
' ctoken ' : continuation [ ' ctoken ' ] ,
' pbj ' : 1 ,
' type ' : ' next ' ,
}
2021-06-25 14:01:10 -04:00
if ' itct ' in continuation :
query [ ' itct ' ] = continuation [ ' itct ' ]
2021-03-14 18:41:11 -04:00
if parent :
query [ ' action_get_comment_replies ' ] = 1
else :
query [ ' action_get_comments ' ] = 1
comment_prog_str = ' ( %d / %d ) ' % ( comment_counts [ 0 ] , comment_counts [ 1 ] )
if page_num == 0 :
if first_continuation :
2021-04-07 07:37:43 -04:00
note_prefix = ' Downloading initial comment continuation page '
2021-03-14 18:41:11 -04:00
else :
2021-04-07 07:37:43 -04:00
note_prefix = ' Downloading comment reply thread %d %s ' % ( comment_counts [ 2 ] , comment_prog_str )
2021-03-14 18:41:11 -04:00
else :
2021-04-07 07:37:43 -04:00
note_prefix = ' %s Downloading comment %s page %d %s ' % (
' ' if parent else ' ' ,
2021-03-14 18:41:11 -04:00
' replies ' if parent else ' ' ,
page_num ,
comment_prog_str )
browse = self . _download_json (
' https://www.youtube.com/comment_service_ajax ' , None ,
' %s %s ' % ( note_prefix , ' (retry # %d ) ' % count if count else ' ' ) ,
headers = headers , query = query ,
data = urlencode_postdata ( {
' session_token ' : session_token_list [ 0 ]
} ) )
except ExtractorError as e :
if isinstance ( e . cause , compat_HTTPError ) and e . cause . code in ( 500 , 503 , 404 , 413 ) :
if e . cause . code == 413 :
2021-04-07 07:37:43 -04:00
self . report_warning ( ' Assumed end of comments (received HTTP Error 413) ' )
2021-03-14 18:41:11 -04:00
return
# Downloading page may result in intermittent 5xx HTTP error
# Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
last_error = ' HTTP Error %s ' % e . cause . code
if e . cause . code == 404 :
2021-04-07 07:37:43 -04:00
last_error = last_error + ' (this API is probably deprecated) '
2021-03-14 18:41:11 -04:00
if count < retries :
continue
raise
else :
session_token = try_get ( browse , lambda x : x [ ' xsrf_token ' ] , compat_str )
if session_token :
session_token_list [ 0 ] = session_token
response = try_get ( browse ,
( lambda x : x [ ' response ' ] ,
2021-06-25 14:01:10 -04:00
lambda x : x [ 1 ] [ ' response ' ] ) , dict ) or { }
2021-03-14 18:41:11 -04:00
if response . get ( ' continuationContents ' ) :
break
# YouTube sometimes gives reload: now json if something went wrong (e.g. bad auth)
2021-06-25 14:01:10 -04:00
if isinstance ( browse , dict ) :
if browse . get ( ' reload ' ) :
raise ExtractorError ( ' Invalid or missing params in continuation request ' , expected = False )
# TODO: not tested, merged from old extractor
err_msg = browse . get ( ' externalErrorMessage ' )
if err_msg :
last_error = err_msg
continue
2021-03-14 18:41:11 -04:00
2021-06-25 14:01:10 -04:00
response_error = try_get ( response , lambda x : x [ ' responseContext ' ] [ ' errors ' ] [ ' error ' ] [ 0 ] , dict ) or { }
err_msg = response_error . get ( ' externalErrorMessage ' )
2021-03-14 18:41:11 -04:00
if err_msg :
2021-06-25 14:01:10 -04:00
last_error = err_msg
continue
2021-03-14 18:41:11 -04:00
# Youtube sometimes sends incomplete data
# See: https://github.com/ytdl-org/youtube-dl/issues/28194
last_error = ' Incomplete data received '
if count > = retries :
2021-04-16 06:01:10 -04:00
raise ExtractorError ( last_error )
2021-03-14 18:41:11 -04:00
if not response :
break
2021-04-11 16:23:04 -04:00
visitor_data = try_get (
response ,
lambda x : x [ ' responseContext ' ] [ ' webResponseContextExtensionData ' ] [ ' ytConfigData ' ] [ ' visitorData ' ] ,
compat_str ) or visitor_data
2021-03-14 18:41:11 -04:00
known_continuation_renderers = {
' itemSectionContinuation ' : extract_thread ,
' commentRepliesContinuation ' : extract_thread
}
# extract next root continuation from the results
continuation_contents = try_get (
response , lambda x : x [ ' continuationContents ' ] , dict ) or { }
for key , value in continuation_contents . items ( ) :
if key not in known_continuation_renderers :
continue
continuation_renderer = value
if first_continuation :
first_continuation = False
expected_comment_count = try_get (
continuation_renderer ,
( lambda x : x [ ' header ' ] [ ' commentsHeaderRenderer ' ] [ ' countText ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ,
lambda x : x [ ' header ' ] [ ' commentsHeaderRenderer ' ] [ ' commentsCount ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ) ,
compat_str )
if expected_comment_count :
comment_counts [ 1 ] = str_to_int ( expected_comment_count )
2021-04-07 07:37:43 -04:00
self . to_screen ( ' Downloading ~ %d comments ' % str_to_int ( expected_comment_count ) )
2021-03-14 18:41:11 -04:00
yield comment_counts [ 1 ]
# TODO: cli arg.
# 1/True for newest, 0/False for popular (default)
comment_sort_index = int ( True )
sort_continuation_renderer = try_get (
continuation_renderer ,
lambda x : x [ ' header ' ] [ ' commentsHeaderRenderer ' ] [ ' sortMenu ' ] [ ' sortFilterSubMenuRenderer ' ] [ ' subMenuItems ' ]
[ comment_sort_index ] [ ' continuation ' ] [ ' reloadContinuationData ' ] , dict )
# If this fails, the initial continuation page
# starts off with popular anyways.
if sort_continuation_renderer :
continuation = YoutubeTabIE . _build_continuation_query (
continuation = sort_continuation_renderer . get ( ' continuation ' ) ,
ctp = sort_continuation_renderer . get ( ' clickTrackingParams ' ) )
2021-04-07 07:37:43 -04:00
self . to_screen ( ' Sorting comments by %s ' % ( ' popular ' if comment_sort_index == 0 else ' newest ' ) )
2021-03-14 18:41:11 -04:00
break
for entry in known_continuation_renderers [ key ] ( continuation_renderer ) :
yield entry
continuation = YoutubeTabIE . _extract_continuation ( continuation_renderer ) # TODO
break
def _extract_comments ( self , ytcfg , video_id , contents , webpage , xsrf_token ) :
""" Entry for comment extraction """
comments = [ ]
known_entry_comment_renderers = (
' itemSectionRenderer ' ,
)
estimated_total = 0
for entry in contents :
for key , renderer in entry . items ( ) :
if key not in known_entry_comment_renderers :
continue
comment_iter = self . _comment_entries (
renderer ,
identity_token = self . _extract_identity_token ( webpage , item_id = video_id ) ,
account_syncid = self . _extract_account_syncid ( ytcfg ) ,
2021-04-11 16:23:04 -04:00
ytcfg = ytcfg ,
2021-03-14 18:41:11 -04:00
session_token_list = [ xsrf_token ] )
for comment in comment_iter :
if isinstance ( comment , int ) :
estimated_total = comment
continue
comments . append ( comment )
break
2021-04-07 07:37:43 -04:00
self . to_screen ( ' Downloaded %d / %d comments ' % ( len ( comments ) , estimated_total ) )
2021-03-14 18:41:11 -04:00
return {
' comments ' : comments ,
' comment_count ' : len ( comments ) ,
}
2021-06-29 18:07:49 -04:00
@staticmethod
def _generate_player_context ( sts = None ) :
context = {
' html5Preference ' : ' HTML5_PREF_WANTS ' ,
}
if sts is not None :
context [ ' signatureTimestamp ' ] = sts
return {
' playbackContext ' : {
' contentPlaybackContext ' : context
}
}
2021-06-18 11:02:52 -04:00
@staticmethod
def _get_video_info_params ( video_id ) :
return {
' video_id ' : video_id ,
' eurl ' : ' https://youtube.googleapis.com/v/ ' + video_id ,
' html5 ' : ' 1 ' ,
' c ' : ' TVHTML5 ' ,
' cver ' : ' 6.20180913 ' ,
}
2013-06-23 13:58:33 -04:00
def _real_extract ( self , url ) :
2015-07-25 11:30:34 -04:00
url , smuggled_data = unsmuggle_url ( url , { } )
2021-02-04 09:37:17 -05:00
video_id = self . _match_id ( url )
2021-05-15 10:38:47 -04:00
is_music_url = smuggled_data . get ( ' is_music_url ' ) or self . is_music_url ( url )
2021-02-04 09:37:17 -05:00
base_url = self . http_scheme ( ) + ' //www.youtube.com/ '
2021-02-27 04:15:56 -05:00
webpage_url = base_url + ' watch?v= ' + video_id
webpage = self . _download_webpage (
2021-04-01 04:28:33 -04:00
webpage_url + ' &bpctr=9999999999&has_verified=1 ' , video_id , fatal = False )
2021-02-04 09:37:17 -05:00
2021-06-29 18:07:49 -04:00
ytcfg = self . _extract_ytcfg ( video_id , webpage ) or self . _get_default_ytcfg ( )
identity_token = self . _extract_identity_token ( webpage , video_id )
syncid = self . _extract_account_syncid ( ytcfg )
headers = self . _generate_api_headers ( ytcfg , identity_token , syncid )
player_url = self . _extract_player_url ( ytcfg , webpage )
player_client = try_get ( self . _configuration_arg ( ' player_client ' ) , lambda x : x [ 0 ] , str ) or ' '
if player_client . upper ( ) not in ( ' WEB ' , ' ANDROID ' ) :
player_client = ' WEB '
force_mobile_client = player_client . upper ( ) == ' ANDROID '
player_skip = self . _configuration_arg ( ' player_skip ' ) or [ ]
2021-05-15 10:38:47 -04:00
def get_text ( x ) :
if not x :
return
text = x . get ( ' simpleText ' )
if text and isinstance ( text , compat_str ) :
return text
runs = x . get ( ' runs ' )
if not isinstance ( runs , list ) :
return
return ' ' . join ( [ r [ ' text ' ] for r in runs if isinstance ( r . get ( ' text ' ) , compat_str ) ] )
ytm_streaming_data = { }
if is_music_url :
2021-06-29 18:07:49 -04:00
ytm_webpage = None
sts = self . _extract_signature_timestamp ( video_id , player_url , ytcfg , fatal = False )
if sts and not force_mobile_client and ' configs ' not in player_skip :
ytm_webpage = self . _download_webpage (
' https://music.youtube.com ' ,
video_id , fatal = False , note = " Downloading remix client config " )
ytm_cfg = self . _extract_ytcfg ( video_id , ytm_webpage ) or { }
ytm_client = ' WEB_REMIX '
if not sts or force_mobile_client :
# Android client already has signature descrambled
# See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
if not sts :
self . report_warning ( ' Falling back to mobile remix client for player API. ' )
ytm_client = ' ANDROID_MUSIC '
ytm_cfg = { }
ytm_headers = self . _generate_api_headers (
ytm_cfg , identity_token , syncid ,
client = ytm_client )
ytm_query = { ' videoId ' : video_id }
ytm_query . update ( self . _generate_player_context ( sts ) )
ytm_player_response = self . _extract_response (
item_id = video_id , ep = ' player ' , query = ytm_query ,
ytcfg = ytm_cfg , headers = ytm_headers , fatal = False ,
default_client = ytm_client ,
note = ' Downloading %s remix player API JSON ' % ( ' mobile ' if force_mobile_client else ' ' ) )
ytm_streaming_data = try_get ( ytm_player_response , lambda x : x [ ' streamingData ' ] ) or { }
2021-02-04 09:37:17 -05:00
player_response = None
if webpage :
player_response = self . _extract_yt_initial_variable (
webpage , self . _YT_INITIAL_PLAYER_RESPONSE_RE ,
video_id , ' initial player response ' )
2021-04-11 16:23:04 -04:00
2021-06-29 18:07:49 -04:00
if not player_response or force_mobile_client :
sts = self . _extract_signature_timestamp ( video_id , player_url , ytcfg , fatal = False )
yt_client = ' WEB '
ytpcfg = ytcfg
ytp_headers = headers
if not sts or force_mobile_client :
# Android client already has signature descrambled
# See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
if not sts :
self . report_warning ( ' Falling back to mobile client for player API. ' )
yt_client = ' ANDROID '
ytpcfg = { }
ytp_headers = self . _generate_api_headers ( ytpcfg , identity_token , syncid , yt_client )
yt_query = { ' videoId ' : video_id }
yt_query . update ( self . _generate_player_context ( sts ) )
player_response = self . _extract_response (
item_id = video_id , ep = ' player ' , query = yt_query ,
ytcfg = ytpcfg , headers = ytp_headers , fatal = False ,
default_client = yt_client ,
note = ' Downloading %s player API JSON ' % ( ' mobile ' if force_mobile_client else ' ' )
)
2021-02-04 09:37:17 -05:00
2021-06-29 18:07:49 -04:00
# Age-gate workarounds
2021-02-04 09:37:17 -05:00
playability_status = player_response . get ( ' playabilityStatus ' ) or { }
2021-06-29 18:07:49 -04:00
if playability_status . get ( ' reason ' ) in self . _AGE_GATE_REASONS :
2021-02-04 09:37:17 -05:00
pr = self . _parse_json ( try_get ( compat_parse_qs (
self . _download_webpage (
base_url + ' get_video_info ' , video_id ,
2021-06-18 11:02:52 -04:00
' Refetching age-gated info webpage ' , ' unable to download video info webpage ' ,
query = self . _get_video_info_params ( video_id ) , fatal = False ) ) ,
2021-02-04 09:37:17 -05:00
lambda x : x [ ' player_response ' ] [ 0 ] ,
compat_str ) or ' {} ' , video_id )
2021-06-29 18:07:49 -04:00
if not pr :
self . report_warning ( ' Falling back to embedded-only age-gate workaround. ' )
embed_webpage = None
sts = self . _extract_signature_timestamp ( video_id , player_url , ytcfg , fatal = False )
if sts and not force_mobile_client and ' configs ' not in player_skip :
embed_webpage = self . _download_webpage (
' https://www.youtube.com/embed/ %s ?html5=1 ' % video_id ,
video_id = video_id , note = ' Downloading age-gated embed config ' )
ytcfg_age = self . _extract_ytcfg ( video_id , embed_webpage ) or { }
# If we extracted the embed webpage, it'll tell us if we can view the video
embedded_pr = self . _parse_json (
try_get ( ytcfg_age , lambda x : x [ ' PLAYER_VARS ' ] [ ' embedded_player_response ' ] , str ) or ' {} ' ,
video_id = video_id )
embedded_ps_reason = try_get ( embedded_pr , lambda x : x [ ' playabilityStatus ' ] [ ' reason ' ] , str ) or ' '
if embedded_ps_reason not in self . _AGE_GATE_REASONS :
yt_client = ' WEB_EMBEDDED_PLAYER '
if not sts or force_mobile_client :
# Android client already has signature descrambled
# See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
if not sts :
self . report_warning (
' Falling back to mobile embedded client for player API (note: some formats may be missing). ' )
yt_client = ' ANDROID_EMBEDDED_PLAYER '
ytcfg_age = { }
ytage_headers = self . _generate_api_headers (
ytcfg_age , identity_token , syncid , client = yt_client )
yt_age_query = { ' videoId ' : video_id }
yt_age_query . update ( self . _generate_player_context ( sts ) )
pr = self . _extract_response (
item_id = video_id , ep = ' player ' , query = yt_age_query ,
ytcfg = ytcfg_age , headers = ytage_headers , fatal = False ,
default_client = yt_client ,
note = ' Downloading %s age-gated player API JSON ' % ( ' mobile ' if force_mobile_client else ' ' )
) or { }
2021-02-04 09:37:17 -05:00
if pr :
player_response = pr
trailer_video_id = try_get (
playability_status ,
lambda x : x [ ' errorScreen ' ] [ ' playerLegacyDesktopYpcTrailerRenderer ' ] [ ' trailerVideoId ' ] ,
compat_str )
if trailer_video_id :
return self . url_result (
trailer_video_id , self . ie_key ( ) , trailer_video_id )
2015-07-25 11:30:34 -04:00
2021-02-04 09:37:17 -05:00
search_meta = (
lambda x : self . _html_search_meta ( x , webpage , default = None ) ) \
if webpage else lambda x : None
2018-11-02 19:26:16 -04:00
2021-02-04 09:37:17 -05:00
video_details = player_response . get ( ' videoDetails ' ) or { }
2020-06-15 15:37:19 -04:00
microformat = try_get (
2021-02-04 09:37:17 -05:00
player_response ,
lambda x : x [ ' microformat ' ] [ ' playerMicroformatRenderer ' ] ,
dict ) or { }
video_title = video_details . get ( ' title ' ) \
or get_text ( microformat . get ( ' title ' ) ) \
or search_meta ( [ ' og:title ' , ' twitter:title ' , ' title ' ] )
video_description = video_details . get ( ' shortDescription ' )
2015-07-25 11:30:34 -04:00
2018-12-15 10:25:12 -05:00
if not smuggled_data . get ( ' force_singlefeed ' , False ) :
2021-05-17 08:23:08 -04:00
if not self . get_param ( ' noplaylist ' ) :
2018-12-15 10:25:12 -05:00
multifeed_metadata_list = try_get (
player_response ,
lambda x : x [ ' multicamera ' ] [ ' playerLegacyMulticameraRenderer ' ] [ ' metadataList ' ] ,
2021-02-04 09:37:17 -05:00
compat_str )
2018-12-15 10:25:12 -05:00
if multifeed_metadata_list :
entries = [ ]
feed_ids = [ ]
for feed in multifeed_metadata_list . split ( ' , ' ) :
# Unquote should take place before split on comma (,) since textual
# fields may contain comma as well (see
2019-03-09 07:14:41 -05:00
# https://github.com/ytdl-org/youtube-dl/issues/8536)
2021-02-04 09:37:17 -05:00
feed_data = compat_parse_qs (
compat_urllib_parse_unquote_plus ( feed ) )
2020-04-09 11:42:43 -04:00
def feed_entry ( name ) :
2021-02-04 09:37:17 -05:00
return try_get (
feed_data , lambda x : x [ name ] [ 0 ] , compat_str )
2020-04-09 11:42:43 -04:00
feed_id = feed_entry ( ' id ' )
if not feed_id :
continue
feed_title = feed_entry ( ' title ' )
title = video_title
if feed_title :
title + = ' ( %s ) ' % feed_title
2018-12-15 10:25:12 -05:00
entries . append ( {
' _type ' : ' url_transparent ' ,
' ie_key ' : ' Youtube ' ,
' url ' : smuggle_url (
2021-02-04 09:37:17 -05:00
base_url + ' watch?v= ' + feed_data [ ' id ' ] [ 0 ] ,
2018-12-15 10:25:12 -05:00
{ ' force_singlefeed ' : True } ) ,
2020-04-09 11:42:43 -04:00
' title ' : title ,
2018-12-15 10:25:12 -05:00
} )
2020-04-09 11:42:43 -04:00
feed_ids . append ( feed_id )
2018-12-15 10:25:12 -05:00
self . to_screen (
' Downloading multifeed video ( %s ) - add --no-playlist to just download video %s '
% ( ' , ' . join ( feed_ids ) , video_id ) )
2021-02-04 09:37:17 -05:00
return self . playlist_result (
entries , video_id , video_title , video_description )
2018-12-15 10:25:12 -05:00
else :
self . to_screen ( ' Downloading just video %s because of --no-playlist ' % video_id )
2015-07-25 11:30:34 -04:00
2021-05-15 10:38:47 -04:00
formats , itags , stream_ids = [ ] , [ ] , [ ]
2021-02-10 16:22:55 -05:00
itag_qualities = { }
2021-05-17 15:41:57 -04:00
q = qualities ( [
' tiny ' , ' audio_quality_low ' , ' audio_quality_medium ' , ' audio_quality_high ' , # Audio only formats
' small ' , ' medium ' , ' large ' , ' hd720 ' , ' hd1080 ' , ' hd1440 ' , ' hd2160 ' , ' hd2880 ' , ' highres '
] )
2021-05-15 10:38:47 -04:00
2021-02-04 09:37:17 -05:00
streaming_data = player_response . get ( ' streamingData ' ) or { }
streaming_formats = streaming_data . get ( ' formats ' ) or [ ]
streaming_formats . extend ( streaming_data . get ( ' adaptiveFormats ' ) or [ ] )
2021-05-15 10:38:47 -04:00
streaming_formats . extend ( ytm_streaming_data . get ( ' formats ' ) or [ ] )
streaming_formats . extend ( ytm_streaming_data . get ( ' adaptiveFormats ' ) or [ ] )
2021-02-04 09:37:17 -05:00
for fmt in streaming_formats :
if fmt . get ( ' targetDurationSec ' ) or fmt . get ( ' drmFamilies ' ) :
continue
2020-08-04 20:30:10 -04:00
2021-02-10 16:22:55 -05:00
itag = str_or_none ( fmt . get ( ' itag ' ) )
2021-05-15 10:38:47 -04:00
audio_track = fmt . get ( ' audioTrack ' ) or { }
stream_id = ' %s . %s ' % ( itag or ' ' , audio_track . get ( ' id ' , ' ' ) )
if stream_id in stream_ids :
continue
2021-02-10 16:22:55 -05:00
quality = fmt . get ( ' quality ' )
2021-05-17 15:41:57 -04:00
if quality == ' tiny ' or not quality :
quality = fmt . get ( ' audioQuality ' , ' ' ) . lower ( ) or quality
2021-02-10 16:22:55 -05:00
if itag and quality :
itag_qualities [ itag ] = quality
# FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
# (adding `&sq=0` to the URL) and parsing emsg box to determine the
# number of fragment that would subsequently requested with (`&sq=N`)
if fmt . get ( ' type ' ) == ' FORMAT_STREAM_TYPE_OTF ' :
continue
2021-02-04 09:37:17 -05:00
fmt_url = fmt . get ( ' url ' )
if not fmt_url :
sc = compat_parse_qs ( fmt . get ( ' signatureCipher ' ) )
fmt_url = url_or_none ( try_get ( sc , lambda x : x [ ' url ' ] [ 0 ] ) )
encrypted_sig = try_get ( sc , lambda x : x [ ' s ' ] [ 0 ] )
if not ( sc and fmt_url and encrypted_sig ) :
continue
if not player_url :
2014-08-02 00:35:18 -04:00
continue
2021-02-04 09:37:17 -05:00
signature = self . _decrypt_signature ( sc [ ' s ' ] [ 0 ] , video_id , player_url )
sp = try_get ( sc , lambda x : x [ ' sp ' ] [ 0 ] ) or ' signature '
fmt_url + = ' & ' + sp + ' = ' + signature
if itag :
itags . append ( itag )
2021-05-15 10:38:47 -04:00
stream_ids . append ( stream_id )
2021-02-10 16:22:55 -05:00
tbr = float_or_none (
fmt . get ( ' averageBitrate ' ) or fmt . get ( ' bitrate ' ) , 1000 )
2021-02-04 09:37:17 -05:00
dct = {
' asr ' : int_or_none ( fmt . get ( ' audioSampleRate ' ) ) ,
' filesize ' : int_or_none ( fmt . get ( ' contentLength ' ) ) ,
' format_id ' : itag ,
2021-05-14 03:01:53 -04:00
' format_note ' : audio_track . get ( ' displayName ' ) or fmt . get ( ' qualityLabel ' ) or quality ,
2021-02-04 09:37:17 -05:00
' fps ' : int_or_none ( fmt . get ( ' fps ' ) ) ,
' height ' : int_or_none ( fmt . get ( ' height ' ) ) ,
2021-02-18 13:12:56 -05:00
' quality ' : q ( quality ) ,
2021-02-10 16:22:55 -05:00
' tbr ' : tbr ,
2021-02-04 09:37:17 -05:00
' url ' : fmt_url ,
' width ' : fmt . get ( ' width ' ) ,
2021-05-14 03:01:53 -04:00
' language ' : audio_track . get ( ' id ' , ' ' ) . split ( ' . ' ) [ 0 ] ,
2021-02-04 09:37:17 -05:00
}
mimetype = fmt . get ( ' mimeType ' )
if mimetype :
mobj = re . match (
r ' ((?:[^/]+)/(?:[^;]+))(?:; \ s*codecs= " ([^ " ]+) " )? ' , mimetype )
if mobj :
dct [ ' ext ' ] = mimetype2ext ( mobj . group ( 1 ) )
dct . update ( parse_codecs ( mobj . group ( 2 ) ) )
2021-02-10 16:22:55 -05:00
no_audio = dct . get ( ' acodec ' ) == ' none '
no_video = dct . get ( ' vcodec ' ) == ' none '
if no_audio :
dct [ ' vbr ' ] = tbr
if no_video :
dct [ ' abr ' ] = tbr
if no_audio or no_video :
2021-02-04 09:37:17 -05:00
dct [ ' downloader_options ' ] = {
# Youtube throttles chunks >~10M
' http_chunk_size ' : 10485760 ,
2019-09-11 11:44:47 -04:00
}
2021-02-24 15:57:04 -05:00
if dct . get ( ' ext ' ) :
dct [ ' container ' ] = dct [ ' ext ' ] + ' _dash '
2021-02-04 09:37:17 -05:00
formats . append ( dct )
2021-06-25 10:05:41 -04:00
skip_manifests = self . _configuration_arg ( ' skip ' ) or [ ]
get_dash = ' dash ' not in skip_manifests and self . get_param ( ' youtube_include_dash_manifest ' , True )
get_hls = ' hls ' not in skip_manifests and self . get_param ( ' youtube_include_hls_manifest ' , True )
2021-05-15 10:38:47 -04:00
for sd in ( streaming_data , ytm_streaming_data ) :
2021-06-25 10:05:41 -04:00
hls_manifest_url = get_hls and sd . get ( ' hlsManifestUrl ' )
2021-05-15 10:38:47 -04:00
if hls_manifest_url :
for f in self . _extract_m3u8_formats (
hls_manifest_url , video_id , ' mp4 ' , fatal = False ) :
itag = self . _search_regex (
r ' /itag/( \ d+) ' , f [ ' url ' ] , ' itag ' , default = None )
if itag :
f [ ' format_id ' ] = itag
2021-05-22 17:38:11 -04:00
formats . append ( f )
2021-02-04 09:37:17 -05:00
2021-06-25 10:05:41 -04:00
dash_manifest_url = get_dash and sd . get ( ' dashManifestUrl ' )
if dash_manifest_url :
for f in self . _extract_mpd_formats (
dash_manifest_url , video_id , fatal = False ) :
itag = f [ ' format_id ' ]
if itag in itags :
continue
if itag in itag_qualities :
f [ ' quality ' ] = q ( itag_qualities [ itag ] )
filesize = int_or_none ( self . _search_regex (
r ' /clen/( \ d+) ' , f . get ( ' fragment_base_url ' )
or f [ ' url ' ] , ' file size ' , default = None ) )
if filesize :
f [ ' filesize ' ] = filesize
formats . append ( f )
2019-09-11 11:44:47 -04:00
2021-02-04 09:37:17 -05:00
if not formats :
2021-05-17 08:23:08 -04:00
if not self . get_param ( ' allow_unplayable_formats ' ) and streaming_data . get ( ' licenseInfos ' ) :
2021-04-16 20:09:58 -04:00
self . raise_no_formats (
2021-02-04 09:37:17 -05:00
' This video is DRM protected. ' , expected = True )
pemr = try_get (
playability_status ,
lambda x : x [ ' errorScreen ' ] [ ' playerErrorMessageRenderer ' ] ,
dict ) or { }
reason = get_text ( pemr . get ( ' reason ' ) ) or playability_status . get ( ' reason ' )
subreason = pemr . get ( ' subreason ' )
if subreason :
subreason = clean_html ( get_text ( subreason ) )
if subreason == ' The uploader has not made this video available in your country. ' :
countries = microformat . get ( ' availableCountries ' )
if not countries :
regions_allowed = search_meta ( ' regionsAllowed ' )
countries = regions_allowed . split ( ' , ' ) if regions_allowed else None
2021-04-16 20:09:58 -04:00
self . raise_geo_restricted ( subreason , countries , metadata_available = True )
2021-02-04 09:37:17 -05:00
reason + = ' \n ' + subreason
if reason :
2021-04-16 20:09:58 -04:00
self . raise_no_formats ( reason , expected = True )
2019-09-11 11:44:47 -04:00
2021-02-04 09:37:17 -05:00
self . _sort_formats ( formats )
2019-09-11 11:44:47 -04:00
2021-02-04 09:37:17 -05:00
keywords = video_details . get ( ' keywords ' ) or [ ]
if not keywords and webpage :
keywords = [
unescapeHTML ( m . group ( ' content ' ) )
for m in re . finditer ( self . _meta_regex ( ' og:video:tag ' ) , webpage ) ]
for keyword in keywords :
if keyword . startswith ( ' yt:stretch= ' ) :
2021-04-16 23:02:33 -04:00
mobj = re . search ( r ' ( \ d+) \ s*: \ s*( \ d+) ' , keyword )
if mobj :
# NB: float is intentional for forcing float division
w , h = ( float ( v ) for v in mobj . groups ( ) )
if w > 0 and h > 0 :
ratio = w / h
for f in formats :
if f . get ( ' vcodec ' ) != ' none ' :
f [ ' stretched_ratio ' ] = ratio
break
2015-01-29 22:43:50 -05:00
2021-02-04 09:37:17 -05:00
thumbnails = [ ]
for container in ( video_details , microformat ) :
for thumbnail in ( try_get (
container ,
lambda x : x [ ' thumbnail ' ] [ ' thumbnails ' ] , list ) or [ ] ) :
thumbnail_url = thumbnail . get ( ' url ' )
if not thumbnail_url :
2019-09-11 11:44:47 -04:00
continue
2021-04-10 20:33:02 -04:00
# Sometimes youtube gives a wrong thumbnail URL. See:
# https://github.com/yt-dlp/yt-dlp/issues/233
# https://github.com/ytdl-org/youtube-dl/issues/28023
if ' maxresdefault ' in thumbnail_url :
thumbnail_url = thumbnail_url . split ( ' ? ' ) [ 0 ]
2021-02-04 09:37:17 -05:00
thumbnails . append ( {
' url ' : thumbnail_url ,
2021-05-18 09:45:11 -04:00
' height ' : int_or_none ( thumbnail . get ( ' height ' ) ) ,
2021-02-04 09:37:17 -05:00
' width ' : int_or_none ( thumbnail . get ( ' width ' ) ) ,
2021-05-18 09:45:11 -04:00
' preference ' : 1 if ' maxresdefault ' in thumbnail_url else - 1
2021-02-04 09:37:17 -05:00
} )
2021-05-18 09:45:11 -04:00
thumbnail_url = search_meta ( [ ' og:image ' , ' twitter:image ' ] )
if thumbnail_url :
thumbnails . append ( {
' url ' : thumbnail_url ,
' preference ' : 1 if ' maxresdefault ' in thumbnail_url else - 1
} )
# All videos have a maxresdefault thumbnail, but sometimes it does not appear in the webpage
# See: https://github.com/ytdl-org/youtube-dl/issues/29049
thumbnails . append ( {
' url ' : ' https://i.ytimg.com/vi/ %s /maxresdefault.jpg ' % video_id ,
' preference ' : 1 ,
} )
self . _remove_duplicate_formats ( thumbnails )
2021-02-04 09:37:17 -05:00
category = microformat . get ( ' category ' ) or search_meta ( ' genre ' )
channel_id = video_details . get ( ' channelId ' ) \
or microformat . get ( ' externalChannelId ' ) \
or search_meta ( ' channelId ' )
duration = int_or_none (
video_details . get ( ' lengthSeconds ' )
or microformat . get ( ' lengthSeconds ' ) ) \
or parse_duration ( search_meta ( ' duration ' ) )
is_live = video_details . get ( ' isLive ' )
2021-07-01 20:29:29 -04:00
is_upcoming = video_details . get ( ' isUpcoming ' )
2021-02-04 09:37:17 -05:00
owner_profile_url = microformat . get ( ' ownerProfileUrl ' )
info = {
' id ' : video_id ,
' title ' : self . _live_title ( video_title ) if is_live else video_title ,
' formats ' : formats ,
' thumbnails ' : thumbnails ,
' description ' : video_description ,
' upload_date ' : unified_strdate (
microformat . get ( ' uploadDate ' )
or search_meta ( ' uploadDate ' ) ) ,
' uploader ' : video_details [ ' author ' ] ,
' uploader_id ' : self . _search_regex ( r ' /(?:channel|user)/([^/?&#]+) ' , owner_profile_url , ' uploader id ' ) if owner_profile_url else None ,
' uploader_url ' : owner_profile_url ,
' channel_id ' : channel_id ,
' channel_url ' : ' https://www.youtube.com/channel/ ' + channel_id if channel_id else None ,
' duration ' : duration ,
' view_count ' : int_or_none (
video_details . get ( ' viewCount ' )
or microformat . get ( ' viewCount ' )
or search_meta ( ' interactionCount ' ) ) ,
' average_rating ' : float_or_none ( video_details . get ( ' averageRating ' ) ) ,
' age_limit ' : 18 if (
microformat . get ( ' isFamilySafe ' ) is False
or search_meta ( ' isFamilyFriendly ' ) == ' false '
or search_meta ( ' og:restrictions:age ' ) == ' 18+ ' ) else 0 ,
' webpage_url ' : webpage_url ,
' categories ' : [ category ] if category else None ,
' tags ' : keywords ,
' is_live ' : is_live ,
' playable_in_embed ' : playability_status . get ( ' playableInEmbed ' ) ,
2021-03-21 17:23:34 -04:00
' was_live ' : video_details . get ( ' isLiveContent ' ) ,
2021-02-04 09:37:17 -05:00
}
2020-06-15 15:29:04 -04:00
2021-02-04 09:37:17 -05:00
pctr = try_get (
player_response ,
lambda x : x [ ' captions ' ] [ ' playerCaptionsTracklistRenderer ' ] , dict )
subtitles = { }
if pctr :
2021-05-13 07:19:40 -04:00
def process_language ( container , base_url , lang_code , sub_name , query ) :
2021-05-12 15:20:02 -04:00
lang_subs = container . setdefault ( lang_code , [ ] )
2021-02-04 09:37:17 -05:00
for fmt in self . _SUBTITLE_FORMATS :
query . update ( {
' fmt ' : fmt ,
} )
lang_subs . append ( {
' ext ' : fmt ,
' url ' : update_url_query ( base_url , query ) ,
2021-05-13 07:19:40 -04:00
' name ' : sub_name ,
2021-02-04 09:37:17 -05:00
} )
2018-06-02 15:08:38 -04:00
2021-02-04 09:37:17 -05:00
for caption_track in ( pctr . get ( ' captionTracks ' ) or [ ] ) :
base_url = caption_track . get ( ' baseUrl ' )
if not base_url :
continue
if caption_track . get ( ' kind ' ) != ' asr ' :
2021-05-12 15:20:02 -04:00
lang_code = (
remove_start ( caption_track . get ( ' vssId ' ) or ' ' , ' . ' ) . replace ( ' . ' , ' - ' )
or caption_track . get ( ' languageCode ' ) )
2021-02-04 09:37:17 -05:00
if not lang_code :
continue
process_language (
2021-05-13 07:19:40 -04:00
subtitles , base_url , lang_code ,
try_get ( caption_track , lambda x : x . get ( ' name ' ) . get ( ' simpleText ' ) ) ,
{ } )
2021-02-04 09:37:17 -05:00
continue
automatic_captions = { }
for translation_language in ( pctr . get ( ' translationLanguages ' ) or [ ] ) :
translation_language_code = translation_language . get ( ' languageCode ' )
if not translation_language_code :
continue
process_language (
automatic_captions , base_url , translation_language_code ,
2021-06-25 13:40:31 -04:00
try_get ( translation_language , (
lambda x : x [ ' languageName ' ] [ ' simpleText ' ] ,
lambda x : x [ ' languageName ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ) ) ,
2021-02-04 09:37:17 -05:00
{ ' tlang ' : translation_language_code } )
info [ ' automatic_captions ' ] = automatic_captions
info [ ' subtitles ' ] = subtitles
2018-06-02 15:08:38 -04:00
2021-02-04 09:37:17 -05:00
parsed_url = compat_urllib_parse_urlparse ( url )
for component in [ parsed_url . fragment , parsed_url . query ] :
query = compat_parse_qs ( component )
for k , v in query . items ( ) :
for d_k , s_ks in [ ( ' start ' , ( ' start ' , ' t ' ) ) , ( ' end ' , ( ' end ' , ) ) ] :
d_k + = ' _time '
if d_k not in info and k in s_ks :
info [ d_k ] = parse_duration ( query [ k ] [ 0 ] )
2019-04-27 04:16:17 -04:00
# Youtube Music Auto-generated description
if video_description :
2020-11-26 12:27:34 -05:00
mobj = re . search ( r ' (?s)(?P<track>[^· \ n]+)·(?P<artist>[^ \ n]+) \ n+(?P<album>[^ \ n]+)(?:.+?℗ \ s*(?P<release_year> \ d {4} )(?! \ d))?(?:.+?Released on \ s*: \ s*(?P<release_date> \ d {4} - \ d {2} - \ d {2} ))?(.+? \ nArtist \ s*: \ s*(?P<clean_artist>[^ \ n]+))?.+ \ nAuto-generated by YouTube \ . \ s*$ ' , video_description )
2019-04-27 04:16:17 -04:00
if mobj :
release_year = mobj . group ( ' release_year ' )
release_date = mobj . group ( ' release_date ' )
if release_date :
release_date = release_date . replace ( ' - ' , ' ' )
if not release_year :
2021-02-04 09:37:17 -05:00
release_year = release_date [ : 4 ]
info . update ( {
' album ' : mobj . group ( ' album ' . strip ( ) ) ,
' artist ' : mobj . group ( ' clean_artist ' ) or ' , ' . join ( a . strip ( ) for a in mobj . group ( ' artist ' ) . split ( ' · ' ) ) ,
' track ' : mobj . group ( ' track ' ) . strip ( ) ,
' release_date ' : release_date ,
2021-02-10 16:22:55 -05:00
' release_year ' : int_or_none ( release_year ) ,
2021-02-04 09:37:17 -05:00
} )
2018-06-02 15:08:38 -04:00
2021-02-04 09:37:17 -05:00
initial_data = None
if webpage :
initial_data = self . _extract_yt_initial_variable (
webpage , self . _YT_INITIAL_DATA_RE , video_id ,
' yt initial data ' )
if not initial_data :
2021-06-29 18:07:49 -04:00
initial_data = self . _extract_response (
item_id = video_id , ep = ' next ' , fatal = False ,
ytcfg = ytcfg , headers = headers , query = { ' videoId ' : video_id } ,
note = ' Downloading initial data API JSON ' )
2021-02-04 09:37:17 -05:00
2021-06-22 20:12:39 -04:00
try :
# This will error if there is no livechat
initial_data [ ' contents ' ] [ ' twoColumnWatchNextResults ' ] [ ' conversationBar ' ] [ ' liveChatRenderer ' ] [ ' continuations ' ] [ 0 ] [ ' reloadContinuationData ' ] [ ' continuation ' ]
info [ ' subtitles ' ] [ ' live_chat ' ] = [ {
' url ' : ' https://www.youtube.com/watch?v= %s ' % video_id , # url is needed to set cookies
' video_id ' : video_id ,
' ext ' : ' json ' ,
2021-07-01 20:29:29 -04:00
' protocol ' : ' youtube_live_chat ' if is_live or is_upcoming else ' youtube_live_chat_replay ' ,
2021-06-22 20:12:39 -04:00
} ]
except ( KeyError , IndexError , TypeError ) :
pass
2021-02-04 09:37:17 -05:00
if initial_data :
chapters = self . _extract_chapters_from_json (
initial_data , video_id , duration )
if not chapters :
for engagment_pannel in ( initial_data . get ( ' engagementPanels ' ) or [ ] ) :
contents = try_get (
engagment_pannel , lambda x : x [ ' engagementPanelSectionListRenderer ' ] [ ' content ' ] [ ' macroMarkersListRenderer ' ] [ ' contents ' ] ,
list )
if not contents :
continue
def chapter_time ( mmlir ) :
return parse_duration (
get_text ( mmlir . get ( ' timeDescription ' ) ) )
chapters = [ ]
for next_num , content in enumerate ( contents , start = 1 ) :
mmlir = content . get ( ' macroMarkersListItemRenderer ' ) or { }
start_time = chapter_time ( mmlir )
end_time = chapter_time ( try_get (
contents , lambda x : x [ next_num ] [ ' macroMarkersListItemRenderer ' ] ) ) \
if next_num < len ( contents ) else duration
if start_time is None or end_time is None :
continue
chapters . append ( {
' start_time ' : start_time ,
' end_time ' : end_time ,
' title ' : get_text ( mmlir . get ( ' title ' ) ) ,
} )
if chapters :
break
if chapters :
info [ ' chapters ' ] = chapters
contents = try_get (
initial_data ,
lambda x : x [ ' contents ' ] [ ' twoColumnWatchNextResults ' ] [ ' results ' ] [ ' results ' ] [ ' contents ' ] ,
list ) or [ ]
for content in contents :
vpir = content . get ( ' videoPrimaryInfoRenderer ' )
if vpir :
stl = vpir . get ( ' superTitleLink ' )
if stl :
stl = get_text ( stl )
if try_get (
vpir ,
lambda x : x [ ' superTitleIcon ' ] [ ' iconType ' ] ) == ' LOCATION_PIN ' :
info [ ' location ' ] = stl
else :
mobj = re . search ( r ' (.+?) \ s*S( \ d+) \ s*• \ s*E( \ d+) ' , stl )
if mobj :
info . update ( {
' series ' : mobj . group ( 1 ) ,
' season_number ' : int ( mobj . group ( 2 ) ) ,
' episode_number ' : int ( mobj . group ( 3 ) ) ,
} )
for tlb in ( try_get (
vpir ,
lambda x : x [ ' videoActions ' ] [ ' menuRenderer ' ] [ ' topLevelButtons ' ] ,
list ) or [ ] ) :
tbr = tlb . get ( ' toggleButtonRenderer ' ) or { }
for getter , regex in [ (
lambda x : x [ ' defaultText ' ] [ ' accessibility ' ] [ ' accessibilityData ' ] ,
r ' (?P<count>[ \ d,]+) \ s*(?P<type>(?:dis)?like) ' ) , ( [
lambda x : x [ ' accessibility ' ] ,
lambda x : x [ ' accessibilityData ' ] [ ' accessibilityData ' ] ,
] , r ' (?P<type>(?:dis)?like) this video along with (?P<count>[ \ d,]+) other people ' ) ] :
label = ( try_get ( tbr , getter , dict ) or { } ) . get ( ' label ' )
if label :
mobj = re . match ( regex , label )
if mobj :
info [ mobj . group ( ' type ' ) + ' _count ' ] = str_to_int ( mobj . group ( ' count ' ) )
break
sbr_tooltip = try_get (
vpir , lambda x : x [ ' sentimentBar ' ] [ ' sentimentBarRenderer ' ] [ ' tooltip ' ] )
if sbr_tooltip :
like_count , dislike_count = sbr_tooltip . split ( ' / ' )
info . update ( {
' like_count ' : str_to_int ( like_count ) ,
' dislike_count ' : str_to_int ( dislike_count ) ,
} )
vsir = content . get ( ' videoSecondaryInfoRenderer ' )
if vsir :
info [ ' channel ' ] = get_text ( try_get (
vsir ,
lambda x : x [ ' owner ' ] [ ' videoOwnerRenderer ' ] [ ' title ' ] ,
2021-04-01 04:28:33 -04:00
dict ) )
2021-02-04 09:37:17 -05:00
rows = try_get (
vsir ,
lambda x : x [ ' metadataRowContainer ' ] [ ' metadataRowContainerRenderer ' ] [ ' rows ' ] ,
list ) or [ ]
multiple_songs = False
for row in rows :
if try_get ( row , lambda x : x [ ' metadataRowRenderer ' ] [ ' hasDividerLine ' ] ) is True :
multiple_songs = True
break
for row in rows :
mrr = row . get ( ' metadataRowRenderer ' ) or { }
mrr_title = mrr . get ( ' title ' )
if not mrr_title :
continue
mrr_title = get_text ( mrr [ ' title ' ] )
mrr_contents_text = get_text ( mrr [ ' contents ' ] [ 0 ] )
if mrr_title == ' License ' :
info [ ' license ' ] = mrr_contents_text
elif not multiple_songs :
if mrr_title == ' Album ' :
info [ ' album ' ] = mrr_contents_text
elif mrr_title == ' Artist ' :
info [ ' artist ' ] = mrr_contents_text
elif mrr_title == ' Song ' :
info [ ' track ' ] = mrr_contents_text
fallbacks = {
' channel ' : ' uploader ' ,
' channel_id ' : ' uploader_id ' ,
' channel_url ' : ' uploader_url ' ,
}
for to , frm in fallbacks . items ( ) :
if not info . get ( to ) :
info [ to ] = info . get ( frm )
for s_k , d_k in [ ( ' artist ' , ' creator ' ) , ( ' track ' , ' alt_title ' ) ] :
v = info . get ( s_k )
if v :
info [ d_k ] = v
2020-09-15 11:16:58 -04:00
2021-03-21 17:23:34 -04:00
is_private = bool_or_none ( video_details . get ( ' isPrivate ' ) )
is_unlisted = bool_or_none ( microformat . get ( ' isUnlisted ' ) )
is_membersonly = None
2021-04-03 04:50:58 -04:00
is_premium = None
2021-03-21 17:23:34 -04:00
if initial_data and is_private is not None :
is_membersonly = False
2021-04-03 04:50:58 -04:00
is_premium = False
2021-03-21 17:23:34 -04:00
contents = try_get ( initial_data , lambda x : x [ ' contents ' ] [ ' twoColumnWatchNextResults ' ] [ ' results ' ] [ ' results ' ] [ ' contents ' ] , list )
for content in contents or [ ] :
badges = try_get ( content , lambda x : x [ ' videoPrimaryInfoRenderer ' ] [ ' badges ' ] , list )
for badge in badges or [ ] :
label = try_get ( badge , lambda x : x [ ' metadataBadgeRenderer ' ] [ ' label ' ] ) or ' '
if label . lower ( ) == ' members only ' :
is_membersonly = True
break
2021-04-03 04:50:58 -04:00
elif label . lower ( ) == ' premium ' :
is_premium = True
break
if is_membersonly or is_premium :
2021-03-21 17:23:34 -04:00
break
# TODO: Add this for playlists
info [ ' availability ' ] = self . _availability (
is_private = is_private ,
2021-04-03 04:50:58 -04:00
needs_premium = is_premium ,
2021-03-21 17:23:34 -04:00
needs_subscription = is_membersonly ,
needs_auth = info [ ' age_limit ' ] > = 18 ,
is_unlisted = None if is_private is None else is_unlisted )
2021-01-27 10:02:51 -05:00
# get xsrf for annotations or comments
2021-05-17 08:23:08 -04:00
get_annotations = self . get_param ( ' writeannotations ' , False )
get_comments = self . get_param ( ' getcomments ' , False )
2021-01-27 10:02:51 -05:00
if get_annotations or get_comments :
2021-01-01 07:26:37 -05:00
xsrf_token = None
2021-02-04 09:37:17 -05:00
ytcfg = self . _extract_ytcfg ( video_id , webpage )
2021-01-01 07:26:37 -05:00
if ytcfg :
xsrf_token = try_get ( ytcfg , lambda x : x [ ' XSRF_TOKEN ' ] , compat_str )
if not xsrf_token :
xsrf_token = self . _search_regex (
r ' ([ \' " ])XSRF_TOKEN \ 1 \ s*: \ s*([ \' " ])(?P<xsrf_token>(?:(?! \ 2).)+) \ 2 ' ,
2021-02-04 09:57:26 -05:00
webpage , ' xsrf token ' , group = ' xsrf_token ' , fatal = False )
2021-01-27 10:02:51 -05:00
# annotations
if get_annotations :
2019-08-09 03:16:53 -04:00
invideo_url = try_get (
player_response , lambda x : x [ ' annotations ' ] [ 0 ] [ ' playerAnnotationsUrlsRenderer ' ] [ ' invideoUrl ' ] , compat_str )
if xsrf_token and invideo_url :
2021-01-01 07:26:37 -05:00
xsrf_field_name = None
if ytcfg :
xsrf_field_name = try_get ( ytcfg , lambda x : x [ ' XSRF_FIELD_NAME ' ] , compat_str )
if not xsrf_field_name :
xsrf_field_name = self . _search_regex (
r ' ([ \' " ])XSRF_FIELD_NAME \ 1 \ s*: \ s*([ \' " ])(?P<xsrf_field_name> \ w+) \ 2 ' ,
2021-02-04 09:57:26 -05:00
webpage , ' xsrf field name ' ,
2021-01-01 07:26:37 -05:00
group = ' xsrf_field_name ' , default = ' session_token ' )
2021-02-04 09:57:26 -05:00
info [ ' annotations ' ] = self . _download_webpage (
2019-08-09 03:16:53 -04:00
self . _proto_relative_url ( invideo_url ) ,
video_id , note = ' Downloading annotations ' ,
errnote = ' Unable to download video annotations ' , fatal = False ,
data = urlencode_postdata ( { xsrf_field_name : xsrf_token } ) )
2018-06-02 15:08:38 -04:00
2021-02-28 09:56:08 -05:00
if get_comments :
2021-03-14 18:41:11 -04:00
info [ ' __post_extractor ' ] = lambda : self . _extract_comments ( ytcfg , video_id , contents , webpage , xsrf_token )
2013-12-17 21:30:55 -05:00
2021-02-04 09:37:17 -05:00
self . mark_watched ( video_id , player_response )
2016-02-29 14:01:33 -05:00
2021-02-04 09:37:17 -05:00
return info
2013-06-23 13:58:33 -04:00
2014-11-23 14:41:03 -05:00
2020-11-19 14:22:59 -05:00
class YoutubeTabIE ( YoutubeBaseInfoExtractor ) :
IE_DESC = ' YouTube.com tab '
2020-11-23 16:47:42 -05:00
_VALID_URL = r ''' (?x)
https ? : / /
( ? : \w + \. ) ?
( ? :
youtube ( ? : kids ) ? \. com |
invidio \. us
) /
( ? :
2021-05-18 10:01:06 -04:00
( ? P < channel_type > channel | c | user | browse ) / |
2020-11-23 16:47:42 -05:00
( ? P < not_channel >
2021-02-18 03:24:06 -05:00
feed / | hashtag / |
2020-11-23 16:47:42 -05:00
( ? : playlist | watch ) \? . * ? \blist =
) |
2021-01-01 07:26:37 -05:00
( ? ! ( ? : % s ) \b ) # Direct URLs
2020-11-23 16:47:42 -05:00
)
( ? P < id > [ ^ / ? \#&]+)
''' % YoutubeBaseInfoExtractor._RESERVED_NAMES
2020-11-19 14:22:59 -05:00
IE_NAME = ' youtube:tab '
2014-09-13 01:19:20 -04:00
_TESTS = [ {
2021-05-18 08:29:15 -04:00
' note ' : ' playlists, multipage ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid ' ,
' playlist_mincount ' : 94 ,
' info_dict ' : {
' id ' : ' UCqj7Cz7revf5maW9g5pgNcg ' ,
' title ' : ' Игорь Клейнер - Playlists ' ,
' description ' : ' md5:be97ee0f14ee314f1f002cf187166ee2 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' Игорь Клейнер ' ,
' uploader_id ' : ' UCqj7Cz7revf5maW9g5pgNcg ' ,
2020-11-19 14:22:59 -05:00
} ,
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' playlists, multipage, different order ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd ' ,
' playlist_mincount ' : 94 ,
' info_dict ' : {
' id ' : ' UCqj7Cz7revf5maW9g5pgNcg ' ,
' title ' : ' Игорь Клейнер - Playlists ' ,
' description ' : ' md5:be97ee0f14ee314f1f002cf187166ee2 ' ,
2021-02-09 11:31:34 -05:00
' uploader_id ' : ' UCqj7Cz7revf5maW9g5pgNcg ' ,
' uploader ' : ' Игорь Клейнер ' ,
2020-11-19 14:22:59 -05:00
} ,
2021-04-16 23:02:33 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' playlists, series ' ,
2021-04-16 23:02:33 -04:00
' url ' : ' https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3 ' ,
' playlist_mincount ' : 5 ,
' info_dict ' : {
' id ' : ' UCYO_jab_esuFRV4b17AJtAw ' ,
' title ' : ' 3Blue1Brown - Playlists ' ,
' description ' : ' md5:e1384e8a133307dd10edee76e875d62f ' ,
2021-05-18 08:29:15 -04:00
' uploader_id ' : ' UCYO_jab_esuFRV4b17AJtAw ' ,
' uploader ' : ' 3Blue1Brown ' ,
2021-04-16 23:02:33 -04:00
} ,
2020-11-19 14:22:59 -05:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' playlists, singlepage ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/user/ThirstForScience/playlists ' ,
' playlist_mincount ' : 4 ,
' info_dict ' : {
' id ' : ' UCAEtajcuhQ6an9WEzY9LEMQ ' ,
' title ' : ' ThirstForScience - Playlists ' ,
' description ' : ' md5:609399d937ea957b0f53cbffb747a14c ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' ThirstForScience ' ,
' uploader_id ' : ' UCAEtajcuhQ6an9WEzY9LEMQ ' ,
2020-11-19 14:22:59 -05:00
}
} , {
' url ' : ' https://www.youtube.com/c/ChristophLaimer/playlists ' ,
' only_matching ' : True ,
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' basic, single video playlist ' ,
2020-03-02 13:46:00 -05:00
' url ' : ' https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc ' ,
2014-09-13 01:19:20 -04:00
' info_dict ' : {
2020-03-02 13:46:00 -05:00
' uploader_id ' : ' UCmlqkdCBesrv2Lak1mF_MxA ' ,
' uploader ' : ' Sergey M. ' ,
' id ' : ' PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc ' ,
2020-09-02 16:37:35 -04:00
' title ' : ' youtube-dl public playlist ' ,
2014-09-13 01:19:20 -04:00
} ,
2020-03-02 13:46:00 -05:00
' playlist_count ' : 1 ,
2014-09-13 01:31:48 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' empty playlist ' ,
2020-03-02 13:46:00 -05:00
' url ' : ' https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf ' ,
2014-09-13 01:31:48 -04:00
' info_dict ' : {
2020-03-02 13:46:00 -05:00
' uploader_id ' : ' UCmlqkdCBesrv2Lak1mF_MxA ' ,
' uploader ' : ' Sergey M. ' ,
' id ' : ' PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf ' ,
2020-09-02 16:37:35 -04:00
' title ' : ' youtube-dl empty playlist ' ,
2014-09-13 01:31:48 -04:00
} ,
' playlist_count ' : 0 ,
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Home tab ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured ' ,
2014-09-13 01:31:48 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Home ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2014-09-13 01:31:48 -04:00
} ,
2020-11-19 14:22:59 -05:00
' playlist_mincount ' : 2 ,
2014-09-13 01:31:48 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Videos tab ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos ' ,
2014-09-13 01:31:48 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Videos ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2014-09-13 01:31:48 -04:00
} ,
2020-11-19 14:22:59 -05:00
' playlist_mincount ' : 975 ,
2014-09-13 01:31:48 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Videos tab, sorted by popular ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid ' ,
2014-09-13 01:31:48 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Videos ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2014-09-13 01:31:48 -04:00
} ,
2020-11-19 14:22:59 -05:00
' playlist_mincount ' : 199 ,
2014-09-13 01:31:48 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Playlists tab ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists ' ,
2014-09-13 01:31:48 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Playlists ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2014-09-13 01:31:48 -04:00
} ,
2020-11-19 14:22:59 -05:00
' playlist_mincount ' : 17 ,
2014-09-24 04:34:29 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Community tab ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community ' ,
2014-09-24 04:34:29 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Community ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2020-11-19 14:22:59 -05:00
} ,
' playlist_mincount ' : 18 ,
2017-02-28 11:06:47 -05:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Channels tab ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels ' ,
2017-02-28 11:06:47 -05:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
' title ' : ' lex will - Channels ' ,
' description ' : ' md5:2163c5d0ff54ed5f598d6a7e6211e488 ' ,
2021-02-09 11:31:34 -05:00
' uploader ' : ' lex will ' ,
' uploader_id ' : ' UCKfVa3S1e4PHvxWcwyMMg8w ' ,
2020-11-19 14:22:59 -05:00
} ,
2021-02-09 11:31:34 -05:00
' playlist_mincount ' : 12 ,
2021-05-18 10:00:21 -04:00
} , {
' note ' : ' Search tab ' ,
' url ' : ' https://www.youtube.com/c/3blue1brown/search?query=linear %20a lgebra ' ,
' playlist_mincount ' : 40 ,
' info_dict ' : {
' id ' : ' UCYO_jab_esuFRV4b17AJtAw ' ,
' title ' : ' 3Blue1Brown - Search - linear algebra ' ,
' description ' : ' md5:e1384e8a133307dd10edee76e875d62f ' ,
' uploader ' : ' 3Blue1Brown ' ,
' uploader_id ' : ' UCYO_jab_esuFRV4b17AJtAw ' ,
} ,
2014-09-24 19:58:49 -04:00
} , {
2020-11-21 09:50:42 -05:00
' url ' : ' https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA ' ,
2020-11-19 14:22:59 -05:00
' only_matching ' : True ,
} , {
2020-11-21 09:50:42 -05:00
' url ' : ' https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA ' ,
2020-11-19 14:22:59 -05:00
' only_matching ' : True ,
} , {
2020-11-21 09:50:42 -05:00
' url ' : ' https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA ' ,
2020-11-19 14:22:59 -05:00
' only_matching ' : True ,
} , {
' note ' : ' Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list. ' ,
' url ' : ' https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC ' ,
' info_dict ' : {
' title ' : ' 29C3: Not my department ' ,
' id ' : ' PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC ' ,
' uploader ' : ' Christiaan008 ' ,
' uploader_id ' : ' UCEPzS1rYsrkqzSLNp76nrcg ' ,
2021-02-09 11:31:34 -05:00
' description ' : ' md5:a14dc1a8ef8307a9807fe136a0660268 ' ,
2020-11-19 14:22:59 -05:00
} ,
' playlist_count ' : 96 ,
} , {
' note ' : ' Large playlist ' ,
' url ' : ' https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q ' ,
2014-09-24 19:58:49 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' title ' : ' Uploads from Cauchemar ' ,
' id ' : ' UUBABnxM4Ar9ten8Mdjj1j0Q ' ,
' uploader ' : ' Cauchemar ' ,
' uploader_id ' : ' UCBABnxM4Ar9ten8Mdjj1j0Q ' ,
2019-07-20 13:01:46 -04:00
} ,
2020-11-19 14:22:59 -05:00
' playlist_mincount ' : 1123 ,
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' even larger playlist, 8832 videos ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' http://www.youtube.com/user/NASAgovVideo/videos ' ,
' only_matching ' : True ,
2014-12-15 13:19:15 -05:00
} , {
' note ' : ' Buggy playlist: the webpage has a " Load more " button but it doesn \' t have more videos ' ,
' url ' : ' https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA ' ,
' info_dict ' : {
2015-02-01 09:33:32 -05:00
' title ' : ' Uploads from Interstellar Movie ' ,
' id ' : ' UUXw-G3eDE9trcvY2sBMM_aA ' ,
2019-07-20 13:01:46 -04:00
' uploader ' : ' Interstellar Movie ' ,
2020-11-19 14:22:59 -05:00
' uploader_id ' : ' UCXw-G3eDE9trcvY2sBMM_aA ' ,
2014-12-15 13:19:15 -05:00
} ,
2016-10-18 16:27:18 -04:00
' playlist_mincount ' : 21 ,
2021-04-16 18:39:08 -04:00
} , {
' note ' : ' Playlist with " show unavailable videos " button ' ,
' url ' : ' https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q ' ,
' info_dict ' : {
' title ' : ' Uploads from Phim Siêu Nhân Nhật Bản ' ,
' id ' : ' UUTYLiWFZy8xtPwxFwX9rV7Q ' ,
' uploader ' : ' Phim Siêu Nhân Nhật Bản ' ,
' uploader_id ' : ' UCTYLiWFZy8xtPwxFwX9rV7Q ' ,
} ,
2021-05-18 08:29:15 -04:00
' playlist_mincount ' : 200 ,
2021-04-16 22:19:49 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Playlist with unavailable videos in page 7 ' ,
2021-04-16 22:19:49 -04:00
' url ' : ' https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w ' ,
' info_dict ' : {
' title ' : ' Uploads from BlankTV ' ,
' id ' : ' UU8l9frL61Yl5KFOl87nIm2w ' ,
' uploader ' : ' BlankTV ' ,
' uploader_id ' : ' UC8l9frL61Yl5KFOl87nIm2w ' ,
} ,
2021-05-18 08:29:15 -04:00
' playlist_mincount ' : 1000 ,
2020-11-19 14:22:59 -05:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' https://github.com/ytdl-org/youtube-dl/issues/21844 ' ,
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba ' ,
' info_dict ' : {
' title ' : ' Data Analysis with Dr Mike Pound ' ,
' id ' : ' PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba ' ,
' uploader_id ' : ' UC9-y-6csu5WGm29I7JiwpnA ' ,
' uploader ' : ' Computerphile ' ,
2021-02-09 11:31:34 -05:00
' description ' : ' md5:7f567c574d13d3f8c0954d9ffee4e487 ' ,
2020-11-19 14:22:59 -05:00
} ,
' playlist_mincount ' : 11 ,
} , {
2020-11-21 09:50:42 -05:00
' url ' : ' https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc ' ,
2020-11-19 14:22:59 -05:00
' only_matching ' : True ,
2016-09-02 12:43:20 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Playlist URL that does not actually serve a playlist ' ,
2016-09-02 12:43:20 -04:00
' url ' : ' https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4 ' ,
' info_dict ' : {
' id ' : ' FqZTN594JQw ' ,
' ext ' : ' webm ' ,
' title ' : " Smiley ' s People 01 detective, Adventure Series, Action " ,
' uploader ' : ' STREEM ' ,
' uploader_id ' : ' UCyPhqAZgwYWZfxElWVbVJng ' ,
2017-01-02 07:08:07 -05:00
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/channel/UCyPhqAZgwYWZfxElWVbVJng ' ,
2016-09-02 12:43:20 -04:00
' upload_date ' : ' 20150526 ' ,
' license ' : ' Standard YouTube License ' ,
' description ' : ' md5:507cdcb5a49ac0da37a920ece610be80 ' ,
' categories ' : [ ' People & Blogs ' ] ,
' tags ' : list ,
2018-11-02 19:26:16 -04:00
' view_count ' : int ,
2016-09-02 12:43:20 -04:00
' like_count ' : int ,
' dislike_count ' : int ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2019-07-20 13:01:46 -04:00
' skip ' : ' This video is not available. ' ,
2016-09-02 12:43:20 -04:00
' add_ie ' : [ YoutubeIE . ie_key ( ) ] ,
2016-10-18 16:27:18 -04:00
} , {
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g ' ,
2018-11-05 07:08:39 -05:00
' only_matching ' : True ,
2019-11-30 11:51:34 -05:00
} , {
2020-11-19 14:22:59 -05:00
' url ' : ' https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM ' ,
2019-11-30 11:51:34 -05:00
' only_matching ' : True ,
2020-11-21 09:50:42 -05:00
} , {
' url ' : ' https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live ' ,
' info_dict ' : {
2021-05-18 08:29:15 -04:00
' id ' : ' X1whbWASnNQ ' , # This will keep changing
2020-11-21 09:50:42 -05:00
' ext ' : ' mp4 ' ,
2021-02-09 11:31:34 -05:00
' title ' : compat_str ,
2020-11-21 09:50:42 -05:00
' uploader ' : ' Sky News ' ,
' uploader_id ' : ' skynews ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/skynews ' ,
2021-05-18 08:29:15 -04:00
' upload_date ' : r ' re: \ d {8} ' ,
' description ' : compat_str ,
2020-11-21 09:50:42 -05:00
' categories ' : [ ' News & Politics ' ] ,
' tags ' : list ,
' like_count ' : int ,
' dislike_count ' : int ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
2021-05-18 08:29:15 -04:00
' expected_warnings ' : [ ' Downloading just video ' , ' Ignoring subtitle tracks found in ' ] ,
2020-11-21 09:50:42 -05:00
} , {
' url ' : ' https://www.youtube.com/user/TheYoungTurks/live ' ,
' info_dict ' : {
' id ' : ' a48o2S1cPoo ' ,
' ext ' : ' mp4 ' ,
' title ' : ' The Young Turks - Live Main Show ' ,
' uploader ' : ' The Young Turks ' ,
' uploader_id ' : ' TheYoungTurks ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/TheYoungTurks ' ,
' upload_date ' : ' 20150715 ' ,
' license ' : ' Standard YouTube License ' ,
' description ' : ' md5:438179573adcdff3c97ebb1ee632b891 ' ,
' categories ' : [ ' News & Politics ' ] ,
' tags ' : [ ' Cenk Uygur (TV Program Creator) ' , ' The Young Turks (Award-Winning Work) ' , ' Talk Show (TV Genre) ' ] ,
' like_count ' : int ,
' dislike_count ' : int ,
} ,
' params ' : {
' skip_download ' : True ,
} ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/c/CommanderVideoHq/live ' ,
' only_matching ' : True ,
2021-05-21 10:34:30 -04:00
} , {
' note ' : ' A channel that is not live. Should raise error ' ,
' url ' : ' https://www.youtube.com/user/numberphile/live ' ,
' only_matching ' : True ,
2020-11-23 16:59:10 -05:00
} , {
' url ' : ' https://www.youtube.com/feed/trending ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/feed/library ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/feed/history ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/feed/subscriptions ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/feed/watch_later ' ,
' only_matching ' : True ,
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Recommended - redirects to home page ' ,
2020-11-23 16:59:10 -05:00
' url ' : ' https://www.youtube.com/feed/recommended ' ,
' only_matching ' : True ,
2021-01-01 07:26:37 -05:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' inline playlist with not always working continuations ' ,
2021-01-01 07:26:37 -05:00
' url ' : ' https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8 ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/course ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/zsecurity ' ,
' only_matching ' : True ,
} , {
' url ' : ' http://www.youtube.com/NASAgovVideo/videos ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/TheYoungTurks/live ' ,
' only_matching ' : True ,
2021-04-03 18:05:17 -04:00
} , {
' url ' : ' https://www.youtube.com/hashtag/cctv9 ' ,
' info_dict ' : {
' id ' : ' cctv9 ' ,
' title ' : ' #cctv9 ' ,
} ,
' playlist_mincount ' : 350 ,
2021-04-16 23:02:33 -04:00
} , {
' url ' : ' https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU ' ,
' only_matching ' : True ,
2021-05-15 10:38:47 -04:00
} , {
2021-05-18 08:29:15 -04:00
' note ' : ' Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist ' ,
2021-05-15 10:38:47 -04:00
' url ' : ' https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq ' ,
' only_matching ' : True
2021-05-18 10:01:06 -04:00
} , {
' note ' : ' /browse/ should redirect to /channel/ ' ,
' url ' : ' https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng ' ,
' only_matching ' : True
} , {
' note ' : ' VLPL, should redirect to playlist?list=PL... ' ,
' url ' : ' https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq ' ,
' info_dict ' : {
' id ' : ' PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq ' ,
' uploader ' : ' NoCopyrightSounds ' ,
' description ' : ' Providing you with copyright free / safe music for gaming, live streaming, studying and more! ' ,
' uploader_id ' : ' UC_aEa8K-EOJ3D6gOs7HcyNg ' ,
' title ' : ' NCS Releases ' ,
} ,
' playlist_mincount ' : 166 ,
2021-05-18 09:02:46 -04:00
} , {
' note ' : ' Topic, should redirect to playlist?list=UU... ' ,
' url ' : ' https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw ' ,
' info_dict ' : {
' id ' : ' UU9ALqqC4aIeG5iDs7i90Bfw ' ,
' uploader_id ' : ' UC9ALqqC4aIeG5iDs7i90Bfw ' ,
' title ' : ' Uploads from Royalty Free Music - Topic ' ,
' uploader ' : ' Royalty Free Music - Topic ' ,
} ,
' expected_warnings ' : [
' A channel/user page was given ' ,
' The URL does not have a videos tab ' ,
] ,
' playlist_mincount ' : 101 ,
} , {
' note ' : ' Topic without a UU playlist ' ,
' url ' : ' https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg ' ,
' info_dict ' : {
' id ' : ' UCtFRv9O2AHqOZjjynzrv-xg ' ,
' title ' : ' UCtFRv9O2AHqOZjjynzrv-xg ' ,
} ,
' expected_warnings ' : [
' A channel/user page was given ' ,
' The URL does not have a videos tab ' ,
' Falling back to channel URL ' ,
] ,
' playlist_mincount ' : 9 ,
2021-05-17 09:07:58 -04:00
} , {
' note ' : ' Youtube music Album ' ,
' url ' : ' https://music.youtube.com/browse/MPREb_gTAcphH99wE ' ,
' info_dict ' : {
' id ' : ' OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0 ' ,
' title ' : ' Album - Royalty Free Music Library V2 (50 Songs) ' ,
} ,
' playlist_count ' : 50 ,
2021-01-01 07:26:37 -05:00
} ]
@classmethod
def suitable ( cls , url ) :
return False if YoutubeIE . suitable ( url ) else super (
YoutubeTabIE , cls ) . suitable ( url )
2020-11-19 14:22:59 -05:00
def _extract_channel_id ( self , webpage ) :
channel_id = self . _html_search_meta (
' channelId ' , webpage , ' channel id ' , default = None )
if channel_id :
return channel_id
channel_url = self . _html_search_meta (
( ' og:url ' , ' al:ios:url ' , ' al:android:url ' , ' al:web:url ' ,
' twitter:url ' , ' twitter:app:url:iphone ' , ' twitter:app:url:ipad ' ,
' twitter:app:url:googleplay ' ) , webpage , ' channel url ' )
return self . _search_regex (
r ' https?://(?:www \ .)?youtube \ .com/channel/([^/?#&])+ ' ,
channel_url , ' channel id ' )
2020-11-03 10:15:16 -05:00
2020-11-19 14:22:59 -05:00
@staticmethod
2021-03-04 13:05:26 -05:00
def _extract_basic_item_renderer ( item ) :
# Modified from _extract_grid_item_renderer
2021-04-16 23:02:33 -04:00
known_basic_renderers = (
' playlistRenderer ' , ' videoRenderer ' , ' channelRenderer ' , ' showRenderer '
2021-03-04 13:05:26 -05:00
)
for key , renderer in item . items ( ) :
2021-04-16 23:02:33 -04:00
if not isinstance ( renderer , dict ) :
2021-03-04 13:05:26 -05:00
continue
2021-04-16 23:02:33 -04:00
elif key in known_basic_renderers :
return renderer
elif key . startswith ( ' grid ' ) and key . endswith ( ' Renderer ' ) :
return renderer
2020-11-19 14:22:59 -05:00
def _grid_entries ( self , grid_renderer ) :
for item in grid_renderer [ ' items ' ] :
if not isinstance ( item , dict ) :
2015-04-28 11:07:56 -04:00
continue
2021-03-04 13:05:26 -05:00
renderer = self . _extract_basic_item_renderer ( item )
2020-11-19 14:22:59 -05:00
if not isinstance ( renderer , dict ) :
continue
title = try_get (
2021-04-16 23:02:33 -04:00
renderer , ( lambda x : x [ ' title ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] ,
lambda x : x [ ' title ' ] [ ' simpleText ' ] ) , compat_str )
2020-11-19 14:22:59 -05:00
# playlist
playlist_id = renderer . get ( ' playlistId ' )
if playlist_id :
yield self . url_result (
' https://www.youtube.com/playlist?list= %s ' % playlist_id ,
ie = YoutubeTabIE . ie_key ( ) , video_id = playlist_id ,
video_title = title )
2021-04-16 23:02:33 -04:00
continue
2020-11-19 14:22:59 -05:00
# video
video_id = renderer . get ( ' videoId ' )
if video_id :
yield self . _extract_video ( renderer )
2021-04-16 23:02:33 -04:00
continue
2020-11-19 14:22:59 -05:00
# channel
channel_id = renderer . get ( ' channelId ' )
if channel_id :
title = try_get (
renderer , lambda x : x [ ' title ' ] [ ' simpleText ' ] , compat_str )
yield self . url_result (
' https://www.youtube.com/channel/ %s ' % channel_id ,
ie = YoutubeTabIE . ie_key ( ) , video_title = title )
2021-04-16 23:02:33 -04:00
continue
# generic endpoint URL support
ep_url = urljoin ( ' https://www.youtube.com/ ' , try_get (
renderer , lambda x : x [ ' navigationEndpoint ' ] [ ' commandMetadata ' ] [ ' webCommandMetadata ' ] [ ' url ' ] ,
compat_str ) )
if ep_url :
for ie in ( YoutubeTabIE , YoutubePlaylistIE , YoutubeIE ) :
if ie . suitable ( ep_url ) :
yield self . url_result (
ep_url , ie = ie . ie_key ( ) , video_id = ie . _match_id ( ep_url ) , video_title = title )
break
2020-11-19 14:22:59 -05:00
2020-11-23 16:59:10 -05:00
def _shelf_entries_from_content ( self , shelf_renderer ) :
content = shelf_renderer . get ( ' content ' )
if not isinstance ( content , dict ) :
2020-11-19 14:22:59 -05:00
return
2021-03-04 13:05:26 -05:00
renderer = content . get ( ' gridRenderer ' ) or content . get ( ' expandedShelfContentsRenderer ' )
2020-11-23 16:59:10 -05:00
if renderer :
# TODO: add support for nested playlists so each shelf is processed
# as separate playlist
# TODO: this includes only first N items
for entry in self . _grid_entries ( renderer ) :
yield entry
renderer = content . get ( ' horizontalListRenderer ' )
if renderer :
# TODO
pass
2020-11-19 14:22:59 -05:00
2021-01-01 07:26:37 -05:00
def _shelf_entries ( self , shelf_renderer , skip_channels = False ) :
2020-11-19 14:22:59 -05:00
ep = try_get (
shelf_renderer , lambda x : x [ ' endpoint ' ] [ ' commandMetadata ' ] [ ' webCommandMetadata ' ] [ ' url ' ] ,
compat_str )
shelf_url = urljoin ( ' https://www.youtube.com ' , ep )
2020-11-23 16:59:10 -05:00
if shelf_url :
2021-01-01 07:26:37 -05:00
# Skipping links to another channels, note that checking for
# endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
# will not work
if skip_channels and ' /channels? ' in shelf_url :
return
2020-11-23 16:59:10 -05:00
title = try_get (
shelf_renderer , lambda x : x [ ' title ' ] [ ' runs ' ] [ 0 ] [ ' text ' ] , compat_str )
yield self . url_result ( shelf_url , video_title = title )
# Shelf may not contain shelf URL, fallback to extraction from content
for entry in self . _shelf_entries_from_content ( shelf_renderer ) :
yield entry
2013-06-23 13:58:33 -04:00
2020-11-19 14:22:59 -05:00
def _playlist_entries ( self , video_list_renderer ) :
for content in video_list_renderer [ ' contents ' ] :
if not isinstance ( content , dict ) :
continue
renderer = content . get ( ' playlistVideoRenderer ' ) or content . get ( ' playlistPanelVideoRenderer ' )
if not isinstance ( renderer , dict ) :
continue
video_id = renderer . get ( ' videoId ' )
if not video_id :
continue
yield self . _extract_video ( renderer )
2017-12-18 15:51:28 -05:00
2020-11-21 17:21:09 -05:00
def _rich_entries ( self , rich_grid_renderer ) :
renderer = try_get (
2020-11-23 16:47:42 -05:00
rich_grid_renderer , lambda x : x [ ' content ' ] [ ' videoRenderer ' ] , dict ) or { }
2020-11-21 17:21:09 -05:00
video_id = renderer . get ( ' videoId ' )
if not video_id :
return
yield self . _extract_video ( renderer )
2020-11-19 14:22:59 -05:00
def _video_entry ( self , video_renderer ) :
video_id = video_renderer . get ( ' videoId ' )
if video_id :
return self . _extract_video ( video_renderer )
2016-09-02 12:43:20 -04:00
2020-11-19 14:22:59 -05:00
def _post_thread_entries ( self , post_thread_renderer ) :
post_renderer = try_get (
post_thread_renderer , lambda x : x [ ' post ' ] [ ' backstagePostRenderer ' ] , dict )
if not post_renderer :
return
# video attachment
video_renderer = try_get (
2021-04-27 02:03:58 -04:00
post_renderer , lambda x : x [ ' backstageAttachment ' ] [ ' videoRenderer ' ] , dict ) or { }
video_id = video_renderer . get ( ' videoId ' )
if video_id :
entry = self . _extract_video ( video_renderer )
2020-11-19 14:22:59 -05:00
if entry :
yield entry
2021-04-27 02:03:58 -04:00
# playlist attachment
playlist_id = try_get (
post_renderer , lambda x : x [ ' backstageAttachment ' ] [ ' playlistRenderer ' ] [ ' playlistId ' ] , compat_str )
if playlist_id :
yield self . url_result (
2021-04-28 10:29:40 -04:00
' https://www.youtube.com/playlist?list= %s ' % playlist_id ,
ie = YoutubeTabIE . ie_key ( ) , video_id = playlist_id )
2020-11-19 14:22:59 -05:00
# inline video links
runs = try_get ( post_renderer , lambda x : x [ ' contentText ' ] [ ' runs ' ] , list ) or [ ]
for run in runs :
if not isinstance ( run , dict ) :
continue
ep_url = try_get (
run , lambda x : x [ ' navigationEndpoint ' ] [ ' urlEndpoint ' ] [ ' url ' ] , compat_str )
if not ep_url :
continue
if not YoutubeIE . suitable ( ep_url ) :
continue
ep_video_id = YoutubeIE . _match_id ( ep_url )
if video_id == ep_video_id :
continue
2021-04-27 02:03:58 -04:00
yield self . url_result ( ep_url , ie = YoutubeIE . ie_key ( ) , video_id = ep_video_id )
2016-09-02 12:43:20 -04:00
2020-11-19 14:22:59 -05:00
def _post_thread_continuation_entries ( self , post_thread_continuation ) :
contents = post_thread_continuation . get ( ' contents ' )
if not isinstance ( contents , list ) :
return
for content in contents :
renderer = content . get ( ' backstagePostThreadRenderer ' )
if not isinstance ( renderer , dict ) :
continue
for entry in self . _post_thread_entries ( renderer ) :
yield entry
2017-12-18 15:51:28 -05:00
2021-04-03 18:05:17 -04:00
r ''' # unused
def _rich_grid_entries ( self , contents ) :
for content in contents :
video_renderer = try_get ( content , lambda x : x [ ' richItemRenderer ' ] [ ' content ' ] [ ' videoRenderer ' ] , dict )
if video_renderer :
entry = self . _video_entry ( video_renderer )
if entry :
yield entry
'''
2021-01-01 07:26:37 -05:00
@staticmethod
def _build_continuation_query ( continuation , ctp = None ) :
query = {
' ctoken ' : continuation ,
' continuation ' : continuation ,
}
if ctp :
query [ ' itct ' ] = ctp
return query
2020-11-19 14:22:59 -05:00
@staticmethod
def _extract_next_continuation_data ( renderer ) :
next_continuation = try_get (
renderer , lambda x : x [ ' continuations ' ] [ 0 ] [ ' nextContinuationData ' ] , dict )
if not next_continuation :
return
continuation = next_continuation . get ( ' continuation ' )
if not continuation :
return
ctp = next_continuation . get ( ' clickTrackingParams ' )
2021-01-01 07:26:37 -05:00
return YoutubeTabIE . _build_continuation_query ( continuation , ctp )
2013-06-23 13:58:33 -04:00
2020-11-19 14:22:59 -05:00
@classmethod
def _extract_continuation ( cls , renderer ) :
next_continuation = cls . _extract_next_continuation_data ( renderer )
if next_continuation :
return next_continuation
2021-02-10 16:22:55 -05:00
contents = [ ]
for key in ( ' contents ' , ' items ' ) :
contents . extend ( try_get ( renderer , lambda x : x [ key ] , list ) or [ ] )
2020-11-19 14:22:59 -05:00
for content in contents :
if not isinstance ( content , dict ) :
continue
continuation_ep = try_get (
content , lambda x : x [ ' continuationItemRenderer ' ] [ ' continuationEndpoint ' ] ,
dict )
if not continuation_ep :
continue
continuation = try_get (
continuation_ep , lambda x : x [ ' continuationCommand ' ] [ ' token ' ] , compat_str )
if not continuation :
continue
ctp = continuation_ep . get ( ' clickTrackingParams ' )
2021-01-01 07:26:37 -05:00
return YoutubeTabIE . _build_continuation_query ( continuation , ctp )
2015-03-26 11:41:09 -04:00
2021-04-11 16:23:04 -04:00
def _entries ( self , tab , item_id , identity_token , account_syncid , ytcfg ) :
2020-11-21 17:21:09 -05:00
2020-11-23 16:47:42 -05:00
def extract_entries ( parent_renderer ) : # this needs to called again for continuation to work with feeds
contents = try_get ( parent_renderer , lambda x : x [ ' contents ' ] , list ) or [ ]
for content in contents :
if not isinstance ( content , dict ) :
2020-11-19 14:22:59 -05:00
continue
2020-11-23 16:47:42 -05:00
is_renderer = try_get ( content , lambda x : x [ ' itemSectionRenderer ' ] , dict )
2020-11-21 17:21:09 -05:00
if not is_renderer :
2020-11-23 16:47:42 -05:00
renderer = content . get ( ' richItemRenderer ' )
2020-11-21 17:21:09 -05:00
if renderer :
for entry in self . _rich_entries ( renderer ) :
yield entry
continuation_list [ 0 ] = self . _extract_continuation ( parent_renderer )
2020-11-19 14:22:59 -05:00
continue
2020-11-21 17:21:09 -05:00
isr_contents = try_get ( is_renderer , lambda x : x [ ' contents ' ] , list ) or [ ]
for isr_content in isr_contents :
if not isinstance ( isr_content , dict ) :
continue
2021-02-09 11:07:59 -05:00
known_renderers = {
' playlistVideoListRenderer ' : self . _playlist_entries ,
' gridRenderer ' : self . _grid_entries ,
' shelfRenderer ' : lambda x : self . _shelf_entries ( x , tab . get ( ' title ' ) != ' Channels ' ) ,
' backstagePostThreadRenderer ' : self . _post_thread_entries ,
' videoRenderer ' : lambda x : [ self . _video_entry ( x ) ] ,
}
for key , renderer in isr_content . items ( ) :
if key not in known_renderers :
continue
for entry in known_renderers [ key ] ( renderer ) :
if entry :
yield entry
2020-11-21 17:21:09 -05:00
continuation_list [ 0 ] = self . _extract_continuation ( renderer )
2021-02-09 11:07:59 -05:00
break
2020-11-23 16:47:42 -05:00
2020-11-21 17:21:09 -05:00
if not continuation_list [ 0 ] :
continuation_list [ 0 ] = self . _extract_continuation ( is_renderer )
2020-11-23 16:47:42 -05:00
if not continuation_list [ 0 ] :
continuation_list [ 0 ] = self . _extract_continuation ( parent_renderer )
2020-11-21 17:21:09 -05:00
continuation_list = [ None ] # Python 2 doesnot support nonlocal
2021-01-01 07:26:37 -05:00
tab_content = try_get ( tab , lambda x : x [ ' content ' ] , dict )
if not tab_content :
return
2020-11-21 17:21:09 -05:00
parent_renderer = (
2021-01-01 07:26:37 -05:00
try_get ( tab_content , lambda x : x [ ' sectionListRenderer ' ] , dict )
or try_get ( tab_content , lambda x : x [ ' richGridRenderer ' ] , dict ) or { } )
2020-11-23 16:47:42 -05:00
for entry in extract_entries ( parent_renderer ) :
yield entry
2020-11-21 17:21:09 -05:00
continuation = continuation_list [ 0 ]
2021-04-11 16:23:04 -04:00
context = self . _extract_context ( ytcfg )
visitor_data = try_get ( context , lambda x : x [ ' client ' ] [ ' visitorData ' ] , compat_str )
2021-03-05 08:59:14 -05:00
2020-11-19 14:22:59 -05:00
for page_num in itertools . count ( 1 ) :
if not continuation :
break
2021-04-15 07:22:59 -04:00
query = {
' continuation ' : continuation [ ' continuation ' ] ,
' clickTracking ' : { ' clickTrackingParams ' : continuation [ ' itct ' ] }
}
2021-04-11 16:23:04 -04:00
headers = self . _generate_api_headers ( ytcfg , identity_token , account_syncid , visitor_data )
2021-04-15 07:22:59 -04:00
response = self . _extract_response (
item_id = ' %s page %s ' % ( item_id , page_num ) ,
query = query , headers = headers , ytcfg = ytcfg ,
check_get_keys = ( ' continuationContents ' , ' onResponseReceivedActions ' , ' onResponseReceivedEndpoints ' ) )
2021-03-03 05:32:40 -05:00
if not response :
2020-11-19 14:22:59 -05:00
break
2021-04-11 16:23:04 -04:00
visitor_data = try_get (
response , lambda x : x [ ' responseContext ' ] [ ' visitorData ' ] , compat_str ) or visitor_data
2016-02-18 11:03:46 -05:00
2021-02-09 11:07:59 -05:00
known_continuation_renderers = {
' playlistVideoListContinuation ' : self . _playlist_entries ,
' gridContinuation ' : self . _grid_entries ,
' itemSectionContinuation ' : self . _post_thread_continuation_entries ,
' sectionListContinuation ' : extract_entries , # for feeds
}
2020-11-19 14:22:59 -05:00
continuation_contents = try_get (
2021-02-09 11:07:59 -05:00
response , lambda x : x [ ' continuationContents ' ] , dict ) or { }
continuation_renderer = None
for key , value in continuation_contents . items ( ) :
if key not in known_continuation_renderers :
2020-11-21 17:21:09 -05:00
continue
2021-02-09 11:07:59 -05:00
continuation_renderer = value
continuation_list = [ None ]
for entry in known_continuation_renderers [ key ] ( continuation_renderer ) :
yield entry
continuation = continuation_list [ 0 ] or self . _extract_continuation ( continuation_renderer )
break
if continuation_renderer :
continue
2013-06-23 13:58:33 -04:00
2021-02-09 10:02:12 -05:00
known_renderers = {
' gridPlaylistRenderer ' : ( self . _grid_entries , ' items ' ) ,
' gridVideoRenderer ' : ( self . _grid_entries , ' items ' ) ,
' playlistVideoRenderer ' : ( self . _playlist_entries , ' contents ' ) ,
2021-03-04 13:05:26 -05:00
' itemSectionRenderer ' : ( extract_entries , ' contents ' ) , # for feeds
2021-02-18 03:24:06 -05:00
' richItemRenderer ' : ( extract_entries , ' contents ' ) , # for hashtag
2021-03-07 00:27:45 -05:00
' backstagePostThreadRenderer ' : ( self . _post_thread_continuation_entries , ' contents ' )
2021-02-09 10:02:12 -05:00
}
2021-04-01 04:28:33 -04:00
on_response_received = dict_get ( response , ( ' onResponseReceivedActions ' , ' onResponseReceivedEndpoints ' ) )
2020-11-19 14:22:59 -05:00
continuation_items = try_get (
2021-04-01 04:28:33 -04:00
on_response_received , lambda x : x [ 0 ] [ ' appendContinuationItemsAction ' ] [ ' continuationItems ' ] , list )
2021-02-09 10:02:12 -05:00
continuation_item = try_get ( continuation_items , lambda x : x [ 0 ] , dict ) or { }
video_items_renderer = None
for key , value in continuation_item . items ( ) :
if key not in known_renderers :
2020-11-19 14:22:59 -05:00
continue
2021-02-09 10:02:12 -05:00
video_items_renderer = { known_renderers [ key ] [ 1 ] : continuation_items }
2021-02-18 03:24:06 -05:00
continuation_list = [ None ]
2021-02-09 10:02:12 -05:00
for entry in known_renderers [ key ] [ 0 ] ( video_items_renderer ) :
yield entry
2021-02-18 03:24:06 -05:00
continuation = continuation_list [ 0 ] or self . _extract_continuation ( video_items_renderer )
2021-02-09 10:02:12 -05:00
break
if video_items_renderer :
continue
2020-11-19 14:22:59 -05:00
break
2016-07-08 21:37:02 -04:00
2020-11-19 14:22:59 -05:00
@staticmethod
def _extract_selected_tab ( tabs ) :
for tab in tabs :
2021-05-18 10:00:21 -04:00
renderer = dict_get ( tab , ( ' tabRenderer ' , ' expandableTabRenderer ' ) ) or { }
if renderer . get ( ' selected ' ) is True :
return renderer
2015-09-13 18:32:20 -04:00
else :
2020-11-19 14:22:59 -05:00
raise ExtractorError ( ' Unable to find selected tab ' )
2014-12-06 08:02:19 -05:00
2020-11-19 14:22:59 -05:00
@staticmethod
def _extract_uploader ( data ) :
uploader = { }
sidebar_renderer = try_get (
data , lambda x : x [ ' sidebar ' ] [ ' playlistSidebarRenderer ' ] [ ' items ' ] , list )
if sidebar_renderer :
for item in sidebar_renderer :
if not isinstance ( item , dict ) :
continue
renderer = item . get ( ' playlistSidebarSecondaryInfoRenderer ' )
if not isinstance ( renderer , dict ) :
continue
owner = try_get (
renderer , lambda x : x [ ' videoOwner ' ] [ ' videoOwnerRenderer ' ] [ ' title ' ] [ ' runs ' ] [ 0 ] , dict )
if owner :
uploader [ ' uploader ' ] = owner . get ( ' text ' )
uploader [ ' uploader_id ' ] = try_get (
owner , lambda x : x [ ' navigationEndpoint ' ] [ ' browseEndpoint ' ] [ ' browseId ' ] , compat_str )
uploader [ ' uploader_url ' ] = urljoin (
' https://www.youtube.com/ ' ,
try_get ( owner , lambda x : x [ ' navigationEndpoint ' ] [ ' browseEndpoint ' ] [ ' canonicalBaseUrl ' ] , compat_str ) )
2021-02-02 15:52:27 -05:00
return { k : v for k , v in uploader . items ( ) if v is not None }
2020-11-19 14:22:59 -05:00
2021-03-05 08:59:14 -05:00
def _extract_from_tabs ( self , item_id , webpage , data , tabs ) :
2021-02-02 11:21:32 -05:00
playlist_id = title = description = channel_url = channel_name = channel_id = None
thumbnails_list = tags = [ ]
2020-11-19 14:22:59 -05:00
selected_tab = self . _extract_selected_tab ( tabs )
renderer = try_get (
data , lambda x : x [ ' metadata ' ] [ ' channelMetadataRenderer ' ] , dict )
if renderer :
2021-02-02 11:21:32 -05:00
channel_name = renderer . get ( ' title ' )
channel_url = renderer . get ( ' channelUrl ' )
channel_id = renderer . get ( ' externalId ' )
2021-04-03 18:05:17 -04:00
else :
2021-01-28 14:01:50 -05:00
renderer = try_get (
data , lambda x : x [ ' metadata ' ] [ ' playlistMetadataRenderer ' ] , dict )
2021-04-03 18:05:17 -04:00
2020-11-19 14:22:59 -05:00
if renderer :
title = renderer . get ( ' title ' )
2021-02-07 09:44:44 -05:00
description = renderer . get ( ' description ' , ' ' )
2021-02-02 11:21:32 -05:00
playlist_id = channel_id
tags = renderer . get ( ' keywords ' , ' ' ) . split ( )
thumbnails_list = (
try_get ( renderer , lambda x : x [ ' avatar ' ] [ ' thumbnails ' ] , list )
2021-02-08 08:50:19 -05:00
or try_get (
data ,
lambda x : x [ ' sidebar ' ] [ ' playlistSidebarRenderer ' ] [ ' items ' ] [ 0 ] [ ' playlistSidebarPrimaryInfoRenderer ' ] [ ' thumbnailRenderer ' ] [ ' playlistVideoThumbnailRenderer ' ] [ ' thumbnail ' ] [ ' thumbnails ' ] ,
list )
2021-02-02 11:21:32 -05:00
or [ ] )
thumbnails = [ ]
for t in thumbnails_list :
if not isinstance ( t , dict ) :
continue
thumbnail_url = url_or_none ( t . get ( ' url ' ) )
if not thumbnail_url :
continue
thumbnails . append ( {
' url ' : thumbnail_url ,
' width ' : int_or_none ( t . get ( ' width ' ) ) ,
' height ' : int_or_none ( t . get ( ' height ' ) ) ,
} )
2020-11-21 17:21:09 -05:00
if playlist_id is None :
2020-11-23 16:47:42 -05:00
playlist_id = item_id
if title is None :
2021-04-03 18:05:17 -04:00
title = (
try_get ( data , lambda x : x [ ' header ' ] [ ' hashtagHeaderRenderer ' ] [ ' hashtag ' ] [ ' simpleText ' ] )
or playlist_id )
2021-02-02 11:21:32 -05:00
title + = format_field ( selected_tab , ' title ' , ' - %s ' )
2021-05-18 10:00:21 -04:00
title + = format_field ( selected_tab , ' expandedText ' , ' - %s ' )
2021-02-02 11:21:32 -05:00
metadata = {
' playlist_id ' : playlist_id ,
' playlist_title ' : title ,
' playlist_description ' : description ,
' uploader ' : channel_name ,
' uploader_id ' : channel_id ,
' uploader_url ' : channel_url ,
' thumbnails ' : thumbnails ,
' tags ' : tags ,
}
if not channel_id :
metadata . update ( self . _extract_uploader ( data ) )
metadata . update ( {
' channel ' : metadata [ ' uploader ' ] ,
' channel_id ' : metadata [ ' uploader_id ' ] ,
' channel_url ' : metadata [ ' uploader_url ' ] } )
return self . playlist_result (
2021-03-05 08:59:14 -05:00
self . _entries (
selected_tab , playlist_id ,
self . _extract_identity_token ( webpage , item_id ) ,
2021-04-11 16:23:04 -04:00
self . _extract_account_syncid ( data ) ,
self . _extract_ytcfg ( item_id , webpage ) ) ,
2021-02-02 11:21:32 -05:00
* * metadata )
2016-07-05 12:30:44 -04:00
2021-04-15 07:22:59 -04:00
def _extract_mix_playlist ( self , playlist , playlist_id , data , webpage ) :
2021-03-07 00:24:54 -05:00
first_id = last_id = None
2021-04-15 07:22:59 -04:00
ytcfg = self . _extract_ytcfg ( playlist_id , webpage )
headers = self . _generate_api_headers (
ytcfg , account_syncid = self . _extract_account_syncid ( data ) ,
identity_token = self . _extract_identity_token ( webpage , item_id = playlist_id ) ,
visitor_data = try_get ( self . _extract_context ( ytcfg ) , lambda x : x [ ' client ' ] [ ' visitorData ' ] , compat_str ) )
2021-03-07 00:24:54 -05:00
for page_num in itertools . count ( 1 ) :
2021-03-04 13:05:26 -05:00
videos = list ( self . _playlist_entries ( playlist ) )
if not videos :
return
2021-03-07 00:24:54 -05:00
start = next ( ( i for i , v in enumerate ( videos ) if v [ ' id ' ] == last_id ) , - 1 ) + 1
if start > = len ( videos ) :
return
for video in videos [ start : ] :
if video [ ' id ' ] == first_id :
self . to_screen ( ' First video %s found again; Assuming end of Mix ' % first_id )
return
yield video
first_id = first_id or videos [ 0 ] [ ' id ' ]
last_id = videos [ - 1 ] [ ' id ' ]
2021-04-15 07:22:59 -04:00
watch_endpoint = try_get (
playlist , lambda x : x [ ' contents ' ] [ - 1 ] [ ' playlistPanelVideoRenderer ' ] [ ' navigationEndpoint ' ] [ ' watchEndpoint ' ] )
query = {
' playlistId ' : playlist_id ,
' videoId ' : watch_endpoint . get ( ' videoId ' ) or last_id ,
' index ' : watch_endpoint . get ( ' index ' ) or len ( videos ) ,
' params ' : watch_endpoint . get ( ' params ' ) or ' OAE % 3D '
}
response = self . _extract_response (
item_id = ' %s page %d ' % ( playlist_id , page_num ) ,
query = query ,
ep = ' next ' ,
headers = headers ,
check_get_keys = ' contents '
)
2021-03-04 13:05:26 -05:00
playlist = try_get (
2021-04-15 07:22:59 -04:00
response , lambda x : x [ ' contents ' ] [ ' twoColumnWatchNextResults ' ] [ ' playlist ' ] [ ' playlist ' ] , dict )
2021-03-04 13:05:26 -05:00
2021-04-15 07:22:59 -04:00
def _extract_from_playlist ( self , item_id , url , data , playlist , webpage ) :
2020-11-19 14:22:59 -05:00
title = playlist . get ( ' title ' ) or try_get (
data , lambda x : x [ ' titleText ' ] [ ' simpleText ' ] , compat_str )
playlist_id = playlist . get ( ' playlistId ' ) or item_id
2021-03-04 13:05:26 -05:00
# Delegating everything except mix playlists to regular tab-based playlist URL
2021-01-01 07:26:37 -05:00
playlist_url = urljoin ( url , try_get (
playlist , lambda x : x [ ' endpoint ' ] [ ' commandMetadata ' ] [ ' webCommandMetadata ' ] [ ' url ' ] ,
compat_str ) )
if playlist_url and playlist_url != url :
return self . url_result (
playlist_url , ie = YoutubeTabIE . ie_key ( ) , video_id = playlist_id ,
video_title = title )
2021-03-04 13:05:26 -05:00
2020-11-19 14:22:59 -05:00
return self . playlist_result (
2021-04-15 07:22:59 -04:00
self . _extract_mix_playlist ( playlist , playlist_id , data , webpage ) ,
2021-03-04 13:05:26 -05:00
playlist_id = playlist_id , playlist_title = title )
2013-06-23 13:58:33 -04:00
2021-04-16 18:39:08 -04:00
def _reload_with_unavailable_videos ( self , item_id , data , webpage ) :
"""
Get playlist with unavailable videos if the ' show unavailable videos ' button exists .
"""
sidebar_renderer = try_get (
2021-04-16 22:19:49 -04:00
data , lambda x : x [ ' sidebar ' ] [ ' playlistSidebarRenderer ' ] [ ' items ' ] , list )
if not sidebar_renderer :
return
browse_id = params = None
2021-04-16 18:39:08 -04:00
for item in sidebar_renderer :
if not isinstance ( item , dict ) :
continue
renderer = item . get ( ' playlistSidebarPrimaryInfoRenderer ' )
menu_renderer = try_get (
renderer , lambda x : x [ ' menu ' ] [ ' menuRenderer ' ] [ ' items ' ] , list ) or [ ]
for menu_item in menu_renderer :
if not isinstance ( menu_item , dict ) :
continue
nav_item_renderer = menu_item . get ( ' menuNavigationItemRenderer ' )
text = try_get (
nav_item_renderer , lambda x : x [ ' text ' ] [ ' simpleText ' ] , compat_str )
if not text or text . lower ( ) != ' show unavailable videos ' :
continue
browse_endpoint = try_get (
nav_item_renderer , lambda x : x [ ' navigationEndpoint ' ] [ ' browseEndpoint ' ] , dict ) or { }
browse_id = browse_endpoint . get ( ' browseId ' )
params = browse_endpoint . get ( ' params ' )
2021-04-16 22:19:49 -04:00
break
ytcfg = self . _extract_ytcfg ( item_id , webpage )
headers = self . _generate_api_headers (
ytcfg , account_syncid = self . _extract_account_syncid ( ytcfg ) ,
identity_token = self . _extract_identity_token ( webpage , item_id = item_id ) ,
visitor_data = try_get (
self . _extract_context ( ytcfg ) , lambda x : x [ ' client ' ] [ ' visitorData ' ] , compat_str ) )
query = {
' params ' : params or ' wgYCCAA= ' ,
' browseId ' : browse_id or ' VL %s ' % item_id
}
return self . _extract_response (
item_id = item_id , headers = headers , query = query ,
check_get_keys = ' contents ' , fatal = False ,
note = ' Downloading API JSON with unavailable videos ' )
2021-04-16 18:39:08 -04:00
2021-03-04 13:05:26 -05:00
def _extract_webpage ( self , url , item_id ) :
2021-05-17 08:23:08 -04:00
retries = self . get_param ( ' extractor_retries ' , 3 )
2021-02-28 18:48:37 -05:00
count = - 1
2021-03-02 16:23:19 -05:00
last_error = ' Incomplete yt initial data recieved '
2021-02-25 16:52:44 -05:00
while count < retries :
2021-02-28 18:48:37 -05:00
count + = 1
2021-02-25 16:52:44 -05:00
# Sometimes youtube returns a webpage with incomplete ytInitialData
2021-02-28 18:48:37 -05:00
# See: https://github.com/yt-dlp/yt-dlp/issues/116
if count :
2021-03-02 16:23:19 -05:00
self . report_warning ( ' %s . Retrying ... ' % last_error )
2021-02-28 19:09:50 -05:00
webpage = self . _download_webpage (
url , item_id ,
2021-03-04 13:05:26 -05:00
' Downloading webpage %s ' % ( ' (retry # %d ) ' % count if count else ' ' ) )
2021-02-25 16:52:44 -05:00
data = self . _extract_yt_initial_data ( item_id , webpage )
if data . get ( ' contents ' ) or data . get ( ' currentVideoEndpoint ' ) :
break
2021-05-17 09:00:50 -04:00
# Extract alerts here only when there is error
self . _extract_and_report_alerts ( data )
2021-03-02 16:23:19 -05:00
if count > = retries :
2021-04-16 06:01:10 -04:00
raise ExtractorError ( last_error )
2021-03-04 13:05:26 -05:00
return webpage , data
2021-05-15 10:38:47 -04:00
@staticmethod
def _smuggle_data ( entries , data ) :
for entry in entries :
if data :
entry [ ' url ' ] = smuggle_url ( entry [ ' url ' ] , data )
yield entry
2021-03-04 13:05:26 -05:00
def _real_extract ( self , url ) :
2021-05-15 10:38:47 -04:00
url , smuggled_data = unsmuggle_url ( url , { } )
if self . is_music_url ( url ) :
smuggled_data [ ' is_music_url ' ] = True
2021-05-18 10:01:06 -04:00
info_dict = self . __real_extract ( url , smuggled_data )
2021-05-15 10:38:47 -04:00
if info_dict . get ( ' entries ' ) :
info_dict [ ' entries ' ] = self . _smuggle_data ( info_dict [ ' entries ' ] , smuggled_data )
return info_dict
2021-05-18 10:01:06 -04:00
_url_re = re . compile ( r ' (?P<pre> %s )(?(channel_type)(?P<tab>/ \ w+))?(?P<post>.*)$ ' % _VALID_URL )
def __real_extract ( self , url , smuggled_data ) :
2021-03-04 13:05:26 -05:00
item_id = self . _match_id ( url )
url = compat_urlparse . urlunparse (
compat_urlparse . urlparse ( url ) . _replace ( netloc = ' www.youtube.com ' ) )
2021-05-17 08:23:08 -04:00
compat_opts = self . get_param ( ' compat_opts ' , [ ] )
2021-03-04 13:05:26 -05:00
2021-05-18 10:01:06 -04:00
def get_mobj ( url ) :
mobj = self . _url_re . match ( url ) . groupdict ( )
2021-05-18 14:20:59 -04:00
mobj . update ( ( k , ' ' ) for k , v in mobj . items ( ) if v is None )
2021-05-18 10:01:06 -04:00
return mobj
mobj = get_mobj ( url )
# Youtube returns incomplete data if tabname is not lower case
pre , tab , post , is_channel = mobj [ ' pre ' ] , mobj [ ' tab ' ] . lower ( ) , mobj [ ' post ' ] , not mobj [ ' not_channel ' ]
if is_channel :
if smuggled_data . get ( ' is_music_url ' ) :
if item_id [ : 2 ] == ' VL ' :
# Youtube music VL channels have an equivalent playlist
item_id = item_id [ 2 : ]
pre , tab , post , is_channel = ' https://www.youtube.com/playlist?list= %s ' % item_id , ' ' , ' ' , False
2021-05-17 09:07:58 -04:00
elif item_id [ : 2 ] == ' MP ' :
# Youtube music albums (/channel/MP...) have a OLAK playlist that can be extracted from the webpage
item_id = self . _search_regex (
r ' \\ x22audioPlaylistId \\ x22: \\ x22([0-9A-Za-z_-]+) \\ x22 ' ,
self . _download_webpage ( ' https://music.youtube.com/channel/ %s ' % item_id , item_id ) ,
' playlist id ' )
pre , tab , post , is_channel = ' https://www.youtube.com/playlist?list= %s ' % item_id , ' ' , ' ' , False
2021-05-18 10:01:06 -04:00
elif mobj [ ' channel_type ' ] == ' browse ' :
# Youtube music /browse/ should be changed to /channel/
pre = ' https://www.youtube.com/channel/ %s ' % item_id
if is_channel and not tab and ' no-youtube-channel-redirect ' not in compat_opts :
# Home URLs should redirect to /videos/
2021-04-16 06:01:10 -04:00
self . report_warning (
2021-03-04 13:05:26 -05:00
' A channel/user page was given. All the channel \' s videos will be downloaded. '
' To download only the videos in the home page, add a " /featured " to the URL ' )
2021-05-18 10:01:06 -04:00
tab = ' /videos '
url = ' ' . join ( ( pre , tab , post ) )
mobj = get_mobj ( url )
2021-03-04 13:05:26 -05:00
# Handle both video/playlist URLs
2021-04-16 23:02:33 -04:00
qs = parse_qs ( url )
2021-03-04 13:05:26 -05:00
video_id = qs . get ( ' v ' , [ None ] ) [ 0 ]
playlist_id = qs . get ( ' list ' , [ None ] ) [ 0 ]
2021-05-18 10:01:06 -04:00
if not video_id and mobj [ ' not_channel ' ] . startswith ( ' watch ' ) :
2021-03-04 13:05:26 -05:00
if not playlist_id :
2021-05-18 10:01:06 -04:00
# If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
2021-03-04 13:05:26 -05:00
raise ExtractorError ( ' Unable to recognize tab page ' )
2021-05-18 10:01:06 -04:00
# Common mistake: https://www.youtube.com/watch?list=playlist_id
2021-04-16 06:01:10 -04:00
self . report_warning ( ' A video URL was given without video ID. Trying to download playlist %s ' % playlist_id )
2021-03-04 13:05:26 -05:00
url = ' https://www.youtube.com/playlist?list= %s ' % playlist_id
2021-05-18 09:02:46 -04:00
mobj = get_mobj ( url )
2021-03-04 13:05:26 -05:00
if video_id and playlist_id :
2021-05-17 08:23:08 -04:00
if self . get_param ( ' noplaylist ' ) :
2021-03-04 13:05:26 -05:00
self . to_screen ( ' Downloading just video %s because of --no-playlist ' % video_id )
return self . url_result ( video_id , ie = YoutubeIE . ie_key ( ) , video_id = video_id )
self . to_screen ( ' Downloading playlist %s ; add --no-playlist to just download video %s ' % ( playlist_id , video_id ) )
webpage , data = self . _extract_webpage ( url , item_id )
2021-02-25 16:52:44 -05:00
2021-05-18 09:02:46 -04:00
tabs = try_get (
data , lambda x : x [ ' contents ' ] [ ' twoColumnBrowseResultsRenderer ' ] [ ' tabs ' ] , list )
if tabs :
selected_tab = self . _extract_selected_tab ( tabs )
tab_name = selected_tab . get ( ' title ' , ' ' )
2021-05-21 10:34:30 -04:00
if ' no-youtube-channel-redirect ' not in compat_opts :
if mobj [ ' tab ' ] == ' /live ' :
# Live tab should have redirected to the video
raise ExtractorError ( ' The channel is not currently live ' , expected = True )
if mobj [ ' tab ' ] == ' /videos ' and tab_name . lower ( ) != mobj [ ' tab ' ] [ 1 : ] :
if not mobj [ ' not_channel ' ] and item_id [ : 2 ] == ' UC ' :
# Topic channels don't have /videos. Use the equivalent playlist instead
self . report_warning ( ' The URL does not have a %s tab. Trying to redirect to playlist UU %s instead ' % ( mobj [ ' tab ' ] [ 1 : ] , item_id [ 2 : ] ) )
pl_id = ' UU %s ' % item_id [ 2 : ]
pl_url = ' https://www.youtube.com/playlist?list= %s %s ' % ( pl_id , mobj [ ' post ' ] )
try :
pl_webpage , pl_data = self . _extract_webpage ( pl_url , pl_id )
for alert_type , alert_message in self . _extract_alerts ( pl_data ) :
if alert_type == ' error ' :
raise ExtractorError ( ' Youtube said: %s ' % alert_message )
item_id , url , webpage , data = pl_id , pl_url , pl_webpage , pl_data
except ExtractorError :
self . report_warning ( ' The playlist gave error. Falling back to channel URL ' )
else :
self . report_warning ( ' The URL does not have a %s tab. %s is being downloaded instead ' % ( mobj [ ' tab ' ] [ 1 : ] , tab_name ) )
2021-05-18 09:02:46 -04:00
self . write_debug ( ' Final URL: %s ' % url )
2021-04-16 18:39:08 -04:00
# YouTube sometimes provides a button to reload playlist with unavailable videos.
2021-05-11 04:00:48 -04:00
if ' no-youtube-unavailable-videos ' not in compat_opts :
data = self . _reload_with_unavailable_videos ( item_id , data , webpage ) or data
2021-05-17 09:00:50 -04:00
self . _extract_and_report_alerts ( data )
2021-04-16 18:39:08 -04:00
2020-11-19 14:22:59 -05:00
tabs = try_get (
data , lambda x : x [ ' contents ' ] [ ' twoColumnBrowseResultsRenderer ' ] [ ' tabs ' ] , list )
if tabs :
2021-03-05 08:59:14 -05:00
return self . _extract_from_tabs ( item_id , webpage , data , tabs )
2021-03-04 13:05:26 -05:00
2020-11-19 14:22:59 -05:00
playlist = try_get (
data , lambda x : x [ ' contents ' ] [ ' twoColumnWatchNextResults ' ] [ ' playlist ' ] [ ' playlist ' ] , dict )
if playlist :
2021-04-15 07:22:59 -04:00
return self . _extract_from_playlist ( item_id , url , data , playlist , webpage )
2021-03-04 13:05:26 -05:00
2020-11-21 09:50:42 -05:00
video_id = try_get (
data , lambda x : x [ ' currentVideoEndpoint ' ] [ ' watchEndpoint ' ] [ ' videoId ' ] ,
compat_str ) or video_id
2020-11-19 14:22:59 -05:00
if video_id :
2021-05-21 10:34:30 -04:00
if mobj [ ' tab ' ] != ' /live ' : # live tab is expected to redirect to video
self . report_warning ( ' Unable to recognize playlist. Downloading just video %s ' % video_id )
2020-11-19 14:22:59 -05:00
return self . url_result ( video_id , ie = YoutubeIE . ie_key ( ) , video_id = video_id )
2021-03-04 13:05:26 -05:00
2020-11-19 14:22:59 -05:00
raise ExtractorError ( ' Unable to recognize tab page ' )
2013-06-23 13:58:33 -04:00
2020-11-19 14:22:59 -05:00
class YoutubePlaylistIE ( InfoExtractor ) :
IE_DESC = ' YouTube.com playlists '
_VALID_URL = r ''' (?x)(?:
( ? : https ? : / / ) ?
( ? : \w + \. ) ?
( ? :
( ? :
youtube ( ? : kids ) ? \. com |
2021-01-01 07:26:37 -05:00
invidio \. us
2020-11-19 14:22:59 -05:00
)
/ . * ? \? . * ? \blist =
) ?
( ? P < id > % ( playlist_id ) s )
) ''' % { ' playlist_id ' : YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
IE_NAME = ' youtube:playlist '
2014-09-24 04:25:47 -04:00
_TESTS = [ {
2020-11-19 14:22:59 -05:00
' note ' : ' issue #673 ' ,
' url ' : ' PLBB231211A4F62143 ' ,
2014-09-24 04:25:47 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' title ' : ' [OLD]Team Fortress 2 (Class-based LP) ' ,
' id ' : ' PLBB231211A4F62143 ' ,
' uploader ' : ' Wickydoo ' ,
' uploader_id ' : ' UCKSpbfbl5kRQpTdL7kMc-1Q ' ,
} ,
' playlist_mincount ' : 29 ,
} , {
' url ' : ' PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl ' ,
' info_dict ' : {
' title ' : ' YDL_safe_search ' ,
' id ' : ' PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl ' ,
} ,
' playlist_count ' : 2 ,
' skip ' : ' This playlist is private ' ,
2016-07-08 21:37:02 -04:00
} , {
2020-11-19 14:22:59 -05:00
' note ' : ' embedded ' ,
' url ' : ' https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu ' ,
' playlist_count ' : 4 ,
2016-07-08 21:37:02 -04:00
' info_dict ' : {
2020-11-19 14:22:59 -05:00
' title ' : ' JODA15 ' ,
' id ' : ' PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu ' ,
' uploader ' : ' milan ' ,
' uploader_id ' : ' UCEI1-PVPcYXjB73Hfelbmaw ' ,
2016-07-08 21:37:02 -04:00
}
2014-09-24 04:25:47 -04:00
} , {
2020-11-19 14:22:59 -05:00
' url ' : ' http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl ' ,
' playlist_mincount ' : 982 ,
' info_dict ' : {
' title ' : ' 2018 Chinese New Singles (11/6 updated) ' ,
' id ' : ' PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl ' ,
' uploader ' : ' LBK ' ,
' uploader_id ' : ' UC21nz3_MesPLqtDqwdvnoxA ' ,
}
2016-06-10 04:37:12 -04:00
} , {
2021-01-01 07:26:37 -05:00
' url ' : ' TLGGrESM50VT6acwMjAyMjAxNw ' ,
' only_matching ' : True ,
} , {
# music album playlist
' url ' : ' OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM ' ,
' only_matching ' : True ,
} ]
@classmethod
def suitable ( cls , url ) :
2021-04-16 23:02:33 -04:00
if YoutubeTabIE . suitable ( url ) :
return False
2021-04-22 05:02:54 -04:00
# Hack for lazy extractors until more generic solution is implemented
# (see #28780)
from . youtube import parse_qs
2021-04-16 23:02:33 -04:00
qs = parse_qs ( url )
if qs . get ( ' v ' , [ None ] ) [ 0 ] :
return False
return super ( YoutubePlaylistIE , cls ) . suitable ( url )
2021-01-01 07:26:37 -05:00
def _real_extract ( self , url ) :
playlist_id = self . _match_id ( url )
2021-05-28 14:19:26 -04:00
is_music_url = YoutubeBaseInfoExtractor . is_music_url ( url )
2021-05-15 10:38:47 -04:00
url = update_url_query (
' https://www.youtube.com/playlist ' ,
parse_qs ( url ) or { ' list ' : playlist_id } )
if is_music_url :
url = smuggle_url ( url , { ' is_music_url ' : True } )
return self . url_result ( url , ie = YoutubeTabIE . ie_key ( ) , video_id = playlist_id )
2021-01-01 07:26:37 -05:00
class YoutubeYtBeIE ( InfoExtractor ) :
2021-01-07 01:41:05 -05:00
IE_DESC = ' youtu.be '
2021-01-01 07:26:37 -05:00
_VALID_URL = r ' https?://youtu \ .be/(?P<id>[0-9A-Za-z_-] {11} )/*?.*? \ blist=(?P<playlist_id> %(playlist_id)s ) ' % { ' playlist_id ' : YoutubeBaseInfoExtractor . _PLAYLIST_ID_RE }
_TESTS = [ {
2020-11-19 14:22:59 -05:00
' url ' : ' https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5 ' ,
' info_dict ' : {
' id ' : ' yeWKywCrFtk ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Small Scale Baler and Braiding Rugs ' ,
' uploader ' : ' Backus-Page House Museum ' ,
' uploader_id ' : ' backuspagemuseum ' ,
' uploader_url ' : r ' re:https?://(?:www \ .)?youtube \ .com/user/backuspagemuseum ' ,
' upload_date ' : ' 20161008 ' ,
' description ' : ' md5:800c0c78d5eb128500bffd4f0b4f2e8a ' ,
' categories ' : [ ' Nonprofits & Activism ' ] ,
' tags ' : list ,
' like_count ' : int ,
' dislike_count ' : int ,
} ,
' params ' : {
' noplaylist ' : True ,
' skip_download ' : True ,
} ,
2020-09-11 23:08:57 -04:00
} , {
2020-11-19 14:22:59 -05:00
' url ' : ' https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21 ' ,
2020-09-11 23:08:57 -04:00
' only_matching ' : True ,
2014-09-24 04:25:47 -04:00
} ]
2020-11-19 14:22:59 -05:00
def _real_extract ( self , url ) :
2021-01-01 07:26:37 -05:00
mobj = re . match ( self . _VALID_URL , url )
video_id = mobj . group ( ' id ' )
playlist_id = mobj . group ( ' playlist_id ' )
2020-11-19 14:22:59 -05:00
return self . url_result (
2021-01-01 07:26:37 -05:00
update_url_query ( ' https://www.youtube.com/watch ' , {
' v ' : video_id ,
' list ' : playlist_id ,
' feature ' : ' youtu.be ' ,
} ) , ie = YoutubeTabIE . ie_key ( ) , video_id = playlist_id )
2020-11-19 14:22:59 -05:00
class YoutubeYtUserIE ( InfoExtractor ) :
2021-01-07 01:41:05 -05:00
IE_DESC = ' YouTube.com user videos, URL or " ytuser " keyword '
2020-11-19 14:22:59 -05:00
_VALID_URL = r ' ytuser:(?P<id>.+) '
_TESTS = [ {
' url ' : ' ytuser:phihag ' ,
' only_matching ' : True ,
} ]
def _real_extract ( self , url ) :
user_id = self . _match_id ( url )
return self . url_result (
' https://www.youtube.com/user/ %s ' % user_id ,
ie = YoutubeTabIE . ie_key ( ) , video_id = user_id )
2016-07-08 21:37:02 -04:00
2013-06-23 14:28:15 -04:00
2020-11-23 16:59:10 -05:00
class YoutubeFavouritesIE ( YoutubeBaseInfoExtractor ) :
2020-11-23 16:47:42 -05:00
IE_NAME = ' youtube:favorites '
IE_DESC = ' YouTube.com liked videos, " :ytfav " for short (requires authentication) '
_VALID_URL = r ' :ytfav(?:ou?rite)?s? '
_LOGIN_REQUIRED = True
_TESTS = [ {
' url ' : ' :ytfav ' ,
' only_matching ' : True ,
} , {
' url ' : ' :ytfavorites ' ,
' only_matching ' : True ,
} ]
def _real_extract ( self , url ) :
return self . url_result (
' https://www.youtube.com/playlist?list=LL ' ,
ie = YoutubeTabIE . ie_key ( ) )
2021-04-15 07:22:59 -04:00
class YoutubeSearchIE ( SearchInfoExtractor , YoutubeTabIE ) :
2021-02-09 11:07:59 -05:00
IE_DESC = ' YouTube.com searches, " ytsearch " keyword '
2015-04-21 13:30:31 -04:00
# there doesn't appear to be a real limit, for example if you search for
# 'python' you get more than 8.000.000 results
_MAX_RESULTS = float ( ' inf ' )
2014-09-13 01:51:06 -04:00
IE_NAME = ' youtube:search '
2013-06-23 14:28:15 -04:00
_SEARCH_KEY = ' ytsearch '
2020-10-24 00:57:14 -04:00
_SEARCH_PARAMS = None
2015-04-22 10:28:33 -04:00
_TESTS = [ ]
2013-06-23 14:28:15 -04:00
2020-10-24 00:57:14 -04:00
def _entries ( self , query , n ) :
2021-03-03 05:32:40 -05:00
data = { ' query ' : query }
2020-10-24 00:57:14 -04:00
if self . _SEARCH_PARAMS :
data [ ' params ' ] = self . _SEARCH_PARAMS
total = 0
for page_num in itertools . count ( 1 ) :
2021-04-15 07:22:59 -04:00
search = self . _extract_response (
item_id = ' query " %s " page %s ' % ( query , page_num ) , ep = ' search ' , query = data ,
check_get_keys = ( ' contents ' , ' onResponseReceivedCommands ' )
)
2020-10-24 00:57:14 -04:00
if not search :
2015-04-21 13:30:31 -04:00
break
2020-10-24 00:57:14 -04:00
slr_contents = try_get (
search ,
( lambda x : x [ ' contents ' ] [ ' twoColumnSearchResultsRenderer ' ] [ ' primaryContents ' ] [ ' sectionListRenderer ' ] [ ' contents ' ] ,
lambda x : x [ ' onResponseReceivedCommands ' ] [ 0 ] [ ' appendContinuationItemsAction ' ] [ ' continuationItems ' ] ) ,
list )
if not slr_contents :
2017-02-02 12:28:24 -05:00
break
2020-11-14 15:03:40 -05:00
# Youtube sometimes adds promoted content to searches,
# changing the index location of videos and token.
# So we search through all entries till we find them.
2021-01-16 07:40:15 -05:00
continuation_token = None
for slr_content in slr_contents :
2021-02-11 06:40:38 -05:00
if continuation_token is None :
continuation_token = try_get (
slr_content ,
lambda x : x [ ' continuationItemRenderer ' ] [ ' continuationEndpoint ' ] [ ' continuationCommand ' ] [ ' token ' ] ,
compat_str )
2021-01-16 07:40:15 -05:00
isr_contents = try_get (
slr_content ,
lambda x : x [ ' itemSectionRenderer ' ] [ ' contents ' ] ,
list )
2020-11-14 15:34:59 -05:00
if not isr_contents :
2021-01-16 07:40:15 -05:00
continue
for content in isr_contents :
if not isinstance ( content , dict ) :
continue
video = content . get ( ' videoRenderer ' )
if not isinstance ( video , dict ) :
continue
video_id = video . get ( ' videoId ' )
if not video_id :
continue
yield self . _extract_video ( video )
total + = 1
if total == n :
return
2020-11-14 15:03:40 -05:00
if not continuation_token :
2020-10-24 00:57:14 -04:00
break
2020-11-14 15:03:40 -05:00
data [ ' continuation ' ] = continuation_token
2013-06-23 14:28:15 -04:00
2020-10-24 00:57:14 -04:00
def _get_n_results ( self , query , n ) :
""" Get a specified number of results for a query """
return self . playlist_result ( self . _entries ( query , n ) , query )
2013-07-01 11:59:28 -04:00
2014-03-03 21:32:28 -05:00
2013-11-02 22:40:48 -04:00
class YoutubeSearchDateIE ( YoutubeSearchIE ) :
2013-12-03 07:55:25 -05:00
IE_NAME = YoutubeSearchIE . IE_NAME + ' :date '
2013-11-02 22:40:48 -04:00
_SEARCH_KEY = ' ytsearchdate '
2021-01-07 01:41:05 -05:00
IE_DESC = ' YouTube.com searches, newest videos first, " ytsearchdate " keyword '
2020-10-24 00:57:14 -04:00
_SEARCH_PARAMS = ' CAI % 3D '
2013-07-01 11:59:28 -04:00
2014-03-03 21:32:28 -05:00
2020-11-22 08:09:09 -05:00
class YoutubeSearchURLIE ( YoutubeSearchIE ) :
2021-02-09 11:07:59 -05:00
IE_DESC = ' YouTube.com search URLs '
2020-11-22 08:09:09 -05:00
IE_NAME = YoutubeSearchIE . IE_NAME + ' _url '
_VALID_URL = r ' https?://(?:www \ .)?youtube \ .com/results \ ?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$) '
2020-11-21 17:46:05 -05:00
# _MAX_RESULTS = 100
2020-11-21 17:21:09 -05:00
_TESTS = [ {
' url ' : ' https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video ' ,
' playlist_mincount ' : 5 ,
' info_dict ' : {
' title ' : ' youtube-dl test video ' ,
}
} , {
' url ' : ' https://www.youtube.com/results?q=test&sp=EgQIBBgB ' ,
' only_matching ' : True ,
} ]
2020-11-22 08:09:09 -05:00
@classmethod
def _make_valid_url ( cls ) :
return cls . _VALID_URL
2020-11-21 17:21:09 -05:00
def _real_extract ( self , url ) :
2020-11-22 08:09:09 -05:00
qs = compat_parse_qs ( compat_urllib_parse_urlparse ( url ) . query )
query = ( qs . get ( ' search_query ' ) or qs . get ( ' q ' ) ) [ 0 ]
self . _SEARCH_PARAMS = qs . get ( ' sp ' , ( ' ' , ) ) [ 0 ]
return self . _get_n_results ( query , self . _MAX_RESULTS )
2020-11-21 17:21:09 -05:00
class YoutubeFeedsInfoExtractor ( YoutubeTabIE ) :
2013-07-20 13:33:40 -04:00
"""
2015-05-15 11:06:59 -04:00
Base class for feed extractors
2020-11-23 16:59:10 -05:00
Subclasses must define the _FEED_NAME property .
2013-07-20 13:33:40 -04:00
"""
2013-07-24 14:40:12 -04:00
_LOGIN_REQUIRED = True
2020-11-21 17:46:05 -05:00
_TESTS = [ ]
2013-07-20 13:33:40 -04:00
@property
def IE_NAME ( self ) :
2014-09-13 01:51:06 -04:00
return ' youtube: %s ' % self . _FEED_NAME
2013-07-07 07:58:23 -04:00
2018-04-21 19:07:32 -04:00
def _real_extract ( self , url ) :
2020-11-23 16:59:10 -05:00
return self . url_result (
' https://www.youtube.com/feed/ %s ' % self . _FEED_NAME ,
ie = YoutubeTabIE . ie_key ( ) )
2015-05-15 11:06:59 -04:00
2020-11-21 17:46:05 -05:00
class YoutubeWatchLaterIE ( InfoExtractor ) :
IE_NAME = ' youtube:watchlater '
2020-11-23 16:47:42 -05:00
IE_DESC = ' Youtube watch later list, " :ytwatchlater " for short (requires authentication) '
2020-11-23 16:59:10 -05:00
_VALID_URL = r ' :ytwatchlater '
2016-02-18 10:50:21 -05:00
_TESTS = [ {
2020-11-19 14:22:59 -05:00
' url ' : ' :ytwatchlater ' ,
2016-02-18 10:50:21 -05:00
' only_matching ' : True ,
} ]
2015-05-15 11:06:59 -04:00
def _real_extract ( self , url ) :
2020-11-21 17:46:05 -05:00
return self . url_result (
' https://www.youtube.com/playlist?list=WL ' , ie = YoutubeTabIE . ie_key ( ) )
2020-11-21 17:21:09 -05:00
2015-05-15 11:06:59 -04:00
class YoutubeRecommendedIE ( YoutubeFeedsInfoExtractor ) :
IE_DESC = ' YouTube.com recommended videos, " :ytrec " for short (requires authentication) '
2020-11-23 16:59:10 -05:00
_VALID_URL = r ' https?://(?:www \ .)?youtube \ .com/?(?:[?#]|$)|:ytrec(?:ommended)? '
2015-05-15 11:06:59 -04:00
_FEED_NAME = ' recommended '
2021-06-12 11:15:08 -04:00
_LOGIN_REQUIRED = False
2020-11-23 16:59:10 -05:00
_TESTS = [ {
' url ' : ' :ytrec ' ,
' only_matching ' : True ,
} , {
' url ' : ' :ytrecommended ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://youtube.com ' ,
' only_matching ' : True ,
} ]
2014-08-31 17:44:43 -04:00
2015-05-15 11:06:59 -04:00
class YoutubeSubscriptionsIE ( YoutubeFeedsInfoExtractor ) :
2020-11-23 16:47:42 -05:00
IE_DESC = ' YouTube.com subscriptions feed, " :ytsubs " for short (requires authentication) '
2020-11-23 16:59:10 -05:00
_VALID_URL = r ' :ytsub(?:scription)?s? '
2015-05-15 11:06:59 -04:00
_FEED_NAME = ' subscriptions '
2020-11-23 16:59:10 -05:00
_TESTS = [ {
' url ' : ' :ytsubs ' ,
' only_matching ' : True ,
} , {
' url ' : ' :ytsubscriptions ' ,
' only_matching ' : True ,
} ]
2014-08-31 17:44:43 -04:00
2015-05-15 11:06:59 -04:00
class YoutubeHistoryIE ( YoutubeFeedsInfoExtractor ) :
2021-03-03 05:32:40 -05:00
IE_DESC = ' Youtube watch history, " :ythis " for short (requires authentication) '
_VALID_URL = r ' :ythis(?:tory)? '
2015-05-15 11:06:59 -04:00
_FEED_NAME = ' history '
2020-11-23 16:59:10 -05:00
_TESTS = [ {
' url ' : ' :ythistory ' ,
' only_matching ' : True ,
} ]
2014-08-31 17:44:43 -04:00
2013-10-07 06:21:24 -04:00
class YoutubeTruncatedURLIE ( InfoExtractor ) :
IE_NAME = ' youtube:truncated_url '
IE_DESC = False # Do not list
2014-01-23 10:14:54 -05:00
_VALID_URL = r ''' (?x)
2015-01-24 05:42:20 -05:00
( ? : https ? : / / ) ?
( ? : \w + \. ) ? [ yY ] [ oO ] [ uU ] [ tT ] [ uU ] [ bB ] [ eE ] ( ? : - nocookie ) ? \. com /
( ? : watch \? ( ? :
2014-07-01 09:48:18 -04:00
feature = [ a - z_ ] + |
2015-01-24 05:42:20 -05:00
annotation_id = annotation_ [ ^ & ] + |
x - yt - cl = [ 0 - 9 ] + |
2015-01-29 21:45:29 -05:00
hl = [ ^ & ] * |
2015-09-13 18:26:12 -04:00
t = [ 0 - 9 ] +
2015-01-24 05:42:20 -05:00
) ?
|
attribution_link \? a = [ ^ & ] +
)
$
2014-01-23 10:14:54 -05:00
'''
2013-10-07 06:21:24 -04:00
2014-07-01 09:48:18 -04:00
_TESTS = [ {
2016-09-17 10:48:20 -04:00
' url ' : ' https://www.youtube.com/watch?annotation_id=annotation_3951667041 ' ,
2014-07-01 09:48:18 -04:00
' only_matching ' : True ,
2014-07-01 09:49:34 -04:00
} , {
2016-09-17 10:48:20 -04:00
' url ' : ' https://www.youtube.com/watch? ' ,
2014-07-01 09:49:34 -04:00
' only_matching ' : True ,
2015-01-24 05:42:20 -05:00
} , {
' url ' : ' https://www.youtube.com/watch?x-yt-cl=84503534 ' ,
' only_matching ' : True ,
} , {
' url ' : ' https://www.youtube.com/watch?feature=foo ' ,
' only_matching ' : True ,
2015-01-29 21:45:29 -05:00
} , {
' url ' : ' https://www.youtube.com/watch?hl=en-GB ' ,
' only_matching ' : True ,
2015-09-13 18:26:12 -04:00
} , {
' url ' : ' https://www.youtube.com/watch?t=2372 ' ,
' only_matching ' : True ,
2014-07-01 09:48:18 -04:00
} ]
2013-10-07 06:21:24 -04:00
def _real_extract ( self , url ) :
raise ExtractorError (
2014-09-13 01:51:06 -04:00
' Did you forget to quote the URL? Remember that & is a meta '
' character in most shells, so you want to put the URL in quotes, '
2020-09-02 16:37:35 -04:00
' like youtube-dl '
2016-09-17 10:48:20 -04:00
' " https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc " '
2020-09-02 16:37:35 -04:00
' or simply youtube-dl BaW_jenozKc . ' ,
2013-10-07 06:21:24 -04:00
expected = True )
2015-01-01 17:44:39 -05:00
class YoutubeTruncatedIDIE ( InfoExtractor ) :
IE_NAME = ' youtube:truncated_id '
IE_DESC = False # Do not list
2015-01-24 05:42:20 -05:00
_VALID_URL = r ' https?://(?:www \ .)?youtube \ .com/watch \ ?v=(?P<id>[0-9A-Za-z_-] { 1,10})$ '
2015-01-01 17:44:39 -05:00
_TESTS = [ {
' url ' : ' https://www.youtube.com/watch?v=N_708QY7Ob ' ,
' only_matching ' : True ,
} ]
def _real_extract ( self , url ) :
video_id = self . _match_id ( url )
raise ExtractorError (
' Incomplete YouTube ID %s . URL %s looks truncated. ' % ( video_id , url ) ,
expected = True )