[core] Workaround erroneous urllib Windows proxy parsing (#7092)

Convert proxies extracted from windows registry to http for older Python versions.
See: https://github.com/python/cpython/issues/86793

Authored by: coletdjnz
This commit is contained in:
coletdjnz 2023-05-27 19:17:27 +12:00 committed by GitHub
parent b87e01c123
commit 3f66b6fe50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 49 additions and 2 deletions

View File

@ -74,7 +74,7 @@ offlinetest: codetest
$(PYTHON) -m pytest -k "not download" $(PYTHON) -m pytest -k "not download"
# XXX: This is hard to maintain # XXX: This is hard to maintain
CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies
yt-dlp: yt_dlp/*.py yt_dlp/*/*.py yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
mkdir -p zip mkdir -p zip
for d in $(CODE_FOLDERS) ; do \ for d in $(CODE_FOLDERS) ; do \

View File

@ -21,9 +21,9 @@ import time
import tokenize import tokenize
import traceback import traceback
import unicodedata import unicodedata
import urllib.request
from .cache import Cache from .cache import Cache
from .compat import urllib # isort: split
from .compat import compat_os_name, compat_shlex_quote from .compat import compat_os_name, compat_shlex_quote
from .cookies import load_cookies from .cookies import load_cookies
from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name

View File

@ -0,0 +1,7 @@
# flake8: noqa: F405
from urllib import * # noqa: F403
from ..compat_utils import passthrough_module
passthrough_module(__name__, 'urllib')
del passthrough_module

View File

@ -0,0 +1,40 @@
# flake8: noqa: F405
from urllib.request import * # noqa: F403
from ..compat_utils import passthrough_module
passthrough_module(__name__, 'urllib.request')
del passthrough_module
from .. import compat_os_name
if compat_os_name == 'nt':
# On older python versions, proxies are extracted from Windows registry erroneously. [1]
# If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
# It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
# it to http on these older python versions to avoid issues
# This also applies for ftp proxy type, as ftp:// proxy scheme is not supported.
# 1: https://github.com/python/cpython/issues/86793
# 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698
import sys
from urllib.request import getproxies_environment, getproxies_registry
def getproxies_registry_patched():
proxies = getproxies_registry()
if (
sys.version_info >= (3, 10, 5) # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
or (3, 9, 13) <= sys.version_info < (3, 10) # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
):
return proxies
for scheme in ('https', 'ftp'):
if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
proxies[scheme] = 'http' + proxies[scheme][len(scheme):]
return proxies
def getproxies():
return getproxies_environment() or getproxies_registry_patched()
del compat_os_name