From 9e491463521c65ca4d1d44a757e0a115f62834f5 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 19 May 2022 19:45:21 +0530 Subject: [PATCH] Add option `--alias` --- Makefile | 2 +- README.md | 16 ++++++ yt_dlp/__init__.py | 40 ++++++++------ yt_dlp/options.py | 135 +++++++++++++++++++++++++++++++++++++-------- yt_dlp/utils.py | 22 +++++--- 5 files changed, 167 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 8ce2e94c5..0ff5626ad 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ yt-dlp: yt_dlp/*.py yt_dlp/*/*.py chmod a+x yt-dlp README.md: yt_dlp/*.py yt_dlp/*/*.py - COLUMNS=80 $(PYTHON) yt_dlp/__main__.py --help | $(PYTHON) devscripts/make_readme.py + COLUMNS=80 $(PYTHON) yt_dlp/__main__.py --ignore-config --help | $(PYTHON) devscripts/make_readme.py CONTRIBUTING.md: README.md $(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md diff --git a/README.md b/README.md index 256388f30..d1e44365a 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,22 @@ You can also fork the project on github and run your fork's [build workflow](.gi configurations by reverting some of the changes made in yt-dlp. See "Differences in default behavior" for details + --alias ALIASES OPTIONS Create aliases for an option string. Unless + an alias starts with a dash "-", it is + prefixed with "--". Arguments are parsed + according to the Python string formatting + mini-language. Eg: --alias get-audio,-X + "-S=aext:{0},abr -x --audio-format {0}" + creates options "--get-audio" and "-X" that + takes an argument (ARG0) and expands to + "-S=aext:ARG0,abr -x --audio-format ARG0". + All defined aliases are listed in the + --help output. Alias options can trigger + more aliases; so be carefull to avoid + defining recursive options. As a safety + measure, each alias may be triggered a + maximum of 100 times. This option can be + used multiple times ## Network Options: --proxy URL Use the specified HTTP/HTTPS/SOCKS proxy. diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 8f890b34a..81b1716df 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -4,6 +4,7 @@ f'You are using an unsupported version of Python. Only Python versions 3.6 and a __license__ = 'Public Domain' import itertools +import optparse import os import re import sys @@ -45,11 +46,18 @@ from .utils import ( setproctitle, std_headers, traverse_obj, + variadic, write_string, ) from .YoutubeDL import YoutubeDL +def _exit(status=0, *args): + for msg in args: + sys.stderr.write(msg) + raise SystemExit(status) + + def get_urls(urls, batchfile, verbose): # Batch file verification batch_urls = [] @@ -66,7 +74,7 @@ def get_urls(urls, batchfile, verbose): if verbose: write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') except OSError: - sys.exit('ERROR: batch file %s could not be read' % batchfile) + _exit(f'ERROR: batch file {batchfile} could not be read') _enc = preferredencoding() return [ url.strip().decode(_enc, 'ignore') if isinstance(url, bytes) else url.strip() @@ -810,10 +818,10 @@ def _real_main(argv=None): if opts.dump_user_agent: ua = traverse_obj(opts.headers, 'User-Agent', casesense=False, default=std_headers['User-Agent']) write_string(f'{ua}\n', out=sys.stdout) - sys.exit(0) + return if print_extractor_information(opts, all_urls): - sys.exit(0) + return with YoutubeDL(ydl_opts) as ydl: actual_use = all_urls or opts.load_info_filename @@ -827,13 +835,13 @@ def _real_main(argv=None): # If updater returns True, exit. Required for windows if run_update(ydl): if actual_use: - sys.exit('ERROR: The program must exit for the update to complete') - sys.exit() + return 100, 'ERROR: The program must exit for the update to complete' + return # Maybe do nothing if not actual_use: if opts.update_self or opts.rm_cachedir: - sys.exit() + return ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) parser.error( @@ -842,30 +850,30 @@ def _real_main(argv=None): try: if opts.load_info_filename is not None: - retcode = ydl.download_with_info_file(expand_path(opts.load_info_filename)) + return ydl.download_with_info_file(expand_path(opts.load_info_filename)) else: - retcode = ydl.download(all_urls) + return ydl.download(all_urls) except DownloadCancelled: ydl.to_screen('Aborting remaining downloads') - retcode = 101 - - sys.exit(retcode) + return 101 def main(argv=None): try: - _real_main(argv) + _exit(*variadic(_real_main(argv))) except DownloadError: - sys.exit(1) + _exit(1) except SameFileError as e: - sys.exit(f'ERROR: {e}') + _exit(f'ERROR: {e}') except KeyboardInterrupt: - sys.exit('\nERROR: Interrupted by user') + _exit('\nERROR: Interrupted by user') except BrokenPipeError as e: # https://docs.python.org/3/library/signal.html#note-on-sigpipe devnull = os.open(os.devnull, os.O_WRONLY) os.dup2(devnull, sys.stdout.fileno()) - sys.exit(f'\nERROR: {e}') + _exit(f'\nERROR: {e}') + except optparse.OptParseError as e: + _exit(2, f'\n{e}') from .extractor import gen_extractors, list_extractors diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 2e8d384c0..1efdc8957 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1,7 +1,10 @@ +import collections +import contextlib import optparse import os.path import re import shlex +import string import sys from .compat import compat_expanduser, compat_get_terminal_size, compat_getenv @@ -15,6 +18,7 @@ from .postprocessor import ( SponsorBlockPP, ) from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE +from .update import detect_variant from .utils import ( OUTTMPL_TYPES, POSTPROCESS_WHEN, @@ -29,15 +33,9 @@ from .version import __version__ def parseOpts(overrideArguments=None, ignore_config_files='if_override'): - parser = create_parser() - root = Config(parser) - + root = Config(create_parser()) if ignore_config_files == 'if_override': ignore_config_files = overrideArguments is not None - if overrideArguments: - root.append_config(overrideArguments, label='Override') - else: - root.append_config(sys.argv[1:], label='Command-line') def _readUserConf(package_name, default=[]): # .config @@ -73,7 +71,7 @@ def parseOpts(overrideArguments=None, ignore_config_files='if_override'): def add_config(label, path, user=False): """ Adds config and returns whether to continue """ - if root.parse_args()[0].ignoreconfig: + if root.parse_known_args()[0].ignoreconfig: return False # Multiple package names can be given here # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for @@ -92,22 +90,44 @@ def parseOpts(overrideArguments=None, ignore_config_files='if_override'): def load_configs(): yield not ignore_config_files yield add_config('Portable', get_executable_path()) - yield add_config('Home', expand_path(root.parse_args()[0].paths.get('home', '')).strip()) + yield add_config('Home', expand_path(root.parse_known_args()[0].paths.get('home', '')).strip()) yield add_config('User', None, user=True) yield add_config('System', '/etc') - if all(load_configs()): - # If ignoreconfig is found inside the system configuration file, - # the user configuration is removed - if root.parse_args()[0].ignoreconfig: - user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None) - if user_conf is not None: - root.configs.pop(user_conf) + opts = optparse.Values({'verbose': True, 'print_help': False}) + try: + if overrideArguments: + root.append_config(overrideArguments, label='Override') + else: + root.append_config(sys.argv[1:], label='Command-line') - opts, args = root.parse_args() - if opts.verbose: - write_string(f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + '\n') - return parser, opts, args + if all(load_configs()): + # If ignoreconfig is found inside the system configuration file, + # the user configuration is removed + if root.parse_known_args()[0].ignoreconfig: + user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None) + if user_conf is not None: + root.configs.pop(user_conf) + + opts, args = root.parse_args() + except optparse.OptParseError: + with contextlib.suppress(optparse.OptParseError): + opts, _ = root.parse_known_args(strict=False) + raise + except (SystemExit, KeyboardInterrupt): + opts.verbose = False + raise + finally: + verbose = opts.verbose and f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + if verbose: + write_string(f'{verbose}\n') + if opts.print_help: + if verbose: + write_string('\n') + root.parser.print_help() + if opts.print_help: + sys.exit() + return root.parser, opts, args class _YoutubeDLHelpFormatter(optparse.IndentedHelpFormatter): @@ -133,10 +153,11 @@ class _YoutubeDLHelpFormatter(optparse.IndentedHelpFormatter): class _YoutubeDLOptionParser(optparse.OptionParser): # optparse is deprecated since python 3.2. So assume a stable interface even for private methods + ALIAS_TRIGGER_LIMIT = 100 def __init__(self): super().__init__( - prog='yt-dlp', + prog='yt-dlp' if detect_variant() == 'source' else None, version=__version__, usage='%prog [OPTIONS] URL [URL...]', epilog='See full documentation at https://github.com/yt-dlp/yt-dlp#readme', @@ -144,6 +165,29 @@ class _YoutubeDLOptionParser(optparse.OptionParser): conflict_handler='resolve', ) + _UNKNOWN_OPTION = (optparse.BadOptionError, optparse.AmbiguousOptionError) + _BAD_OPTION = optparse.OptionValueError + + def parse_known_args(self, args=None, values=None, strict=True): + """Same as parse_args, but ignore unknown switches. Similar to argparse.parse_known_args""" + self.rargs, self.largs = self._get_args(args), [] + self.values = values or self.get_default_values() + while self.rargs: + try: + self._process_args(self.largs, self.rargs, self.values) + except optparse.OptParseError as err: + if isinstance(err, self._UNKNOWN_OPTION): + self.largs.append(err.opt_str) + elif strict: + if isinstance(err, self._BAD_OPTION): + self.error(str(err)) + raise + return self.check_values(self.values, self.largs) + + def error(self, msg): + msg = f'{self.get_prog_name()}: error: {msg.strip()}\n' + raise optparse.OptParseError(f'{self.get_usage()}\n{msg}' if self.usage else msg) + def _get_args(self, args): return sys.argv[1:] if args is None else list(args) @@ -223,11 +267,44 @@ def create_parser(): setattr(parser.values, option.dest, out_dict) parser = _YoutubeDLOptionParser() + alias_group = optparse.OptionGroup(parser, 'Aliases') + Formatter = string.Formatter() + + def _create_alias(option, opt_str, value, parser): + aliases, opts = value + try: + nargs = len({i if f == '' else f + for i, (_, f, _, _) in enumerate(Formatter.parse(opts)) if f is not None}) + opts.format(*map(str, range(nargs))) # validate + except Exception as err: + raise optparse.OptionValueError(f'wrong {opt_str} OPTIONS formatting; {err}') + if alias_group not in parser.option_groups: + parser.add_option_group(alias_group) + + aliases = (x if x.startswith('-') else f'--{x}' for x in map(str.strip, aliases.split(','))) + try: + alias_group.add_option( + *aliases, help=opts, nargs=nargs, type='str' if nargs else None, + dest='_triggered_aliases', default=collections.defaultdict(int), + metavar=' '.join(f'ARG{i}' for i in range(nargs)), action='callback', + callback=_alias_callback, callback_kwargs={'opts': opts, 'nargs': nargs}) + except Exception as err: + raise optparse.OptionValueError(f'wrong {opt_str} formatting; {err}') + + def _alias_callback(option, opt_str, value, parser, opts, nargs): + counter = getattr(parser.values, option.dest) + counter[opt_str] += 1 + if counter[opt_str] > parser.ALIAS_TRIGGER_LIMIT: + raise optparse.OptionValueError(f'Alias {opt_str} exceeded invocation limit') + if nargs == 1: + value = [value] + assert (nargs == 0 and value is None) or len(value) == nargs + parser.rargs[:0] = shlex.split( + opts if value is None else opts.format(*map(shlex.quote, value))) general = optparse.OptionGroup(parser, 'General Options') general.add_option( - '-h', '--help', - action='help', + '-h', '--help', dest='print_help', action='store_true', help='Print this help text and exit') general.add_option( '--version', @@ -344,6 +421,18 @@ def create_parser(): 'Options that can help keep compatibility with youtube-dl or youtube-dlc ' 'configurations by reverting some of the changes made in yt-dlp. ' 'See "Differences in default behavior" for details')) + general.add_option( + '--alias', metavar='ALIASES OPTIONS', dest='_', type='str', nargs=2, + action='callback', callback=_create_alias, + help=( + 'Create aliases for an option string. Unless an alias starts with a dash "-", it is prefixed with "--". ' + 'Arguments are parsed according to the Python string formatting mini-language. ' + 'Eg: --alias get-audio,-X "-S=aext:{0},abr -x --audio-format {0}" creates options ' + '"--get-audio" and "-X" that takes an argument (ARG0) and expands to ' + '"-S=aext:ARG0,abr -x --audio-format ARG0". All defined aliases are listed in the --help output. ' + 'Alias options can trigger more aliases; so be carefull to avoid defining recursive options. ' + f'As a safety measure, each alias may be triggered a maximum of {_YoutubeDLOptionParser.ALIAS_TRIGGER_LIMIT} times. ' + 'This option can be used multiple times')) network = optparse.OptionGroup(parser, 'Network Options') network.add_option( diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index bcdb7d55b..f02f71177 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -5153,11 +5153,12 @@ def parse_http_range(range): class Config: own_args = None + parsed_args = None filename = None __initialized = False def __init__(self, parser, label=None): - self._parser, self.label = parser, label + self.parser, self.label = parser, label self._loaded_paths, self.configs = set(), [] def init(self, args=None, filename=None): @@ -5170,14 +5171,16 @@ class Config: return False self._loaded_paths.add(location) - self.__initialized = True - self.own_args, self.filename = args, filename - for location in self._parser.parse_args(args)[0].config_locations or []: + self.own_args, self.__initialized = args, True + opts, _ = self.parser.parse_known_args(args) + self.parsed_args, self.filename = args, filename + + for location in opts.config_locations or []: location = os.path.join(directory, expand_path(location)) if os.path.isdir(location): location = os.path.join(location, 'yt-dlp.conf') if not os.path.exists(location): - self._parser.error(f'config location {location} does not exist') + self.parser.error(f'config location {location} does not exist') self.append_config(self.read_file(location), location) return True @@ -5223,7 +5226,7 @@ class Config: return opts def append_config(self, *args, label=None): - config = type(self)(self._parser, label) + config = type(self)(self.parser, label) config._loaded_paths = self._loaded_paths if config.init(*args): self.configs.append(config) @@ -5232,10 +5235,13 @@ class Config: def all_args(self): for config in reversed(self.configs): yield from config.all_args - yield from self.own_args or [] + yield from self.parsed_args or [] + + def parse_known_args(self, **kwargs): + return self.parser.parse_known_args(self.all_args, **kwargs) def parse_args(self): - return self._parser.parse_args(self.all_args) + return self.parser.parse_args(self.all_args) class WebSocketsWrapper():