From 134c913cca8e526a0128c62741689c0d0d05df03 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Jul 2022 02:14:23 +0530 Subject: [PATCH] Discard info_dict from memory if no longer needed Closes #1399 --- yt_dlp/YoutubeDL.py | 20 ++++++++++++++++---- yt_dlp/__init__.py | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 85219ac95c..7e9c0949b2 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -319,9 +319,14 @@ class YoutubeDL: default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing encoding: Use this encoding instead of the system-specified. - extract_flat: Do not resolve URLs, return the immediate result. - Pass in 'in_playlist' to only show this behavior for - playlist items. + extract_flat: Whether to resolve and process url_results further + * False: Always process (default) + * True: Never process + * 'in_playlist': Do not process inside playlist/multi_video + * 'discard': Always process, but don't return the result + from inside playlist/multi_video + * 'discard_in_playlist': Same as "discard", but only for + playlists (not multi_video) wait_for_video: If given, wait for scheduled streams to become available. The value should be a tuple containing the range (min_secs, max_secs) to wait between retries @@ -1725,6 +1730,12 @@ class YoutubeDL: self.to_screen(f'[{ie_result["extractor"]}] Playlist {title}: Downloading {n_entries} videos' f'{format_field(ie_result, "playlist_count", " of %s")}') + keep_resolved_entries = self.params.get('extract_flat') != 'discard' + if self.params.get('extract_flat') == 'discard_in_playlist': + keep_resolved_entries = ie_result['_type'] != 'playlist' + if keep_resolved_entries: + self.write_debug('The information of all playlist entries will be held in memory') + failures = 0 max_failures = self.params.get('skip_playlist_after_errors') or float('inf') for i, (playlist_index, entry) in enumerate(entries): @@ -1765,7 +1776,8 @@ class YoutubeDL: self.report_error( f'Skipping the remaining entries in playlist "{title}" since {failures} items failed extraction') break - resolved_entries[i] = (playlist_index, entry_result) + if keep_resolved_entries: + resolved_entries[i] = (playlist_index, entry_result) # Update with processed data ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries)) or ([], []) diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index fd44e1ab96..7caf41c607 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -688,6 +688,21 @@ def parse_options(argv=None): 'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl' )) + playlist_pps = [pp for pp in postprocessors if pp.get('when') == 'playlist'] + write_playlist_infojson = (opts.writeinfojson and not opts.clean_infojson + and opts.allow_playlist_files and opts.outtmpl.get('pl_infojson') != '') + if not any(( + opts.extract_flat, + opts.dump_single_json, + opts.forceprint.get('playlist'), + opts.print_to_file.get('playlist'), + write_playlist_infojson, + )): + if not playlist_pps: + opts.extract_flat = 'discard' + elif playlist_pps == [{'key': 'FFmpegConcat', 'only_multi_video': True, 'when': 'playlist'}]: + opts.extract_flat = 'discard_in_playlist' + final_ext = ( opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS