From 141a8dff98874a426d7fbe772e0a8421bb42656f Mon Sep 17 00:00:00 2001 From: coletdjnz Date: Thu, 6 Apr 2023 19:44:22 +1200 Subject: [PATCH] [extractor/youtube] Fix comment loop detection for pinned comments (#6714) Pinned comments may repeat a second time - this is expected. Fixes https://github.com/yt-dlp/yt-dlp/issues/6712 Authored by: coletdjnz --- yt_dlp/extractor/youtube.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index ca56f112bb..6dc36f9b99 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -3316,9 +3316,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor): comment = self._extract_comment(comment_renderer, parent) if not comment: continue + is_pinned = bool(traverse_obj(comment_renderer, 'pinnedCommentBadge')) + comment_id = comment['id'] + if is_pinned: + tracker['pinned_comment_ids'].add(comment_id) # Sometimes YouTube may break and give us infinite looping comments. # See: https://github.com/yt-dlp/yt-dlp/issues/6290 - if comment['id'] in tracker['seen_comment_ids']: + if comment_id in tracker['seen_comment_ids']: + if comment_id in tracker['pinned_comment_ids'] and not is_pinned: + # Pinned comments may appear a second time in newest first sort + # See: https://github.com/yt-dlp/yt-dlp/issues/6712 + continue self.report_warning('Detected YouTube comments looping. Stopping comment extraction as we probably cannot get any more.') yield else: @@ -3348,7 +3356,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): current_page_thread=0, total_parent_comments=0, total_reply_comments=0, - seen_comment_ids=set()) + seen_comment_ids=set(), + pinned_comment_ids=set() + ) # TODO: Deprecated # YouTube comments have a max depth of 2