c

downloader.py

@@ -149,27 +149,33 @@ class ytdl_logger(object):
 # https://github.com/yt-dlp/yt-dlp#embedding-examples
 ydl_opts = {
     'format': f'(bestvideo[filesize<{args.max_size}M][vcodec^=av01][height>=1080][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec=vp9.2][height>=1080][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec=vp9][height>=1080][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec^=av01][height>=1080]/bestvideo[filesize<{args.max_size}M][vcodec=vp9.2][height>=1080]/bestvideo[filesize<{args.max_size}M][vcodec=vp9][height>=1080]/bestvideo[filesize<{args.max_size}M][height>=1080]/bestvideo[filesize<{args.max_size}M][vcodec^=av01][height>=720][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec=vp9.2][height>=720][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec=vp9][height>=720][fps>30]/bestvideo[filesize<{args.max_size}M][vcodec^=av01][height>=720]/bestvideo[filesize<{args.max_size}M][vcodec=vp9.2][height>=720]/bestvideo[filesize<{args.max_size}M][vcodec=vp9][height>=720]/bestvideo[filesize<{args.max_size}M][height>=720]/bestvideo[filesize<{args.max_size}M])+(bestaudio[acodec=opus]/bestaudio)/best',
-    'outtmpl': f'{args.output}/%(title)s --- %(uploader)s --- %(uploader_id)s --- %(id)s',
+    'outtmpl': f'{args.output}/%(id)s --- %(title)s --- %(uploader)s --- %(uploader_id)s',
     'merge_output_format': 'mkv',
     'logtostderr': True,
     'embedchapters': True,
-    # 'writethumbnail': True, # Save the thumbnail to a file. Embedding seems to be broken right now so this is an alternative.
+    'writethumbnail': True,  # Save the thumbnail to a file. Embedding seems to be broken right now so this is an alternative.
     'embedthumbnail': True,
+    'embeddescription': True,
     'writesubtitles': True,
     # 'allsubtitles': True, # Download every language.
     'subtitlesformat': 'vtt',
     'subtitleslangs': ['en'],
     'writeautomaticsub': True,
+    # 'writedescription': True,
+    'ignoreerrors': True,
+    'continuedl': False,
+    'addmetadata': True,
+    'writeinfojson': True,
     'postprocessors': [
         {'key': 'FFmpegEmbedSubtitle'},
         {'key': 'FFmpegMetadata', 'add_metadata': True},
         {'key': 'EmbedThumbnail', 'already_have_thumbnail': True},
+        # {'key': 'FFmpegSubtitlesConvertor', 'format': 'srt'}
     ],
-
 }
 
 main_opts = dict(ydl_opts, **{'logger': ytdl_logger()})
-thread_opts = dict(ydl_opts, **{'logger': ydl.ytdl_no_logger()})
+# thread_opts = dict(ydl_opts, **{'logger': ydl.ytdl_no_logger()})
 yt_dlp = ydl.YDL(main_opts)
 
 # Init bars
@@ -196,7 +202,7 @@ for i, target_url in tqdm(enumerate(url_list), total=len(url_list), position=0,
     download_queue = []
     s = set()
     for p, video in enumerate(playlist['entries']):
-        if video['id'] not in downloaded_videos or video['id'] not in s:
+        if video['id'] not in downloaded_videos and video['id'] not in s:
             download_queue.append(video)
             s.add(video['id'])
     playlist_bar.update(len(downloaded_videos))
@@ -208,8 +214,8 @@ for i, target_url in tqdm(enumerate(url_list), total=len(url_list), position=0,
             for result in pool.imap_unordered(download_video,
                                               ((video, {
                                                   'bars': video_bars,
-                                                  'download_archive': downloaded_videos,
-                                                  'ydl_opts': thread_opts,
+                                                  'ydl_opts': ydl_opts,
+                                                  'output_dir': args.output
                                               }) for video in download_queue)):
                 # Save the video ID to the file
                 if result['downloaded_video_id']:
@@ -224,7 +230,7 @@ for i, target_url in tqdm(enumerate(url_list), total=len(url_list), position=0,
                     log_info_twice(line)
                 playlist_bar.update()
     else:
-        playlist_bar.write(f"All videos already downloaded for '{playlist['title']}'")
+        playlist_bar.write(f"All videos already downloaded for '{playlist['title']}'.")
     log_info_twice(f"Finished item: '{playlist['title']}' {target_url}")
 log_info_twice(f"Finished process in {round(math.ceil(time.time() - start_time) / 60, 2)} min.")
 

process/funcs.py

@@ -24,11 +24,12 @@ def restart_program():
 
 def setup_file_logger(name, log_file, level=logging.INFO, format_str: str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s', filemode='a', no_console: bool = True):
     formatter = logging.Formatter(format_str)
-    handler = logging.FileHandler(log_file, mode=filemode)
-    handler.setFormatter(formatter)
 
     logger = logging.getLogger(name)
     logger.setLevel(level)
+    handler = logging.FileHandler(log_file, mode=filemode)
+    handler.setLevel(level)
+    handler.setFormatter(formatter)
     logger.addHandler(handler)
 
     # Silence console logging
@@ -49,10 +50,10 @@ def get_silent_logger(name, level=logging.INFO, silent: bool = True):
     return logger
 
 
-def remove_duplicates_from_playlist(playlist):
+def remove_duplicates_from_playlist(entries):
     videos = []
     s = set()
-    for p, video in enumerate(playlist['entries']):
+    for p, video in enumerate(entries):
         if video['id'] not in s:
             videos.append(video)
             s.add(video['id'])

process/threads.py

@@ -6,6 +6,24 @@ import numpy as np
 from tqdm.auto import tqdm
 
 import ydl.yt_dlp as ydl
+from process.funcs import setup_file_logger
+
+
+class ytdl_logger(object):
+    def __init__(self, logger):
+        self.logger = logger
+
+    def debug(self, msg):
+        self.logger.info(msg)
+
+    def info(self, msg):
+        self.logger.info(msg)
+
+    def warning(self, msg):
+        self.logger.warning(msg)
+
+    def error(self, msg):
+        self.logger.error(msg)
 
 
 def is_manager_lock_locked(lock) -> bool:
@@ -56,6 +74,7 @@ def download_video(args) -> dict:
         desc_with = int(np.round(os.get_terminal_size()[0] * (1 / 4)))
         bar = tqdm(total=100, position=(offset if locked else None), desc=video['title'].ljust(desc_with)[:desc_with], bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}{postfix}]', leave=False)
 
+    kwargs['ydl_opts']['logger'] = ytdl_logger(setup_file_logger(video['id'], kwargs['output_dir'] / f"{video['id']}.log"))
     yt_dlp = ydl.YDL(kwargs['ydl_opts'])
     output_dict = {'downloaded_video_id': None, 'blacklist_video_id': None, 'video_error_logger_msg': [], 'status_msg': [], 'logger_msg': []}  # empty object
     start_time = time.time()