c

downloader.py

@@ -12,7 +12,7 @@ from multiprocessing import Manager, Pool, cpu_count
 from tqdm.auto import tqdm
 
 import ydl.yt_dlp as ydl
-from process.funcs import get_silent_logger, restart_program, setup_file_logger
+from process.funcs import get_silent_logger, remove_duplicates_from_playlist, restart_program, setup_file_logger
 from process.threads import download_video
 from ydl.files import create_directories, resolve_path
 
@@ -184,7 +184,7 @@ if not args.daemon:
 
 for i, target_url in tqdm(enumerate(url_list), total=len(url_list), position=0, desc='Inputs', disable=args.daemon):
     logger.info('Fetching playlist...')
-    playlist = yt_dlp.playlist_contents(target_url)
+    playlist = remove_duplicates_from_playlist(yt_dlp.playlist_contents(target_url))
 
     log_info_twice(f"Downloading item: '{playlist['title']}' {target_url}")
 

process/funcs.py

@@ -47,3 +47,13 @@ def get_silent_logger(name, level=logging.INFO, silent: bool = True):
     else:
         logger.setLevel(level)
     return logger
+
+
+def remove_duplicates_from_playlist(playlist):
+    videos = []
+    s = set()
+    for p, video in enumerate(playlist['entries']):
+        if video['id'] not in s:
+            videos.append(video)
+            s.add(video['id'])
+    return videos