Mirrors
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[cleanup] Misc cleanup and refactor (#2173)

This commit is contained in:
pukkandan 2022-04-18 02:28:28 +05:30
parent b6dc37fe2a
commit 19a0394044
No known key found for this signature in database
GPG Key ID: 7EEE9E1E817D0A39
31 changed files with 263 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
devscripts/make_readme.py
View File

@ -6,22 +6,25 @@ import re
import sys import sys
README_FILE = 'README.md' README_FILE = 'README.md'
helptext = sys.stdin.read()
OPTIONS_START = 'General Options:'
OPTIONS_END = 'CONFIGURATION'
EPILOG_START = 'See full documentation'
helptext = sys.stdin.read()
if isinstance(helptext, bytes): if isinstance(helptext, bytes):
helptext = helptext.decode('utf-8') helptext = helptext.decode('utf-8')
start, end = helptext.index(f'\n {OPTIONS_START}'), helptext.index(f'\n{EPILOG_START}')
options = re.sub(r'(?m)^ (\w.+)$', r'## \1', helptext[start + 1: end + 1])
with open(README_FILE, encoding='utf-8') as f: with open(README_FILE, encoding='utf-8') as f:
oldreadme = f.read() readme = f.read()
header = oldreadme[:oldreadme.index('## General Options:')] header = readme[:readme.index(f'## {OPTIONS_START}')]
footer = oldreadme[oldreadme.index('# CONFIGURATION'):] footer = readme[readme.index(f'# {OPTIONS_END}'):]
options = helptext[helptext.index(' General Options:'):]
options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options)
options = options + '\n'
with open(README_FILE, 'w', encoding='utf-8') as f: with open(README_FILE, 'w', encoding='utf-8') as f:
f.write(header) for part in (header, options, footer):
f.write(options) f.write(part)
f.write(footer)

2
setup.cfg
View File

@ -2,5 +2,5 @@
universal = True universal = True
[flake8] [flake8]
exclude = yt_dlp/extractor/__init__.py,devscripts/buildserver.py,devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv,devscripts/create-github-release.py,devscripts/release.sh,devscripts/show-downloads-statistics.py exclude = devscripts/lazy_load_template.py,devscripts/make_issue_template.py,setup.py,build,.git,venv
ignore = E402,E501,E731,E741,W503 ignore = E402,E501,E731,E741,W503

4
supportedsites.md
View File

@ -1147,8 +1147,8 @@
- **Sport5** - **Sport5**
- **SportBox** - **SportBox**
- **SportDeutschland** - **SportDeutschland**
- **spotify** - **spotify**: Spotify episodes
- **spotify:show** - **spotify:show**: Spotify shows
- **Spreaker** - **Spreaker**
- **SpreakerPage** - **SpreakerPage**
- **SpreakerShow** - **SpreakerShow**

4
test/test_compat.py
View File

@ -35,9 +35,11 @@ class TestCompat(unittest.TestCase):
def test_compat_expanduser(self): def test_compat_expanduser(self):
old_home = os.environ.get('HOME') old_home = os.environ.get('HOME')
test_str = r'C:\Documents and Settings\тест\Application Data' test_str = R'C:\Documents and Settings\тест\Application Data'
try:
compat_setenv('HOME', test_str) compat_setenv('HOME', test_str)
self.assertEqual(compat_expanduser('~'), test_str) self.assertEqual(compat_expanduser('~'), test_str)
finally:
compat_setenv('HOME', old_home or '') compat_setenv('HOME', old_home or '')
def test_all_present(self): def test_all_present(self):

11
test/test_execution.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import contextlib
import os import os
import subprocess import subprocess
import sys import sys
@ -22,14 +23,14 @@ class TestExecution(unittest.TestCase):
subprocess.check_call([sys.executable, '-c', 'import yt_dlp'], cwd=rootDir) subprocess.check_call([sys.executable, '-c', 'import yt_dlp'], cwd=rootDir)
def test_module_exec(self): def test_module_exec(self):
subprocess.check_call([sys.executable, '-m', 'yt_dlp', '--version'], cwd=rootDir, stdout=_DEV_NULL) subprocess.check_call([sys.executable, '-m', 'yt_dlp', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
def test_main_exec(self): def test_main_exec(self):
subprocess.check_call([sys.executable, 'yt_dlp/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) subprocess.check_call([sys.executable, 'yt_dlp/__main__.py', '--ignore-config', '--version'], cwd=rootDir, stdout=_DEV_NULL)
def test_cmdline_umlauts(self): def test_cmdline_umlauts(self):
p = subprocess.Popen( p = subprocess.Popen(
[sys.executable, 'yt_dlp/__main__.py', encodeArgument('ä'), '--version'], [sys.executable, 'yt_dlp/__main__.py', '--ignore-config', encodeArgument('ä'), '--version'],
cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE) cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
_, stderr = p.communicate() _, stderr = p.communicate()
self.assertFalse(stderr) self.assertFalse(stderr)
@ -39,10 +40,8 @@ class TestExecution(unittest.TestCase):
subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL) subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL)
subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL) subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
finally: finally:
try: with contextlib.suppress(OSError):
os.remove('yt_dlp/extractor/lazy_extractors.py') os.remove('yt_dlp/extractor/lazy_extractors.py')
except OSError:
pass
if __name__ == '__main__': if __name__ == '__main__':

12
test/test_utils.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Allow direct execution # Allow direct execution
import contextlib
import os import os
import sys import sys
import unittest import unittest
@ -267,11 +268,18 @@ class TestUtil(unittest.TestCase):
compat_setenv('yt_dlp_EXPATH_PATH', 'expanded') compat_setenv('yt_dlp_EXPATH_PATH', 'expanded')
self.assertEqual(expand_path(env('yt_dlp_EXPATH_PATH')), 'expanded') self.assertEqual(expand_path(env('yt_dlp_EXPATH_PATH')), 'expanded')
old_home = os.environ.get('HOME')
test_str = R'C:\Documents and Settings\тест\Application Data'
try:
compat_setenv('HOME', test_str)
self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME')) self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME'))
self.assertEqual(expand_path('~'), compat_getenv('HOME')) self.assertEqual(expand_path('~'), compat_getenv('HOME'))
self.assertEqual( self.assertEqual(
expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')), expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')),
'%s/expanded' % compat_getenv('HOME')) '%s/expanded' % compat_getenv('HOME'))
finally:
compat_setenv('HOME', old_home or '')
def test_prepend_extension(self): def test_prepend_extension(self):
self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext') self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
@ -1814,10 +1822,8 @@ Line 1
else: else:
self.assertFalse(testing_write, f'{test_mode} is not blocked by {lock_mode}') self.assertFalse(testing_write, f'{test_mode} is not blocked by {lock_mode}')
finally: finally:
try: with contextlib.suppress(OSError):
os.remove(FILE) os.remove(FILE)
except Exception:
pass
if __name__ == '__main__': if __name__ == '__main__':

12
test/test_verbose_output.py
View File

@ -13,7 +13,8 @@ class TestVerboseOutput(unittest.TestCase):
def test_private_info_arg(self): def test_private_info_arg(self):
outp = subprocess.Popen( outp = subprocess.Popen(
[ [
sys.executable, 'yt_dlp/__main__.py', '-v', sys.executable, 'yt_dlp/__main__.py',
'-v', '--ignore-config',
'--username', 'johnsmith@gmail.com', '--username', 'johnsmith@gmail.com',
'--password', 'my_secret_password', '--password', 'my_secret_password',
], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -26,7 +27,8 @@ class TestVerboseOutput(unittest.TestCase):
def test_private_info_shortarg(self): def test_private_info_shortarg(self):
outp = subprocess.Popen( outp = subprocess.Popen(
[ [
sys.executable, 'yt_dlp/__main__.py', '-v', sys.executable, 'yt_dlp/__main__.py',
'-v', '--ignore-config',
'-u', 'johnsmith@gmail.com', '-u', 'johnsmith@gmail.com',
'-p', 'my_secret_password', '-p', 'my_secret_password',
], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -39,7 +41,8 @@ class TestVerboseOutput(unittest.TestCase):
def test_private_info_eq(self): def test_private_info_eq(self):
outp = subprocess.Popen( outp = subprocess.Popen(
[ [
sys.executable, 'yt_dlp/__main__.py', '-v', sys.executable, 'yt_dlp/__main__.py',
'-v', '--ignore-config',
'--username=johnsmith@gmail.com', '--username=johnsmith@gmail.com',
'--password=my_secret_password', '--password=my_secret_password',
], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -52,7 +55,8 @@ class TestVerboseOutput(unittest.TestCase):
def test_private_info_shortarg_eq(self): def test_private_info_shortarg_eq(self):
outp = subprocess.Popen( outp = subprocess.Popen(
[ [
sys.executable, 'yt_dlp/__main__.py', '-v', sys.executable, 'yt_dlp/__main__.py',
'-v', '--ignore-config',
'-u=johnsmith@gmail.com', '-u=johnsmith@gmail.com',
'-p=my_secret_password', '-p=my_secret_password',
], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

1
test/test_write_annotations.py.disabled
View File

@ -6,7 +6,6 @@ import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import io
import xml.etree.ElementTree import xml.etree.ElementTree
from test.helper import get_params, is_download_test, try_rm from test.helper import get_params, is_download_test, try_rm

5
test/test_youtube_signature.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Allow direct execution # Allow direct execution
import contextlib
import os import os
import sys import sys
import unittest import unittest
@ -127,11 +128,9 @@ class TestSignature(unittest.TestCase):
os.mkdir(self.TESTDATA_DIR) os.mkdir(self.TESTDATA_DIR)
def tearDown(self): def tearDown(self):
try: with contextlib.suppress(OSError):
for f in os.listdir(self.TESTDATA_DIR): for f in os.listdir(self.TESTDATA_DIR):
os.remove(f) os.remove(f)
except OSError:
pass
def t_factory(name, sig_func, url_pattern): def t_factory(name, sig_func, url_pattern):

34
yt_dlp/YoutubeDL.py
View File

@ -23,7 +23,6 @@ import tokenize
import traceback import traceback
import unicodedata import unicodedata
import urllib.request import urllib.request
from enum import Enum
from string import ascii_letters from string import ascii_letters
from .cache import Cache from .cache import Cache
@ -82,6 +81,7 @@ from .utils import (
ISO3166Utils, ISO3166Utils,
LazyList, LazyList,
MaxDownloadsReached, MaxDownloadsReached,
Namespace,
PagedList, PagedList,
PerRequestProxyHandler, PerRequestProxyHandler,
Popen, Popen,
@ -878,14 +878,15 @@ class YoutubeDL:
raise DownloadError(message, exc_info) raise DownloadError(message, exc_info)
self._download_retcode = 1 self._download_retcode = 1
class Styles(Enum): Styles = Namespace(
HEADERS = 'yellow' HEADERS='yellow',
EMPHASIS = 'light blue' EMPHASIS='light blue',
ID = 'green' ID='green',
DELIM = 'blue' DELIM='blue',
ERROR = 'red' ERROR='red',
WARNING = 'yellow' WARNING='yellow',
SUPPRESS = 'light black' SUPPRESS='light black',
)
def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_encoding=False): def _format_text(self, handle, allow_colors, text, f, fallback=None, *, test_encoding=False):
text = str(text) text = str(text)
@ -896,8 +897,6 @@ class YoutubeDL:
text = text.encode(encoding, 'ignore').decode(encoding) text = text.encode(encoding, 'ignore').decode(encoding)
if fallback is not None and text != original_text: if fallback is not None and text != original_text:
text = fallback text = fallback
if isinstance(f, Enum):
f = f.value
return format_text(text, f) if allow_colors else text if fallback is None else fallback return format_text(text, f) if allow_colors else text if fallback is None else fallback
def _format_screen(self, *args, **kwargs): def _format_screen(self, *args, **kwargs):
@ -1760,7 +1759,8 @@ class YoutubeDL:
playlist_index, entry = entry_tuple playlist_index, entry = entry_tuple
if 'playlist-index' in self.params.get('compat_opts', []): if 'playlist-index' in self.params.get('compat_opts', []):
playlist_index = playlistitems[i - 1] if playlistitems else i + playliststart - 1 playlist_index = playlistitems[i - 1] if playlistitems else i + playliststart - 1
self.to_screen(f'[download] Downloading video {i} of {n_entries}') self.to_screen('[download] Downloading video %s of %s' % (
self._format_screen(i, self.Styles.ID), self._format_screen(n_entries, self.Styles.EMPHASIS)))
# This __x_forwarded_for_ip thing is a bit ugly but requires # This __x_forwarded_for_ip thing is a bit ugly but requires
# minimal changes # minimal changes
if x_forwarded_for: if x_forwarded_for:
@ -2337,11 +2337,9 @@ class YoutubeDL:
if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None: if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
# Working around out-of-range timestamp values (e.g. negative ones on Windows, # Working around out-of-range timestamp values (e.g. negative ones on Windows,
# see http://bugs.python.org/issue1646728) # see http://bugs.python.org/issue1646728)
try: with contextlib.suppress(ValueError, OverflowError, OSError):
upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key]) upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
info_dict[date_key] = upload_date.strftime('%Y%m%d') info_dict[date_key] = upload_date.strftime('%Y%m%d')
except (ValueError, OverflowError, OSError):
pass
live_keys = ('is_live', 'was_live') live_keys = ('is_live', 'was_live')
live_status = info_dict.get('live_status') live_status = info_dict.get('live_status')
@ -3631,10 +3629,8 @@ class YoutubeDL:
if re.match('[0-9a-f]+', out): if re.match('[0-9a-f]+', out):
write_debug('Git HEAD: %s' % out) write_debug('Git HEAD: %s' % out)
except Exception: except Exception:
try: with contextlib.suppress(Exception):
sys.exc_clear() sys.exc_clear()
except Exception:
pass
def python_implementation(): def python_implementation():
impl_name = platform.python_implementation() impl_name = platform.python_implementation()
@ -3651,7 +3647,7 @@ class YoutubeDL:
exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self) exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self)
ffmpeg_features = {key for key, val in ffmpeg_features.items() if val} ffmpeg_features = {key for key, val in ffmpeg_features.items() if val}
if ffmpeg_features: if ffmpeg_features:
exe_versions['ffmpeg'] += ' (%s)' % ','.join(ffmpeg_features) exe_versions['ffmpeg'] += ' (%s)' % ','.join(sorted(ffmpeg_features))
exe_versions['rtmpdump'] = rtmpdump_version() exe_versions['rtmpdump'] = rtmpdump_version()
exe_versions['phantomjs'] = PhantomJSwrapper._version() exe_versions['phantomjs'] = PhantomJSwrapper._version()

3
yt_dlp/__init__.py
View File

@ -404,7 +404,8 @@ def validate_options(opts):
report_conflict('--sponskrub', 'sponskrub', '--remove-chapters', 'remove_chapters') report_conflict('--sponskrub', 'sponskrub', '--remove-chapters', 'remove_chapters')
report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-mark', 'sponsorblock_mark') report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-mark', 'sponsorblock_mark')
report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-remove', 'sponsorblock_remove') report_conflict('--sponskrub', 'sponskrub', '--sponsorblock-remove', 'sponsorblock_remove')
report_conflict('--sponskrub-cut', 'sponskrub_cut', '--split-chapter', 'split_chapters', val1=opts.sponskrub and opts.sponskrub_cut) report_conflict('--sponskrub-cut', 'sponskrub_cut', '--split-chapter', 'split_chapters',
val1=opts.sponskrub and opts.sponskrub_cut)
# Conflicts with --allow-unplayable-formats # Conflicts with --allow-unplayable-formats
report_conflict('--add-metadata', 'addmetadata') report_conflict('--add-metadata', 'addmetadata')

2
yt_dlp/aes.py
View File

@ -493,7 +493,7 @@ def ghash(subkey, data):
last_y = [0] * BLOCK_SIZE_BYTES last_y = [0] * BLOCK_SIZE_BYTES
for i in range(0, len(data), BLOCK_SIZE_BYTES): for i in range(0, len(data), BLOCK_SIZE_BYTES):
block = data[i : i + BLOCK_SIZE_BYTES] # noqa: E203 block = data[i: i + BLOCK_SIZE_BYTES]
last_y = block_product(xor(last_y, block), subkey) last_y = block_product(xor(last_y, block), subkey)
return last_y return last_y

5
yt_dlp/cache.py
View File

@ -1,3 +1,4 @@
import contextlib
import errno import errno
import json import json
import os import os
@ -57,7 +58,7 @@ class Cache:
return default return default
cache_fn = self._get_cache_fn(section, key, dtype) cache_fn = self._get_cache_fn(section, key, dtype)
try: with contextlib.suppress(OSError):
try: try:
with open(cache_fn, encoding='utf-8') as cachef: with open(cache_fn, encoding='utf-8') as cachef:
self._ydl.write_debug(f'Loading {section}.{key} from cache') self._ydl.write_debug(f'Loading {section}.{key} from cache')
@ -68,8 +69,6 @@ class Cache:
except OSError as oe: except OSError as oe:
file_size = str(oe) file_size = str(oe)
self._ydl.report_warning(f'Cache retrieval from {cache_fn} failed ({file_size})') self._ydl.report_warning(f'Cache retrieval from {cache_fn} failed ({file_size})')
except OSError:
pass # No cache available
return default return default

23
yt_dlp/compat.py
View File

@ -1,6 +1,7 @@
import asyncio import asyncio
import base64 import base64
import collections import collections
import contextlib
import ctypes import ctypes
import getpass import getpass
import html import html
@ -54,14 +55,11 @@ if compat_os_name == 'nt':
def compat_shlex_quote(s): def compat_shlex_quote(s):
return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
else: else:
from shlex import quote as compat_shlex_quote from shlex import quote as compat_shlex_quote # noqa: F401
def compat_ord(c): def compat_ord(c):
if type(c) is int: return c if isinstance(c, int) else ord(c)
return c
else:
return ord(c)
def compat_setenv(key, value, env=os.environ): def compat_setenv(key, value, env=os.environ):
@ -118,16 +116,17 @@ except ImportError:
# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl # Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
# See https://github.com/yt-dlp/yt-dlp/issues/792 # See https://github.com/yt-dlp/yt-dlp/issues/792
# https://docs.python.org/3/library/os.path.html#os.path.expanduser # https://docs.python.org/3/library/os.path.html#os.path.expanduser
if compat_os_name in ('nt', 'ce') and 'HOME' in os.environ: if compat_os_name in ('nt', 'ce'):
_userhome = os.environ['HOME']
def compat_expanduser(path): def compat_expanduser(path):
if not path.startswith('~'): HOME = os.environ.get('HOME')
if not HOME:
return os.path.expanduser(path)
elif not path.startswith('~'):
return path return path
i = path.replace('\\', '/', 1).find('/') # ~user i = path.replace('\\', '/', 1).find('/') # ~user
if i < 0: if i < 0:
i = len(path) i = len(path)
userhome = os.path.join(os.path.dirname(_userhome), path[1:i]) if i > 1 else _userhome userhome = os.path.join(os.path.dirname(HOME), path[1:i]) if i > 1 else HOME
return userhome + path[i:] return userhome + path[i:]
else: else:
compat_expanduser = os.path.expanduser compat_expanduser = os.path.expanduser
@ -158,11 +157,9 @@ def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.pytho
global WINDOWS_VT_MODE global WINDOWS_VT_MODE
startupinfo = subprocess.STARTUPINFO() startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
try: with contextlib.suppress(Exception):
subprocess.Popen('', shell=True, startupinfo=startupinfo).wait() subprocess.Popen('', shell=True, startupinfo=startupinfo).wait()
WINDOWS_VT_MODE = True WINDOWS_VT_MODE = True
except Exception:
pass
# Deprecated # Deprecated

35
yt_dlp/cookies.py
View File

@ -167,7 +167,7 @@ def _firefox_browser_dir():
if sys.platform in ('linux', 'linux2'): if sys.platform in ('linux', 'linux2'):
return os.path.expanduser('~/.mozilla/firefox') return os.path.expanduser('~/.mozilla/firefox')
elif sys.platform == 'win32': elif sys.platform == 'win32':
return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles') return os.path.expandvars(R'%APPDATA%\Mozilla\Firefox\Profiles')
elif sys.platform == 'darwin': elif sys.platform == 'darwin':
return os.path.expanduser('~/Library/Application Support/Firefox') return os.path.expanduser('~/Library/Application Support/Firefox')
else: else:
@ -191,12 +191,12 @@ def _get_chromium_based_browser_settings(browser_name):
appdata_local = os.path.expandvars('%LOCALAPPDATA%') appdata_local = os.path.expandvars('%LOCALAPPDATA%')
appdata_roaming = os.path.expandvars('%APPDATA%') appdata_roaming = os.path.expandvars('%APPDATA%')
browser_dir = { browser_dir = {
'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'), 'brave': os.path.join(appdata_local, R'BraveSoftware\Brave-Browser\User Data'),
'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'), 'chrome': os.path.join(appdata_local, R'Google\Chrome\User Data'),
'chromium': os.path.join(appdata_local, r'Chromium\User Data'), 'chromium': os.path.join(appdata_local, R'Chromium\User Data'),
'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'), 'edge': os.path.join(appdata_local, R'Microsoft\Edge\User Data'),
'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'), 'opera': os.path.join(appdata_roaming, R'Opera Software\Opera Stable'),
'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'), 'vivaldi': os.path.join(appdata_local, R'Vivaldi\User Data'),
}[browser_name] }[browser_name]
elif sys.platform == 'darwin': elif sys.platform == 'darwin':
@ -237,8 +237,8 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
logger.info(f'Extracting cookies from {browser_name}') logger.info(f'Extracting cookies from {browser_name}')
if not SQLITE_AVAILABLE: if not SQLITE_AVAILABLE:
logger.warning(('Cannot extract cookies from {} without sqlite3 support. ' logger.warning(f'Cannot extract cookies from {browser_name} without sqlite3 support. '
'Please use a python interpreter compiled with sqlite3 support').format(browser_name)) 'Please use a python interpreter compiled with sqlite3 support')
return YoutubeDLCookieJar() return YoutubeDLCookieJar()
config = _get_chromium_based_browser_settings(browser_name) config = _get_chromium_based_browser_settings(browser_name)
@ -269,8 +269,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
cursor.connection.text_factory = bytes cursor.connection.text_factory = bytes
column_names = _get_column_names(cursor, 'cookies') column_names = _get_column_names(cursor, 'cookies')
secure_column = 'is_secure' if 'is_secure' in column_names else 'secure' secure_column = 'is_secure' if 'is_secure' in column_names else 'secure'
cursor.execute('SELECT host_key, name, value, encrypted_value, path, ' cursor.execute(f'SELECT host_key, name, value, encrypted_value, path, expires_utc, {secure_column} FROM cookies')
'expires_utc, {} FROM cookies'.format(secure_column))
jar = YoutubeDLCookieJar() jar = YoutubeDLCookieJar()
failed_cookies = 0 failed_cookies = 0
unencrypted_cookies = 0 unencrypted_cookies = 0
@ -346,11 +345,11 @@ class ChromeCookieDecryptor:
""" """
def decrypt(self, encrypted_value): def decrypt(self, encrypted_value):
raise NotImplementedError raise NotImplementedError('Must be implemented by sub classes')
@property @property
def cookie_counts(self): def cookie_counts(self):
raise NotImplementedError raise NotImplementedError('Must be implemented by sub classes')
def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None): def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=None):
@ -361,8 +360,7 @@ def get_cookie_decryptor(browser_root, browser_keyring_name, logger, *, keyring=
elif sys.platform == 'win32': elif sys.platform == 'win32':
return WindowsChromeCookieDecryptor(browser_root, logger) return WindowsChromeCookieDecryptor(browser_root, logger)
else: else:
raise NotImplementedError('Chrome cookie decryption is not supported ' raise NotImplementedError(f'Chrome cookie decryption is not supported on this platform: {sys.platform}')
'on this platform: {}'.format(sys.platform))
class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): class LinuxChromeCookieDecryptor(ChromeCookieDecryptor):
@ -546,8 +544,7 @@ class DataParser:
def skip(self, num_bytes, description='unknown'): def skip(self, num_bytes, description='unknown'):
if num_bytes > 0: if num_bytes > 0:
self._logger.debug('skipping {} bytes ({}): {}'.format( self._logger.debug(f'skipping {num_bytes} bytes ({description}): {self.read_bytes(num_bytes)!r}')
num_bytes, description, self.read_bytes(num_bytes)))
elif num_bytes < 0: elif num_bytes < 0:
raise ParserError(f'invalid skip of {num_bytes} bytes') raise ParserError(f'invalid skip of {num_bytes} bytes')
@ -784,8 +781,8 @@ def _get_kwallet_password(browser_keyring_name, logger):
stdout, stderr = proc.communicate_or_kill() stdout, stderr = proc.communicate_or_kill()
if proc.returncode != 0: if proc.returncode != 0:
logger.error('kwallet-query failed with return code {}. Please consult ' logger.error(f'kwallet-query failed with return code {proc.returncode}. Please consult '
'the kwallet-query man page for details'.format(proc.returncode)) 'the kwallet-query man page for details')
return b'' return b''
else: else:
if stdout.lower().startswith(b'failed to read'): if stdout.lower().startswith(b'failed to read'):

98
yt_dlp/downloader/common.py
View File

@ -1,3 +1,4 @@
import contextlib
import errno import errno
import os import os
import random import random
@ -12,6 +13,7 @@ from ..minicurses import (
) )
from ..utils import ( from ..utils import (
LockingUnsupportedError, LockingUnsupportedError,
Namespace,
decodeArgument, decodeArgument,
encodeFilename, encodeFilename,
error_to_compat_str, error_to_compat_str,
@ -70,12 +72,30 @@ class FileDownloader:
def __init__(self, ydl, params): def __init__(self, ydl, params):
"""Create a FileDownloader object with the given options.""" """Create a FileDownloader object with the given options."""
self.ydl = ydl self._set_ydl(ydl)
self._progress_hooks = [] self._progress_hooks = []
self.params = params self.params = params
self._prepare_multiline_status() self._prepare_multiline_status()
self.add_progress_hook(self.report_progress) self.add_progress_hook(self.report_progress)
def _set_ydl(self, ydl):
self.ydl = ydl
for func in (
'deprecation_warning',
'report_error',
'report_file_already_downloaded',
'report_warning',
'to_console_title',
'to_stderr',
'trouble',
'write_debug',
):
setattr(self, func, getattr(ydl, func))
def to_screen(self, *args, **kargs):
self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
@staticmethod @staticmethod
def format_seconds(seconds): def format_seconds(seconds):
time = timetuple_from_msec(seconds * 1000) time = timetuple_from_msec(seconds * 1000)
@ -157,27 +177,6 @@ class FileDownloader:
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
return int(round(number * multiplier)) return int(round(number * multiplier))
def to_screen(self, *args, **kargs):
self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs)
def to_stderr(self, message):
self.ydl.to_stderr(message)
def to_console_title(self, message):
self.ydl.to_console_title(message)
def trouble(self, *args, **kargs):
self.ydl.trouble(*args, **kargs)
def report_warning(self, *args, **kargs):
self.ydl.report_warning(*args, **kargs)
def report_error(self, *args, **kargs):
self.ydl.report_error(*args, **kargs)
def write_debug(self, *args, **kargs):
self.ydl.write_debug(*args, **kargs)
def slow_down(self, start_time, now, byte_counter): def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit.""" """Sleep if the download speed is over the rate limit."""
rate_limit = self.params.get('ratelimit') rate_limit = self.params.get('ratelimit')
@ -263,10 +262,8 @@ class FileDownloader:
# Ignore obviously invalid dates # Ignore obviously invalid dates
if filetime == 0: if filetime == 0:
return return
try: with contextlib.suppress(Exception):
os.utime(filename, (time.time(), filetime)) os.utime(filename, (time.time(), filetime))
except Exception:
pass
return filetime return filetime
def report_destination(self, filename): def report_destination(self, filename):
@ -287,18 +284,18 @@ class FileDownloader:
def _finish_multiline_status(self): def _finish_multiline_status(self):
self._multiline.end() self._multiline.end()
_progress_styles = { ProgressStyles = Namespace(
'downloaded_bytes': 'light blue', downloaded_bytes='light blue',
'percent': 'light blue', percent='light blue',
'eta': 'yellow', eta='yellow',
'speed': 'green', speed='green',
'elapsed': 'bold white', elapsed='bold white',
'total_bytes': '', total_bytes='',
'total_bytes_estimate': '', total_bytes_estimate='',
} )
def _report_progress_status(self, s, default_template): def _report_progress_status(self, s, default_template):
for name, style in self._progress_styles.items(): for name, style in self.ProgressStyles._asdict().items():
name = f'_{name}_str' name = f'_{name}_str'
if name not in s: if name not in s:
continue continue
@ -391,10 +388,6 @@ class FileDownloader:
'[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...' '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
% (error_to_compat_str(err), count, self.format_retries(retries))) % (error_to_compat_str(err), count, self.format_retries(retries)))
def report_file_already_downloaded(self, *args, **kwargs):
"""Report file has already been fully downloaded."""
return self.ydl.report_file_already_downloaded(*args, **kwargs)
def report_unable_to_resume(self): def report_unable_to_resume(self):
"""Report it was impossible to resume download.""" """Report it was impossible to resume download."""
self.to_screen('[download] Unable to resume') self.to_screen('[download] Unable to resume')
@ -433,25 +426,16 @@ class FileDownloader:
self._finish_multiline_status() self._finish_multiline_status()
return True, False return True, False
if subtitle is False: if subtitle:
min_sleep_interval = self.params.get('sleep_interval') sleep_interval = self.params.get('sleep_interval_subtitles') or 0
if min_sleep_interval:
max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
self.to_screen(
'[download] Sleeping %s seconds ...' % (
int(sleep_interval) if sleep_interval.is_integer()
else '%.2f' % sleep_interval))
time.sleep(sleep_interval)
else: else:
sleep_interval_sub = 0 min_sleep_interval = self.params.get('sleep_interval') or 0
if type(self.params.get('sleep_interval_subtitles')) is int: sleep_interval = random.uniform(
sleep_interval_sub = self.params.get('sleep_interval_subtitles') min_sleep_interval, self.params.get('max_sleep_interval', min_sleep_interval))
if sleep_interval_sub > 0: if sleep_interval > 0:
self.to_screen( self.to_screen(f'[download] Sleeping {sleep_interval:.2f} seconds ...')
'[download] Sleeping %s seconds ...' % ( time.sleep(sleep_interval)
sleep_interval_sub))
time.sleep(sleep_interval_sub)
ret = self.real_download(filename, info_dict) ret = self.real_download(filename, info_dict)
self._finish_multiline_status() self._finish_multiline_status()
return ret, True return ret, True

8
yt_dlp/downloader/fragment.py
View File

@ -1,3 +1,4 @@
import contextlib
import http.client import http.client
import json import json
import math import math
@ -310,10 +311,8 @@ class FragmentFD(FileDownloader):
if self.params.get('updatetime', True): if self.params.get('updatetime', True):
filetime = ctx.get('fragment_filetime') filetime = ctx.get('fragment_filetime')
if filetime: if filetime:
try: with contextlib.suppress(Exception):
os.utime(ctx['filename'], (time.time(), filetime)) os.utime(ctx['filename'], (time.time(), filetime))
except Exception:
pass
downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename'])) downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
self._hook_progress({ self._hook_progress({
@ -523,7 +522,8 @@ class FragmentFD(FileDownloader):
break break
try: try:
download_fragment(fragment, ctx) download_fragment(fragment, ctx)
result = append_fragment(decrypt_fragment(fragment, self._read_fragment(ctx)), fragment['frag_index'], ctx) result = append_fragment(
decrypt_fragment(fragment, self._read_fragment(ctx)), fragment['frag_index'], ctx)
except KeyboardInterrupt: except KeyboardInterrupt:
if info_dict.get('is_live'): if info_dict.get('is_live'):
break break

5
yt_dlp/downloader/websocket.py
View File

@ -1,4 +1,5 @@
import asyncio import asyncio
import contextlib
import os import os
import signal import signal
import threading import threading
@ -29,11 +30,9 @@ class FFmpegSinkFD(FileDownloader):
except (BrokenPipeError, OSError): except (BrokenPipeError, OSError):
pass pass
finally: finally:
try: with contextlib.suppress(OSError):
stdin.flush() stdin.flush()
stdin.close() stdin.close()
except OSError:
pass
os.kill(os.getpid(), signal.SIGINT) os.kill(os.getpid(), signal.SIGINT)
class FFmpegStdinFD(FFmpegFD): class FFmpegStdinFD(FFmpegFD):

13
yt_dlp/extractor/__init__.py
View File

@ -1,24 +1,23 @@
import contextlib
import os import os
from ..utils import load_plugins from ..utils import load_plugins
_LAZY_LOADER = False _LAZY_LOADER = False
if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'): if not os.environ.get('YTDLP_NO_LAZY_EXTRACTORS'):
try: with contextlib.suppress(ImportError):
from .lazy_extractors import * from .lazy_extractors import * # noqa: F403
from .lazy_extractors import _ALL_CLASSES from .lazy_extractors import _ALL_CLASSES
_LAZY_LOADER = True _LAZY_LOADER = True
except ImportError:
pass
if not _LAZY_LOADER: if not _LAZY_LOADER:
from .extractors import * from .extractors import * # noqa: F403
_ALL_CLASSES = [ _ALL_CLASSES = [ # noqa: F811
klass klass
for name, klass in globals().items() for name, klass in globals().items()
if name.endswith('IE') and name != 'GenericIE' if name.endswith('IE') and name != 'GenericIE'
] ]
_ALL_CLASSES.append(GenericIE) _ALL_CLASSES.append(GenericIE) # noqa: F405
_PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals()) _PLUGIN_CLASSES = load_plugins('extractor', 'IE', globals())
_ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES _ALL_CLASSES = list(_PLUGIN_CLASSES.values()) + _ALL_CLASSES

7
yt_dlp/extractor/cpac.py
View File

@ -9,13 +9,6 @@ from ..utils import (
urljoin, urljoin,
) )
# compat_range
try:
if callable(xrange):
range = xrange
except (NameError, TypeError):
pass
class CPACIE(InfoExtractor): class CPACIE(InfoExtractor):
IE_NAME = 'cpac' IE_NAME = 'cpac'

3
yt_dlp/extractor/extractors.py
View File

@ -1,4 +1,5 @@
# flake8: noqa # flake8: noqa: F401
from .abc import ( from .abc import (
ABCIE, ABCIE,
ABCIViewIE, ABCIViewIE,

9
yt_dlp/extractor/openload.py
View File

@ -1,3 +1,4 @@
import contextlib
import json import json
import os import os
import subprocess import subprocess
@ -31,13 +32,11 @@ def cookie_to_dict(cookie):
cookie_dict['secure'] = cookie.secure cookie_dict['secure'] = cookie.secure
if cookie.discard is not None: if cookie.discard is not None:
cookie_dict['discard'] = cookie.discard cookie_dict['discard'] = cookie.discard
try: with contextlib.suppress(TypeError):
if (cookie.has_nonstandard_attr('httpOnly') if (cookie.has_nonstandard_attr('httpOnly')
or cookie.has_nonstandard_attr('httponly') or cookie.has_nonstandard_attr('httponly')
or cookie.has_nonstandard_attr('HttpOnly')): or cookie.has_nonstandard_attr('HttpOnly')):
cookie_dict['httponly'] = True cookie_dict['httponly'] = True
except TypeError:
pass
return cookie_dict return cookie_dict
@ -129,10 +128,8 @@ class PhantomJSwrapper:
def __del__(self): def __del__(self):
for name in self._TMP_FILE_NAMES: for name in self._TMP_FILE_NAMES:
try: with contextlib.suppress(OSError, KeyError):
os.remove(self._TMP_FILES[name].name) os.remove(self._TMP_FILES[name].name)
except (OSError, KeyError):
pass
def _save_cookies(self, url): def _save_cookies(self, url):
cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar) cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)

7
yt_dlp/extractor/rtve.py
View File

@ -1,6 +1,5 @@
import base64 import base64
import io import io
import sys
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
@ -17,8 +16,6 @@ from ..utils import (
try_get, try_get,
) )
_bytes_to_chr = (lambda x: x) if sys.version_info[0] == 2 else (lambda x: map(chr, x))
class RTVEALaCartaIE(InfoExtractor): class RTVEALaCartaIE(InfoExtractor):
IE_NAME = 'rtve.es:alacarta' IE_NAME = 'rtve.es:alacarta'
@ -87,7 +84,7 @@ class RTVEALaCartaIE(InfoExtractor):
alphabet = [] alphabet = []
e = 0 e = 0
d = 0 d = 0
for l in _bytes_to_chr(alphabet_data): for l in alphabet_data.decode('iso-8859-1'):
if d == 0: if d == 0:
alphabet.append(l) alphabet.append(l)
d = e = (e + 1) % 4 d = e = (e + 1) % 4
@ -97,7 +94,7 @@ class RTVEALaCartaIE(InfoExtractor):
f = 0 f = 0
e = 3 e = 3
b = 1 b = 1
for letter in _bytes_to_chr(url_data): for letter in url_data.decode('iso-8859-1'):
if f == 0: if f == 0:
l = int(letter) * 10 l = int(letter) * 10
f = 1 f = 1

2
yt_dlp/extractor/spotify.py
View File

@ -102,6 +102,7 @@ class SpotifyBaseIE(InfoExtractor):
class SpotifyIE(SpotifyBaseIE): class SpotifyIE(SpotifyBaseIE):
IE_NAME = 'spotify' IE_NAME = 'spotify'
IE_DESC = 'Spotify episodes'
_VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'episode' _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'episode'
_TESTS = [{ _TESTS = [{
'url': 'https://open.spotify.com/episode/4Z7GAJ50bgctf6uclHlWKo', 'url': 'https://open.spotify.com/episode/4Z7GAJ50bgctf6uclHlWKo',
@ -131,6 +132,7 @@ class SpotifyIE(SpotifyBaseIE):
class SpotifyShowIE(SpotifyBaseIE): class SpotifyShowIE(SpotifyBaseIE):
IE_NAME = 'spotify:show' IE_NAME = 'spotify:show'
IE_DESC = 'Spotify shows'
_VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'show' _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'show'
_TEST = { _TEST = {
'url': 'https://open.spotify.com/show/4PM9Ke6l66IRNpottHKV9M', 'url': 'https://open.spotify.com/show/4PM9Ke6l66IRNpottHKV9M',

10
yt_dlp/extractor/youtube.py
View File

@ -3586,17 +3586,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
headers=self.generate_api_headers(ytcfg=master_ytcfg), headers=self.generate_api_headers(ytcfg=master_ytcfg),
note='Downloading initial data API JSON') note='Downloading initial data API JSON')
try: try: # This will error if there is no livechat
# This will error if there is no livechat
initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation'] initial_data['contents']['twoColumnWatchNextResults']['conversationBar']['liveChatRenderer']['continuations'][0]['reloadContinuationData']['continuation']
except (KeyError, IndexError, TypeError):
pass
else:
info.setdefault('subtitles', {})['live_chat'] = [{ info.setdefault('subtitles', {})['live_chat'] = [{
'url': 'https://www.youtube.com/watch?v=%s' % video_id, # url is needed to set cookies 'url': f'https://www.youtube.com/watch?v={video_id}', # url is needed to set cookies
'video_id': video_id, 'video_id': video_id,
'ext': 'json', 'ext': 'json',
'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay', 'protocol': 'youtube_live_chat' if is_live or is_upcoming else 'youtube_live_chat_replay',
}] }]
except (KeyError, IndexError, TypeError):
pass
if initial_data: if initial_data:
info['chapters'] = ( info['chapters'] = (

41
yt_dlp/jsinterp.py
View File

@ -1,7 +1,8 @@
import collections
import contextlib
import json import json
import operator import operator
import re import re
from collections.abc import MutableMapping
from .utils import ExtractorError, remove_quotes from .utils import ExtractorError, remove_quotes
@ -35,38 +36,17 @@ class JS_Continue(ExtractorError):
ExtractorError.__init__(self, 'Invalid continue') ExtractorError.__init__(self, 'Invalid continue')
class LocalNameSpace(MutableMapping): class LocalNameSpace(collections.ChainMap):
def __init__(self, *stack):
self.stack = tuple(stack)
def __getitem__(self, key):
for scope in self.stack:
if key in scope:
return scope[key]
raise KeyError(key)
def __setitem__(self, key, value): def __setitem__(self, key, value):
for scope in self.stack: for scope in self.maps:
if key in scope: if key in scope:
scope[key] = value scope[key] = value
break return
else: self.maps[0][key] = value
self.stack[0][key] = value
return value
def __delitem__(self, key): def __delitem__(self, key):
raise NotImplementedError('Deleting is not supported') raise NotImplementedError('Deleting is not supported')
def __iter__(self):
for scope in self.stack:
yield from scope
def __len__(self, key):
return len(iter(self))
def __repr__(self):
return f'LocalNameSpace{self.stack}'
class JSInterpreter: class JSInterpreter:
def __init__(self, code, objects=None): def __init__(self, code, objects=None):
@ -302,10 +282,8 @@ class JSInterpreter:
if var_m: if var_m:
return local_vars[var_m.group('name')] return local_vars[var_m.group('name')]
try: with contextlib.suppress(ValueError):
return json.loads(expr) return json.loads(expr)
except ValueError:
pass
m = re.match( m = re.match(
r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr) r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
@ -521,14 +499,13 @@ class JSInterpreter:
def build_function(self, argnames, code, *global_stack): def build_function(self, argnames, code, *global_stack):
global_stack = list(global_stack) or [{}] global_stack = list(global_stack) or [{}]
local_vars = global_stack.pop(0)
def resf(args, **kwargs): def resf(args, **kwargs):
local_vars.update({ global_stack[0].update({
**dict(zip(argnames, args)), **dict(zip(argnames, args)),
**kwargs **kwargs
}) })
var_stack = LocalNameSpace(local_vars, *global_stack) var_stack = LocalNameSpace(*global_stack)
for stmt in self._separate(code.replace('\n', ''), ';'): for stmt in self._separate(code.replace('\n', ''), ';'):
ret, should_abort = self.interpret_statement(stmt, var_stack) ret, should_abort = self.interpret_statement(stmt, var_stack)
if should_abort: if should_abort:

72
yt_dlp/options.py
View File

@ -21,6 +21,7 @@ from .utils import (
Config, Config,
expand_path, expand_path,
get_executable_path, get_executable_path,
join_nonempty,
remove_end, remove_end,
write_string, write_string,
) )
@ -109,9 +110,43 @@ def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
return parser, opts, args return parser, opts, args
class _YoutubeDLHelpFormatter(optparse.IndentedHelpFormatter):
def __init__(self):
# No need to wrap help messages if we're on a wide console
max_width = compat_get_terminal_size().columns or 80
# 47% is chosen because that is how README.md is currently formatted
# and moving help text even further to the right is undesirable.
# This can be reduced in the future to get a prettier output
super().__init__(width=max_width, max_help_position=int(0.47 * max_width))
@staticmethod
def format_option_strings(option):
""" ('-o', '--option') -> -o, --format METAVAR """
opts = join_nonempty(
option._short_opts and option._short_opts[0],
option._long_opts and option._long_opts[0],
delim=', ')
if option.takes_value():
opts += f' {option.metavar}'
return opts
class _YoutubeDLOptionParser(optparse.OptionParser): class _YoutubeDLOptionParser(optparse.OptionParser):
# optparse is deprecated since python 3.2. So assume a stable interface even for private methods # optparse is deprecated since python 3.2. So assume a stable interface even for private methods
def __init__(self):
super().__init__(
prog='yt-dlp',
version=__version__,
usage='%prog [OPTIONS] URL [URL...]',
epilog='See full documentation at https://github.com/yt-dlp/yt-dlp#readme',
formatter=_YoutubeDLHelpFormatter(),
conflict_handler='resolve',
)
def _get_args(self, args):
return sys.argv[1:] if args is None else list(args)
def _match_long_opt(self, opt): def _match_long_opt(self, opt):
"""Improve ambigious argument resolution by comparing option objects instead of argument strings""" """Improve ambigious argument resolution by comparing option objects instead of argument strings"""
try: try:
@ -123,23 +158,6 @@ class _YoutubeDLOptionParser(optparse.OptionParser):
def create_parser(): def create_parser():
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
opts = []
if option._short_opts:
opts.append(option._short_opts[0])
if option._long_opts:
opts.append(option._long_opts[0])
if len(opts) > 1:
opts.insert(1, ', ')
if option.takes_value():
opts.append(' %s' % option.metavar)
return ''.join(opts)
def _list_from_options_callback(option, opt_str, value, parser, append=True, delim=',', process=str.strip): def _list_from_options_callback(option, opt_str, value, parser, append=True, delim=',', process=str.strip):
# append can be True, False or -1 (prepend) # append can be True, False or -1 (prepend)
current = list(getattr(parser.values, option.dest)) if append else [] current = list(getattr(parser.values, option.dest)) if append else []
@ -204,23 +222,7 @@ def create_parser():
out_dict[key] = out_dict.get(key, []) + [val] if append else val out_dict[key] = out_dict.get(key, []) + [val] if append else val
setattr(parser.values, option.dest, out_dict) setattr(parser.values, option.dest, out_dict)
# No need to wrap help messages if we're on a wide console parser = _YoutubeDLOptionParser()
columns = compat_get_terminal_size().columns
max_width = columns if columns else 80
# 47% is chosen because that is how README.md is currently formatted
# and moving help text even further to the right is undesirable.
# This can be reduced in the future to get a prettier output
max_help_position = int(0.47 * max_width)
fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
fmt.format_option_strings = _format_option_string
parser = _YoutubeDLOptionParser(
version=__version__,
formatter=fmt,
usage='%prog [OPTIONS] URL [URL...]',
conflict_handler='resolve'
)
general = optparse.OptionGroup(parser, 'General Options') general = optparse.OptionGroup(parser, 'General Options')
general.add_option( general.add_option(
@ -1048,7 +1050,7 @@ def create_parser():
verbosity.add_option( verbosity.add_option(
'-C', '--call-home', '-C', '--call-home',
dest='call_home', action='store_true', default=False, dest='call_home', action='store_true', default=False,
# help='[Broken] Contact the yt-dlp server for debugging') # help='Contact the yt-dlp server for debugging')
help=optparse.SUPPRESS_HELP) help=optparse.SUPPRESS_HELP)
verbosity.add_option( verbosity.add_option(
'--no-call-home', '--no-call-home',

2
yt_dlp/postprocessor/common.py
View File

@ -69,8 +69,8 @@ class PostProcessor(metaclass=PostProcessorMetaClass):
return name[6:] if name[:6].lower() == 'ffmpeg' else name return name[6:] if name[:6].lower() == 'ffmpeg' else name
def to_screen(self, text, prefix=True, *args, **kwargs): def to_screen(self, text, prefix=True, *args, **kwargs):
tag = '[%s] ' % self.PP_NAME if prefix else ''
if self._downloader: if self._downloader:
tag = '[%s] ' % self.PP_NAME if prefix else ''
return self._downloader.to_screen(f'{tag}{text}', *args, **kwargs) return self._downloader.to_screen(f'{tag}{text}', *args, **kwargs)
def report_warning(self, text, *args, **kwargs): def report_warning(self, text, *args, **kwargs):

22
yt_dlp/postprocessor/metadataparser.py
View File

@ -1,29 +1,25 @@
import re import re
from enum import Enum
from .common import PostProcessor from .common import PostProcessor
from ..utils import Namespace
class MetadataParserPP(PostProcessor): class MetadataParserPP(PostProcessor):
class Actions(Enum):
INTERPRET = 'interpretter'
REPLACE = 'replacer'
def __init__(self, downloader, actions): def __init__(self, downloader, actions):
PostProcessor.__init__(self, downloader) super().__init__(self, downloader)
self._actions = [] self._actions = []
for f in actions: for f in actions:
action = f[0] action, *args = f
assert isinstance(action, self.Actions) assert action in self.Actions
self._actions.append(getattr(self, action.value)(*f[1:])) self._actions.append(action(*args))
@classmethod @classmethod
def validate_action(cls, action, *data): def validate_action(cls, action, *data):
''' Each action can be: """Each action can be:
(Actions.INTERPRET, from, to) OR (Actions.INTERPRET, from, to) OR
(Actions.REPLACE, field, search, replace) (Actions.REPLACE, field, search, replace)
''' """
if not isinstance(action, cls.Actions): if action not in cls.Actions:
raise ValueError(f'{action!r} is not a valid action') raise ValueError(f'{action!r} is not a valid action')
getattr(cls, action.value)(cls, *data) # So this can raise error to validate getattr(cls, action.value)(cls, *data) # So this can raise error to validate
@ -99,6 +95,8 @@ class MetadataParserPP(PostProcessor):
search_re = re.compile(search) search_re = re.compile(search)
return f return f
Actions = Namespace(INTERPRET=interpretter, REPLACE=replacer)
class MetadataFromFieldPP(MetadataParserPP): class MetadataFromFieldPP(MetadataParserPP):
@classmethod @classmethod

103
yt_dlp/utils.py
View File

@ -70,6 +70,7 @@ from .socks import ProxyType, sockssocket
try: try:
import certifi import certifi
# The certificate may not be bundled in executable # The certificate may not be bundled in executable
has_certifi = os.path.exists(certifi.where()) has_certifi = os.path.exists(certifi.where())
except ImportError: except ImportError:
@ -282,22 +283,16 @@ def write_json_file(obj, fn):
if sys.platform == 'win32': if sys.platform == 'win32':
# Need to remove existing file on Windows, else os.rename raises # Need to remove existing file on Windows, else os.rename raises
# WindowsError or FileExistsError. # WindowsError or FileExistsError.
try: with contextlib.suppress(OSError):
os.unlink(fn) os.unlink(fn)
except OSError: with contextlib.suppress(OSError):
pass
try:
mask = os.umask(0) mask = os.umask(0)
os.umask(mask) os.umask(mask)
os.chmod(tf.name, 0o666 & ~mask) os.chmod(tf.name, 0o666 & ~mask)
except OSError:
pass
os.rename(tf.name, fn) os.rename(tf.name, fn)
except Exception: except Exception:
try: with contextlib.suppress(OSError):
os.remove(tf.name) os.remove(tf.name)
except OSError:
pass
raise raise
@ -575,12 +570,9 @@ def extract_attributes(html_element):
}. }.
""" """
parser = HTMLAttributeParser() parser = HTMLAttributeParser()
try: with contextlib.suppress(compat_HTMLParseError):
parser.feed(html_element) parser.feed(html_element)
parser.close() parser.close()
# Older Python may throw HTMLParseError in case of malformed HTML
except compat_HTMLParseError:
pass
return parser.attrs return parser.attrs
@ -800,10 +792,8 @@ def _htmlentity_transform(entity_with_semicolon):
else: else:
base = 10 base = 10
# See https://github.com/ytdl-org/youtube-dl/issues/7518 # See https://github.com/ytdl-org/youtube-dl/issues/7518
try: with contextlib.suppress(ValueError):
return compat_chr(int(numstr, base)) return compat_chr(int(numstr, base))
except ValueError:
pass
# Unknown entity in name, return its literal representation # Unknown entity in name, return its literal representation
return '&%s;' % entity return '&%s;' % entity
@ -812,7 +802,7 @@ def _htmlentity_transform(entity_with_semicolon):
def unescapeHTML(s): def unescapeHTML(s):
if s is None: if s is None:
return None return None
assert type(s) == compat_str assert isinstance(s, str)
return re.sub( return re.sub(
r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s) r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
@ -865,7 +855,7 @@ def get_subprocess_encoding():
def encodeFilename(s, for_subprocess=False): def encodeFilename(s, for_subprocess=False):
assert type(s) == str assert isinstance(s, str)
return s return s
@ -924,10 +914,8 @@ def _ssl_load_windows_store_certs(ssl_context, storename):
except PermissionError: except PermissionError:
return return
for cert in certs: for cert in certs:
try: with contextlib.suppress(ssl.SSLError):
ssl_context.load_verify_locations(cadata=cert) ssl_context.load_verify_locations(cadata=cert)
except ssl.SSLError:
pass
def make_HTTPS_handler(params, **kwargs): def make_HTTPS_handler(params, **kwargs):
@ -1391,7 +1379,7 @@ def make_socks_conn_class(base_class, socks_proxy):
def connect(self): def connect(self):
self.sock = sockssocket() self.sock = sockssocket()
self.sock.setproxy(*proxy_args) self.sock.setproxy(*proxy_args)
if type(self.timeout) in (int, float): if isinstance(self.timeout, (int, float)):
self.sock.settimeout(self.timeout) self.sock.settimeout(self.timeout)
self.sock.connect((self.host, self.port)) self.sock.connect((self.host, self.port))
@ -1526,9 +1514,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
try: try:
cf.write(prepare_line(line)) cf.write(prepare_line(line))
except compat_cookiejar.LoadError as e: except compat_cookiejar.LoadError as e:
write_string( write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
'WARNING: skipping cookie file entry due to %s: %r\n'
% (e, line), sys.stderr)
continue continue
cf.seek(0) cf.seek(0)
self._really_load(cf, filename, ignore_discard, ignore_expires) self._really_load(cf, filename, ignore_discard, ignore_expires)
@ -1646,12 +1632,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
if timezone is None: if timezone is None:
timezone, date_str = extract_timezone(date_str) timezone, date_str = extract_timezone(date_str)
try: with contextlib.suppress(ValueError):
date_format = f'%Y-%m-%d{delimiter}%H:%M:%S' date_format = f'%Y-%m-%d{delimiter}%H:%M:%S'
dt = datetime.datetime.strptime(date_str, date_format) - timezone dt = datetime.datetime.strptime(date_str, date_format) - timezone
return calendar.timegm(dt.timetuple()) return calendar.timegm(dt.timetuple())
except ValueError:
pass
def date_formats(day_first=True): def date_formats(day_first=True):
@ -1671,17 +1655,13 @@ def unified_strdate(date_str, day_first=True):
_, date_str = extract_timezone(date_str) _, date_str = extract_timezone(date_str)
for expression in date_formats(day_first): for expression in date_formats(day_first):
try: with contextlib.suppress(ValueError):
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d') upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
except ValueError:
pass
if upload_date is None: if upload_date is None:
timetuple = email.utils.parsedate_tz(date_str) timetuple = email.utils.parsedate_tz(date_str)
if timetuple: if timetuple:
try: with contextlib.suppress(ValueError):
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d') upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
except ValueError:
pass
if upload_date is not None: if upload_date is not None:
return compat_str(upload_date) return compat_str(upload_date)
@ -1709,11 +1689,9 @@ def unified_timestamp(date_str, day_first=True):
date_str = m.group(1) date_str = m.group(1)
for expression in date_formats(day_first): for expression in date_formats(day_first):
try: with contextlib.suppress(ValueError):
dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta) dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
return calendar.timegm(dt.timetuple()) return calendar.timegm(dt.timetuple())
except ValueError:
pass
timetuple = email.utils.parsedate_tz(date_str) timetuple = email.utils.parsedate_tz(date_str)
if timetuple: if timetuple:
return calendar.timegm(timetuple) + pm_delta * 3600 return calendar.timegm(timetuple) + pm_delta * 3600
@ -1879,9 +1857,8 @@ def get_windows_version():
def write_string(s, out=None, encoding=None): def write_string(s, out=None, encoding=None):
if out is None: assert isinstance(s, str)
out = sys.stderr out = out or sys.stderr
assert type(s) == compat_str
if 'b' in getattr(out, 'mode', ''): if 'b' in getattr(out, 'mode', ''):
byt = s.encode(encoding or preferredencoding(), 'ignore') byt = s.encode(encoding or preferredencoding(), 'ignore')
@ -2483,18 +2460,10 @@ def parse_duration(s):
else: else:
return None return None
duration = 0
if secs:
duration += float(secs)
if mins:
duration += float(mins) * 60
if hours:
duration += float(hours) * 60 * 60
if days:
duration += float(days) * 24 * 60 * 60
if ms: if ms:
duration += float(ms.replace(':', '.')) ms = ms.replace(':', '.')
return duration return sum(float(part or 0) * mult for part, mult in (
(days, 86400), (hours, 3600), (mins, 60), (secs, 1), (ms, 1)))
def prepend_extension(filename, ext, expected_real_ext=None): def prepend_extension(filename, ext, expected_real_ext=None):
@ -2957,9 +2926,10 @@ TV_PARENTAL_GUIDELINES = {
def parse_age_limit(s): def parse_age_limit(s):
if type(s) == int: # isinstance(False, int) is True. So type() must be used instead
if type(s) is int:
return s if 0 <= s <= 21 else None return s if 0 <= s <= 21 else None
if not isinstance(s, str): elif not isinstance(s, str):
return None return None
m = re.match(r'^(?P<age>\d{1,2})\+?$', s) m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
if m: if m:
@ -3227,7 +3197,7 @@ def parse_codecs(codecs_str):
if not tcodec: if not tcodec:
tcodec = full_codec tcodec = full_codec
else: else:
write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr) write_string(f'WARNING: Unknown codec {full_codec}\n')
if vcodec or acodec or tcodec: if vcodec or acodec or tcodec:
return { return {
'vcodec': vcodec or 'none', 'vcodec': vcodec or 'none',
@ -4934,7 +4904,7 @@ def get_executable_path():
def load_plugins(name, suffix, namespace): def load_plugins(name, suffix, namespace):
classes = {} classes = {}
try: with contextlib.suppress(FileNotFoundError):
plugins_spec = importlib.util.spec_from_file_location( plugins_spec = importlib.util.spec_from_file_location(
name, os.path.join(get_executable_path(), 'ytdlp_plugins', name, '__init__.py')) name, os.path.join(get_executable_path(), 'ytdlp_plugins', name, '__init__.py'))
plugins = importlib.util.module_from_spec(plugins_spec) plugins = importlib.util.module_from_spec(plugins_spec)
@ -4947,8 +4917,6 @@ def load_plugins(name, suffix, namespace):
continue continue
klass = getattr(plugins, name) klass = getattr(plugins, name)
classes[name] = namespace[name] = klass classes[name] = namespace[name] = klass
except FileNotFoundError:
pass
return classes return classes
@ -4957,13 +4925,14 @@ def traverse_obj(
casesense=True, is_user_input=False, traverse_string=False): casesense=True, is_user_input=False, traverse_string=False):
''' Traverse nested list/dict/tuple ''' Traverse nested list/dict/tuple
@param path_list A list of paths which are checked one by one. @param path_list A list of paths which are checked one by one.
Each path is a list of keys where each key is a string, Each path is a list of keys where each key is a:
a function, a tuple of strings/None or "...". - None: Do nothing
When a fuction is given, it takes the key and value as arguments - string: A dictionary key
and returns whether the key matches or not. When a tuple is given, - int: An index into a list
all the keys given in the tuple are traversed, and - tuple: A list of keys all of which will be traversed
"..." traverses all the keys in the object - Ellipsis: Fetch all values in the object
"None" returns the object without traversal - Function: Takes the key and value as arguments
and returns whether the key matches or not
@param default Default value to return @param default Default value to return
@param expected_type Only accept final value of this type (Can also be any callable) @param expected_type Only accept final value of this type (Can also be any callable)
@param get_all Return all the values obtained from a path or only the first one @param get_all Return all the values obtained from a path or only the first one
@ -5253,7 +5222,7 @@ class Config:
yield from self.own_args or [] yield from self.own_args or []
def parse_args(self): def parse_args(self):
return self._parser.parse_args(list(self.all_args)) return self._parser.parse_args(self.all_args)
class WebSocketsWrapper(): class WebSocketsWrapper():
@ -5339,3 +5308,7 @@ class classproperty:
def __get__(self, _, cls): def __get__(self, _, cls):
return self.f(cls) return self.f(cls)
def Namespace(**kwargs):
return collections.namedtuple('Namespace', kwargs)(**kwargs)

10
yt_dlp/webvtt.py
View File

@ -103,14 +103,8 @@ def _parse_ts(ts):
Convert a parsed WebVTT timestamp (a re.Match obtained from _REGEX_TS) Convert a parsed WebVTT timestamp (a re.Match obtained from _REGEX_TS)
into an MPEG PES timestamp: a tick counter at 90 kHz resolution. into an MPEG PES timestamp: a tick counter at 90 kHz resolution.
""" """
return 90 * sum(
h, min, s, ms = ts.groups() int(part or 0) * mult for part, mult in zip(ts.groups(), (3600_000, 60_000, 1000, 1)))
return 90 * (
int(h or 0) * 3600000 + # noqa: W504,E221,E222
int(min) * 60000 + # noqa: W504,E221,E222
int(s) * 1000 + # noqa: W504,E221,E222
int(ms) # noqa: W504,E221,E222
)
def _format_ts(ts): def _format_ts(ts):