[devscripts] Create `utils` and refactor

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -2,6 +2,13 @@ name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -2,6 +2,13 @@ name: Site support request
 description: Request support for a new site
 labels: [triage, site-request]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -2,6 +2,13 @@ name: Site feature request
 description: Request a new functionality for a supported site
 labels: [triage, site-enhancement]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -2,6 +2,13 @@ name: Bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -2,6 +2,13 @@ name: Feature request
 description: Request a new functionality unrelated to any particular site or extractor
 labels: [triage, enhancement]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE/6_question.yml

@@ -2,12 +2,19 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\* field
+          required: true
   - type: markdown
     attributes:
       value: |
         ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
         If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt whether this is the right template, **use another template**!
+        If you are in doubt whether this is the right template, **USE ANOTHER TEMPLATE**!
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -2,6 +2,7 @@ name: Broken site
 description: Report broken or misfunctioning site
 labels: [triage, site-bug]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -2,6 +2,7 @@ name: Site support request
 description: Request support for a new site
 labels: [triage, site-request]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -2,6 +2,7 @@ name: Site feature request
 description: Request a new functionality for a supported site
 labels: [triage, site-enhancement]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -2,6 +2,7 @@ name: Bug report
 description: Report a bug unrelated to any particular site or extractor
 labels: [triage, bug]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -2,6 +2,7 @@ name: Feature request
 description: Request a new functionality unrelated to any particular site or extractor
 labels: [triage, enhancement]
 body:
+  %(no_skip)s
   - type: checkboxes
     id: checklist
     attributes:

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -2,12 +2,13 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  %(no_skip)s
   - type: markdown
     attributes:
       value: |
         ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
         If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt whether this is the right template, **use another template**!
+        If you are in doubt whether this is the right template, **USE ANOTHER TEMPLATE**!
   - type: checkboxes
     id: checklist
     attributes:

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,5 @@
+**IMPORTANT**: PRs without the template will be CLOSED
+
 ### Description of your *pull request* and other information
 
 </details>

README.md

@@ -312,7 +312,7 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
 ## COMPILE
 
 ### Standalone PyInstaller Builds
-To build the Windows/MacOS executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (32/64 bit) as the Python used.
+To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). Once you have all the necessary dependencies installed, simply run `pyinst.py`. The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used.
 
     python3 -m pip install -U pyinstaller -r requirements.txt
     python3 devscripts/make_lazy_extractors.py

devscripts/make_issue_template.py

@@ -7,20 +7,14 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
 import re
 
-
+from devscripts.utils import (
-def read(fname):
+    get_filename_args,
-    with open(fname, encoding='utf-8') as f:
+    read_file,
-        return f.read()
+    read_version,
-
+    write_file,
-
+)
-# Get the version without importing the package
-def read_version(fname):
-    exec(compile(read(fname), fname, 'exec'))
-    return locals()['__version__']
-
 
 VERBOSE_TMPL = '''
   - type: checkboxes
@@ -58,20 +52,24 @@ VERBOSE_TMPL = '''
       required: true
 '''.strip()
 
+NO_SKIP = '''
+  - type: checkboxes
+    attributes:
+      label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE
+      description: Fill all fields even if you think it is irrelevant for the issue
+      options:
+        - label: I understand that I will be **blocked** if I remove or skip any mandatory\\* field
+          required: true
+'''.strip()
+
 
 def main():
-    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
+    fields = {'version': read_version(), 'no_skip': NO_SKIP}
-    _, args = parser.parse_args()
-    if len(args) != 2:
-        parser.error('Expected an input and an output filename')
-
-    fields = {'version': read_version('yt_dlp/version.py')}
     fields['verbose'] = VERBOSE_TMPL % fields
     fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])
 
-    infile, outfile = args
+    infile, outfile = get_filename_args(has_infile=True)
-    with open(outfile, 'w', encoding='utf-8') as outf:
+    write_file(outfile, read_file(infile) % fields)
-        outf.write(read(infile) % fields)
 
 
 if __name__ == '__main__':

devscripts/make_lazy_extractors.py

@@ -7,9 +7,10 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
 from inspect import getsource
 
+from devscripts.utils import get_filename_args, read_file, write_file
+
 NO_ATTR = object()
 STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_NETRC_MACHINE', 'age_limit']
 CLASS_METHODS = [
@@ -19,17 +20,11 @@ IE_TEMPLATE = '''
 class {name}({bases}):
     _module = {module!r}
 '''
-with open('devscripts/lazy_load_template.py', encoding='utf-8') as f:
+MODULE_TEMPLATE = read_file('devscripts/lazy_load_template.py')
-    MODULE_TEMPLATE = f.read()
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog [OUTFILE.py]')
+    lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py')
-    args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py']
-    if len(args) != 1:
-        parser.error('Expected only an output filename')
-
-    lazy_extractors_filename = args[0]
     if os.path.exists(lazy_extractors_filename):
         os.remove(lazy_extractors_filename)
 
@@ -46,8 +41,7 @@ def main():
         *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor),
     ))
 
-    with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
+    write_file(lazy_extractors_filename, f'{module_src}\n')
-        f.write(f'{module_src}\n')
 
 
 def get_all_ies():

devscripts/make_readme.py

@@ -5,10 +5,17 @@ yt-dlp --help | make_readme.py
 This must be run in a console of correct width
 """
 
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 
 import functools
 import re
-import sys
+
+from devscripts.utils import read_file, write_file
 
 README_FILE = 'README.md'
 
@@ -63,11 +70,9 @@ PATCHES = (
     ),
 )
 
-with open(README_FILE, encoding='utf-8') as f:
+readme = read_file(README_FILE)
-    readme = f.read()
 
-with open(README_FILE, 'w', encoding='utf-8') as f:
+write_file(README_FILE, ''.join((
-    f.write(''.join((
     take_section(readme, end=f'## {OPTIONS_START}'),
     functools.reduce(apply_patch, PATCHES, options),
     take_section(readme, f'# {OPTIONS_END}'),

devscripts/make_supportedsites.py

@@ -7,21 +7,13 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
+from devscripts.utils import get_filename_args, write_file
-
 from yt_dlp.extractor import list_extractor_classes
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    _, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Expected an output filename')
-
     out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False)
-
+    write_file(get_filename_args(), f'# Supported sites\n{out}\n')
-    with open(args[0], 'w', encoding='utf-8') as outf:
-        outf.write(f'# Supported sites\n{out}\n')
 
 
 if __name__ == '__main__':

devscripts/prepare_manpage.py

@@ -1,9 +1,22 @@
 #!/usr/bin/env python3
 
-import optparse
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
 import os.path
 import re
 
+from devscripts.utils import (
+    compose_functions,
+    get_filename_args,
+    read_file,
+    write_file,
+)
+
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
 
@@ -22,25 +35,6 @@ yt\-dlp \- A youtube-dl fork with additional features and patches
 '''
 
 
-def main():
-    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    _, args = parser.parse_args()
-    if len(args) != 1:
-        parser.error('Expected an output filename')
-
-    outfile, = args
-
-    with open(README_FILE, encoding='utf-8') as f:
-        readme = f.read()
-
-    readme = filter_excluded_sections(readme)
-    readme = move_sections(readme)
-    readme = filter_options(readme)
-
-    with open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(PREFIX + readme)
-
-
 def filter_excluded_sections(readme):
     EXCLUDED_SECTION_BEGIN_STRING = re.escape('<!-- MANPAGE: BEGIN EXCLUDED SECTION -->')
     EXCLUDED_SECTION_END_STRING = re.escape('<!-- MANPAGE: END EXCLUDED SECTION -->')
@@ -92,5 +86,12 @@ def filter_options(readme):
     return readme.replace(section, options, 1)
 
 
+TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options)
+
+
+def main():
+    write_file(get_filename_args(), PREFIX + TRANSFORM(read_file(README_FILE)))
+
+
 if __name__ == '__main__':
     main()

devscripts/update-formulae.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python3
 
+"""
+Usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
+version can be either 0-aligned (yt-dlp version) or normalized (PyPi version)
+"""
+
 # Allow direct execution
 import os
 import sys
@@ -11,8 +16,7 @@ import json
 import re
 import urllib.request
 
-# usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
+from devscripts.utils import read_file, write_file
-# version can be either 0-aligned (yt-dlp version) or normalized (PyPl version)
 
 filename, version = sys.argv[1:]
 
@@ -27,11 +31,9 @@ tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.
 sha256sum = tarball_file['digests']['sha256']
 url = tarball_file['url']
 
-with open(filename) as r:
+formulae_text = read_file(filename)
-    formulae_text = r.read()
 
 formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1)
 formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1)
 
-with open(filename, 'w') as w:
+write_file(filename, formulae_text)
-    w.write(formulae_text)

devscripts/update-version.py

@@ -7,32 +7,35 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import contextlib
 import subprocess
 import sys
 from datetime import datetime
 
-with open('yt_dlp/version.py') as f:
+from devscripts.utils import read_version, write_file
-    exec(compile(f.read(), 'yt_dlp/version.py', 'exec'))
-old_version = locals()['__version__']
 
-old_version_list = old_version.split('.')
 
-old_ver = '.'.join(old_version_list[:3])
+def get_new_version(revision):
-old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
+    version = datetime.utcnow().strftime('%Y.%m.%d')
 
-ver = datetime.utcnow().strftime("%Y.%m.%d")
+    if revision:
+        assert revision.isdigit(), 'Revision must be a number'
+    else:
+        old_version = read_version().split('.')
+        if version.split('.') == old_version[:3]:
+            revision = str(int((old_version + [0])[3]) + 1)
 
-rev = (sys.argv[1:] or [''])[0]  # Use first argument, if present as revision number
+    return f'{version}.{revision}' if revision else version
-if not rev:
-    rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
 
-VERSION = '.'.join((ver, rev)) if rev else ver
 
-try:
+def get_git_head():
+    with contextlib.suppress(Exception):
         sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE)
-    GIT_HEAD = sp.communicate()[0].decode().strip() or None
+        return sp.communicate()[0].decode().strip() or None
-except Exception:
+
-    GIT_HEAD = None
+
+VERSION = get_new_version((sys.argv + [''])[1])
+GIT_HEAD = get_git_head()
 
 VERSION_FILE = f'''\
 # Autogenerated by devscripts/update-version.py
@@ -42,8 +45,6 @@ __version__ = {VERSION!r}
 RELEASE_GIT_HEAD = {GIT_HEAD!r}
 '''
 
-with open('yt_dlp/version.py', 'wt') as f:
+write_file('yt_dlp/version.py', VERSION_FILE)
-    f.write(VERSION_FILE)
+print(f'::set-output name=ytdlp_version::{VERSION}')
-
-print('::set-output name=ytdlp_version::' + VERSION)
 print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}')

devscripts/utils.py

@@ -0,0 +1,35 @@
+import argparse
+import functools
+
+
+def read_file(fname):
+    with open(fname, encoding='utf-8') as f:
+        return f.read()
+
+
+def write_file(fname, content):
+    with open(fname, 'w', encoding='utf-8') as f:
+        return f.write(content)
+
+
+# Get the version without importing the package
+def read_version(fname='yt_dlp/version.py'):
+    exec(compile(read_file(fname), fname, 'exec'))
+    return locals()['__version__']
+
+
+def get_filename_args(has_infile=False, default_outfile=None):
+    parser = argparse.ArgumentParser()
+    if has_infile:
+        parser.add_argument('infile', help='Input file')
+    kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
+    parser.add_argument('outfile', **kwargs, help='Output file')
+
+    opts = parser.parse_args()
+    if has_infile:
+        return opts.infile, opts.outfile
+    return opts.outfile
+
+
+def compose_functions(*functions):
+    return lambda x: functools.reduce(lambda y, f: f(y), functions, x)

pyinst.py

@@ -1,11 +1,17 @@
 #!/usr/bin/env python3
 
+# Allow direct execution
 import os
-import platform
 import sys
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import platform
+
 from PyInstaller.__main__ import run as run_pyinstaller
 
+from devscripts.utils import read_version
+
 OS_NAME, MACHINE, ARCH = sys.platform, platform.machine(), platform.architecture()[0][:2]
 if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
     # NB: Windows x86 has MACHINE = AMD64 irrespective of bitness
@@ -13,8 +19,7 @@ if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
 
 
 def main():
-    opts = parse_options()
+    opts, version = parse_options(), read_version()
-    version = read_version('yt_dlp/version.py')
 
     onedir = '--onedir' in opts or '-D' in opts
     if not onedir and '-F' not in opts and '--onefile' not in opts:
@@ -53,13 +58,6 @@ def parse_options():
     return opts
 
 
-# Get the version from yt_dlp/version.py without importing the package
-def read_version(fname):
-    with open(fname, encoding='utf-8') as f:
-        exec(compile(f.read(), fname, 'exec'))
-        return locals()['__version__']
-
-
 def exe(onedir):
     """@returns (name, path)"""
     name = '_'.join(filter(None, (

setup.py

@@ -12,28 +12,18 @@ except ImportError:
     from distutils.core import Command, setup
     setuptools_available = False
 
+from devscripts.utils import read_file, read_version
 
-def read(fname):
+VERSION = read_version()
-    with open(fname, encoding='utf-8') as f:
-        return f.read()
-
-
-# Get the version from yt_dlp/version.py without importing the package
-def read_version(fname):
-    exec(compile(read(fname), fname, 'exec'))
-    return locals()['__version__']
-
-
-VERSION = read_version('yt_dlp/version.py')
 
 DESCRIPTION = 'A youtube-dl fork with additional features and patches'
 
 LONG_DESCRIPTION = '\n\n'.join((
     'Official repository: <https://github.com/yt-dlp/yt-dlp>',
     '**PS**: Some links in this document will not work since this is a copy of the README.md from Github',
-    read('README.md')))
+    read_file('README.md')))
 
-REQUIREMENTS = read('requirements.txt').splitlines()
+REQUIREMENTS = read_file('requirements.txt').splitlines()
 
 
 def packages():
@@ -121,7 +111,7 @@ class build_lazy_extractors(Command):
         if self.dry_run:
             print('Skipping build of lazy extractors in dry run mode')
             return
-        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'])
+        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
 
 
 params = py2exe_params() if sys.argv[1:2] == ['py2exe'] else build_params()