Mirrors
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[build] Improve release process (#880) 

* Automate more of the release process by animelover1984, pukkandan - closes #823
* Fix sha256 by nihil-admirari - closes #385
* Bring back brew taps by nao20010128nao #865
* Provide `--onedir` zip for windows by pukkandan - Closes #1024, #661, #705 and #890

Authored by: pukkandan, animelover1984, nihil-admirari, nao20010128nao
This commit is contained in:
pukkandan 2021-09-24 06:31:43 +05:30 committed by GitHub
parent e27cc5d864
commit 4c88ff87fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 162 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

186
.github/workflows/build.yml vendored
View File

@ -12,11 +12,15 @@ jobs:
outputs: outputs:
ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
upload_url: ${{ steps.create_release.outputs.upload_url }} upload_url: ${{ steps.create_release.outputs.upload_url }}
sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }} sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }} sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }}
sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }}
sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v2 uses: actions/setup-python@v2
with: with:
@ -25,11 +29,76 @@ jobs:
run: sudo apt-get -y install zip pandoc man run: sudo apt-get -y install zip pandoc man
- name: Bump version - name: Bump version
id: bump_version id: bump_version
run: python devscripts/update-version.py run: |
python devscripts/update-version.py
make issuetemplates
- name: Print version - name: Print version
run: echo "${{ steps.bump_version.outputs.ytdlp_version }}" run: echo "${{ steps.bump_version.outputs.ytdlp_version }}"
- name: Update master
id: push_update
run: |
git config --global user.email "${{ github.event.pusher.email }}"
git config --global user.name "${{ github.event.pusher.name }}"
git add -u
git commit -m "[version] update" -m ":ci skip all"
git pull --rebase origin ${{ github.event.repository.master_branch }}
git push origin ${{ github.event.ref }}:${{ github.event.repository.master_branch }}
echo ::set-output name=head_sha::$(git rev-parse HEAD)
- name: Get Changelog
id: get_changelog
run: |
changelog=$(cat Changelog.md | grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)')
echo "changelog<<EOF" >> $GITHUB_ENV
echo "$changelog" >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- name: Run Make - name: Run Make
run: make all tar run: make all tar
- name: Get SHA2-256SUMS for yt-dlp
id: sha256_bin
run: echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')"
- name: Get SHA2-256SUMS for yt-dlp.tar.gz
id: sha256_tar
run: echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')"
- name: Get SHA2-512SUMS for yt-dlp
id: sha512_bin
run: echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')"
- name: Get SHA2-512SUMS for yt-dlp.tar.gz
id: sha512_tar
run: echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')"
- name: Install SSH private key
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
if: "env.BREW_TOKEN != ''"
uses: webfactory/ssh-agent@v0.5.3
with:
ssh-private-key: ${{ env.BREW_TOKEN }}
- name: Update Homebrew Formulae
env:
BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
if: "env.BREW_TOKEN != ''"
run: |
git clone git@github.com:yt-dlp/homebrew-taps taps/
python3 devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ steps.bump_version.outputs.ytdlp_version }}"
git -C taps/ config user.name github-actions
git -C taps/ config user.email github-actions@example.com
git -C taps/ commit -am 'yt-dlp: ${{ steps.bump_version.outputs.ytdlp_version }}'
git -C taps/ push
- name: Install dependencies for pypi
env:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
if: "env.PYPI_TOKEN != ''"
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Build and publish on pypi
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
if: "env.TWINE_PASSWORD != ''"
run: |
rm -rf dist/*
python setup.py sdist bdist_wheel
twine upload dist/*
- name: Create Release - name: Create Release
id: create_release id: create_release
uses: actions/create-release@v1 uses: actions/create-release@v1
@ -38,9 +107,10 @@ jobs:
with: with:
tag_name: ${{ steps.bump_version.outputs.ytdlp_version }} tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }} release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
commitish: ${{ steps.push_update.outputs.head_sha }}
body: | body: |
Changelog: Changelog:
PLACEHOLDER ${{ env.changelog }}
draft: false draft: false
prerelease: false prerelease: false
- name: Upload yt-dlp Unix binary - name: Upload yt-dlp Unix binary
@ -62,36 +132,16 @@ jobs:
asset_path: ./yt-dlp.tar.gz asset_path: ./yt-dlp.tar.gz
asset_name: yt-dlp.tar.gz asset_name: yt-dlp.tar.gz
asset_content_type: application/gzip asset_content_type: application/gzip
- name: Get SHA2-256SUMS for yt-dlp
id: sha256_file
run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')"
- name: Get SHA2-512SUMS for yt-dlp
id: sha512_file
run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')"
- name: Install dependencies for pypi
env:
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
if: "env.PYPI_TOKEN != ''"
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Build and publish on pypi
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
if: "env.TWINE_PASSWORD != ''"
run: |
rm -rf dist/*
python setup.py sdist bdist_wheel
twine upload dist/*
build_windows: build_windows:
runs-on: windows-latest runs-on: windows-latest
needs: build_unix needs: build_unix
outputs: outputs:
sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }} sha256_win: ${{ steps.sha256_win.outputs.sha256_win }}
sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }} sha512_win: ${{ steps.sha512_win.outputs.sha512_win }}
sha256_win_zip: ${{ steps.sha256_win_zip.outputs.sha256_win_zip }}
sha512_win_zip: ${{ steps.sha512_win_zip.outputs.sha512_win_zip }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -104,7 +154,7 @@ jobs:
run: python -m pip install --upgrade pip setuptools wheel run: python -m pip install --upgrade pip setuptools wheel
- name: Install Requirements - name: Install Requirements
# Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
run: pip install "https://yt-dlp.github.io/pyinstaller-builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
- name: Bump version - name: Bump version
id: bump_version id: bump_version
run: python devscripts/update-version.py run: python devscripts/update-version.py
@ -123,19 +173,41 @@ jobs:
asset_name: yt-dlp.exe asset_name: yt-dlp.exe
asset_content_type: application/vnd.microsoft.portable-executable asset_content_type: application/vnd.microsoft.portable-executable
- name: Get SHA2-256SUMS for yt-dlp.exe - name: Get SHA2-256SUMS for yt-dlp.exe
id: sha256_file_win id: sha256_win
run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" run: echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
- name: Get SHA2-512SUMS for yt-dlp.exe - name: Get SHA2-512SUMS for yt-dlp.exe
id: sha512_file_win id: sha512_win
run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" run: echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
- name: Run PyInstaller Script with --onedir
run: python pyinst.py 64 --onedir
- uses: papeloto/action-zip@v1
with:
files: ./dist/yt-dlp
dest: ./dist/yt-dlp.zip
- name: Upload yt-dlp.zip Windows onedir
id: upload-release-windows-zip
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ needs.build_unix.outputs.upload_url }}
asset_path: ./dist/yt-dlp.zip
asset_name: yt-dlp.zip
asset_content_type: application/zip
- name: Get SHA2-256SUMS for yt-dlp.zip
id: sha256_win_zip
run: echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA256).Hash.ToLower())"
- name: Get SHA2-512SUMS for yt-dlp.zip
id: sha512_win_zip
run: echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA512).Hash.ToLower())"
build_windows32: build_windows32:
runs-on: windows-latest runs-on: windows-latest
needs: [build_unix, build_windows] needs: [build_unix, build_windows]
outputs: outputs:
sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }} sha256_win32: ${{ steps.sha256_win32.outputs.sha256_win32 }}
sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }} sha512_win32: ${{ steps.sha512_win32.outputs.sha512_win32 }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -148,7 +220,7 @@ jobs:
- name: Upgrade pip and enable wheel support - name: Upgrade pip and enable wheel support
run: python -m pip install --upgrade pip setuptools wheel run: python -m pip install --upgrade pip setuptools wheel
- name: Install Requirements - name: Install Requirements
run: pip install "https://yt-dlp.github.io/pyinstaller-builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
- name: Bump version - name: Bump version
id: bump_version id: bump_version
run: python devscripts/update-version.py run: python devscripts/update-version.py
@ -167,11 +239,11 @@ jobs:
asset_name: yt-dlp_x86.exe asset_name: yt-dlp_x86.exe
asset_content_type: application/vnd.microsoft.portable-executable asset_content_type: application/vnd.microsoft.portable-executable
- name: Get SHA2-256SUMS for yt-dlp_x86.exe - name: Get SHA2-256SUMS for yt-dlp_x86.exe
id: sha256_file_win32 id: sha256_win32
run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" run: echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
- name: Get SHA2-512SUMS for yt-dlp_x86.exe - name: Get SHA2-512SUMS for yt-dlp_x86.exe
id: sha512_file_win32 id: sha512_win32
run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" run: echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())"
finish: finish:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -180,15 +252,17 @@ jobs:
steps: steps:
- name: Make SHA2-256SUMS file - name: Make SHA2-256SUMS file
env: env:
SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }} SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }}
SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }} SHA256_WIN_ZIP: ${{ needs.build_windows.outputs.sha256_win_zip }}
SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }} SHA256_WIN32: ${{ needs.build_windows32.outputs.sha256_win32 }}
YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }} SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }}
SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }}
run: | run: |
echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS echo "${{ env.SHA256_WIN }} yt-dlp.exe" >> SHA2-256SUMS
echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS echo "${{ env.SHA256_WIN32 }} yt-dlp_x86.exe" >> SHA2-256SUMS
echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS echo "${{ env.SHA256_BIN }} yt-dlp" >> SHA2-256SUMS
echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS echo "${{ env.SHA256_TAR }} yt-dlp.tar.gz" >> SHA2-256SUMS
echo "${{ env.SHA256_WIN_ZIP }} yt-dlp.zip" >> SHA2-256SUMS
- name: Upload 256SUMS file - name: Upload 256SUMS file
id: upload-sums id: upload-sums
uses: actions/upload-release-asset@v1 uses: actions/upload-release-asset@v1
@ -201,13 +275,17 @@ jobs:
asset_content_type: text/plain asset_content_type: text/plain
- name: Make SHA2-512SUMS file - name: Make SHA2-512SUMS file
env: env:
SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }} SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }}
SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }} SHA512_WIN_ZIP: ${{ needs.build_windows.outputs.sha512_win_zip }}
SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }} SHA512_WIN32: ${{ needs.build_windows32.outputs.sha512_win32 }}
SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }}
SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }}
run: | run: |
echo "${{ env.SHA512_WINDOWS }} yt-dlp.exe" >> SHA2-512SUMS echo "${{ env.SHA512_WIN }} yt-dlp.exe" >> SHA2-512SUMS
echo "${{ env.SHA512_WINDOWS32 }} yt-dlp_x86.exe" >> SHA2-512SUMS echo "${{ env.SHA512_WIN32 }} yt-dlp_x86.exe" >> SHA2-512SUMS
echo "${{ env.SHA512_UNIX }} yt-dlp" >> SHA2-512SUMS echo "${{ env.SHA512_BIN }} yt-dlp" >> SHA2-512SUMS
echo "${{ env.SHA512_TAR }} yt-dlp.tar.gz" >> SHA2-512SUMS
echo "${{ env.SHA512_WIN_ZIP }} yt-dlp.zip" >> SHA2-512SUMS
- name: Upload 512SUMS file - name: Upload 512SUMS file
id: upload-512sums id: upload-512sums
uses: actions/upload-release-asset@v1 uses: actions/upload-release-asset@v1

7
Changelog.md
View File

@ -7,14 +7,9 @@
* Update Changelog.md and CONTRIBUTORS * Update Changelog.md and CONTRIBUTORS
* Change "Merged with ytdl" version in Readme.md if needed * Change "Merged with ytdl" version in Readme.md if needed
* Add new/fixed extractors in "new features" section of Readme.md * Add new/fixed extractors in "new features" section of Readme.md
* Commit to master as `Release <version>` * Commit as `Release <version>`
* Push to origin/release using `git push origin master:release` * Push to origin/release using `git push origin master:release`
build task will now run build task will now run
* Update version.py using `devscripts\update-version.py`
* Run `make issuetemplates`
* Commit to master as `[version] update :ci skip all`
* Push to origin/master
* Update changelog in /releases
--> -->

7
pyinst.py
View File

@ -15,9 +15,11 @@ import PyInstaller.__main__
arch = sys.argv[1] if len(sys.argv) > 1 else platform.architecture()[0][:2] arch = sys.argv[1] if len(sys.argv) > 1 else platform.architecture()[0][:2]
assert arch in ('32', '64') assert arch in ('32', '64')
print('Building %sbit version' % arch)
_x86 = '_x86' if arch == '32' else '' _x86 = '_x86' if arch == '32' else ''
opts = sys.argv[2:] or ['--onefile']
print(f'Building {arch}bit version with options {opts}')
FILE_DESCRIPTION = 'yt-dlp%s' % (' (32 Bit)' if _x86 else '') FILE_DESCRIPTION = 'yt-dlp%s' % (' (32 Bit)' if _x86 else '')
# root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) # root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
@ -72,11 +74,12 @@ excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc']
PyInstaller.__main__.run([ PyInstaller.__main__.run([
'--name=yt-dlp%s' % _x86, '--name=yt-dlp%s' % _x86,
'--onefile',
'--icon=devscripts/logo.ico', '--icon=devscripts/logo.ico',
*[f'--exclude-module={module}' for module in excluded_modules], *[f'--exclude-module={module}' for module in excluded_modules],
*[f'--hidden-import={module}' for module in dependancies], *[f'--hidden-import={module}' for module in dependancies],
'--upx-exclude=vcruntime140.dll', '--upx-exclude=vcruntime140.dll',
'--noconfirm',
*opts,
'yt_dlp/__main__.py', 'yt_dlp/__main__.py',
]) ])
SetVersion('dist/yt-dlp%s.exe' % _x86, VERSION_FILE) SetVersion('dist/yt-dlp%s.exe' % _x86, VERSION_FILE)

10
yt_dlp/YoutubeDL.py
View File

@ -27,7 +27,6 @@ import traceback
import random import random
from string import ascii_letters from string import ascii_letters
from zipimport import zipimporter
from .compat import ( from .compat import (
compat_basestring, compat_basestring,
@ -143,6 +142,7 @@ from .postprocessor import (
FFmpegPostProcessor, FFmpegPostProcessor,
MoveFilesAfterDownloadPP, MoveFilesAfterDownloadPP,
) )
from .update import detect_variant
from .version import __version__ from .version import __version__
if compat_os_name == 'nt': if compat_os_name == 'nt':
@ -3266,12 +3266,8 @@ class YoutubeDL(object):
self.get_encoding())) self.get_encoding()))
write_string(encoding_str, encoding=None) write_string(encoding_str, encoding=None)
source = ( source = detect_variant()
'(exe)' if hasattr(sys, 'frozen') self._write_string('[debug] yt-dlp version %s%s\n' % (__version__, '' if source == 'unknown' else f' ({source})'))
else '(zip)' if isinstance(globals().get('__loader__'), zipimporter)
else '(source)' if os.path.basename(sys.argv[0]) == '__main__.py'
else '')
self._write_string('[debug] yt-dlp version %s %s\n' % (__version__, source))
if _LAZY_LOADER: if _LAZY_LOADER:
self._write_string('[debug] Lazy loading extractors enabled\n') self._write_string('[debug] Lazy loading extractors enabled\n')
if _PLUGIN_CLASSES: if _PLUGIN_CLASSES:

32
yt_dlp/update.py
View File

@ -31,6 +31,18 @@ def rsa_verify(message, signature, key):
''' '''
def detect_variant():
if hasattr(sys, 'frozen') and getattr(sys, '_MEIPASS', None):
if sys._MEIPASS == os.path.dirname(sys.executable):
return 'dir'
return 'exe'
elif isinstance(globals().get('__loader__'), zipimporter):
return 'zip'
elif os.path.basename(sys.argv[0]) == '__main__.py':
return 'source'
return 'unknown'
def update_self(to_screen, verbose, opener): def update_self(to_screen, verbose, opener):
''' Exists for backward compatibility. Use run_update(ydl) instead ''' ''' Exists for backward compatibility. Use run_update(ydl) instead '''
@ -87,13 +99,14 @@ def run_update(ydl):
h.update(mv[:n]) h.update(mv[:n])
return h.hexdigest() return h.hexdigest()
err = None ERRORS = {
if isinstance(globals().get('__loader__'), zipimporter): 'exe': None,
pass 'zip': None,
elif hasattr(sys, 'frozen'): 'dir': 'Auto-update is not supported for unpackaged windows executable. Re-download the latest release',
pass 'source': 'You cannot update when running from source code',
else: 'unknown': 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Use that to update',
err = 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Please use that to update' }
err = ERRORS.get(detect_variant(), ERRORS['unknown'])
if err: if err:
return report_error(err, expected=True) return report_error(err, expected=True)
@ -138,11 +151,6 @@ def run_update(ydl):
if not urlh: if not urlh:
return None return None
hash_data = ydl._opener.open(urlh).read().decode('utf-8') hash_data = ydl._opener.open(urlh).read().decode('utf-8')
if hash_data.startswith('version:'):
# Old colon-separated hash file
return dict(ln.split(':') for ln in hash_data.splitlines()).get(filename)
else:
# GNU-style hash file
return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename) return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename)
if not os.access(filename, os.W_OK): if not os.access(filename, os.W_OK):