From 95e82347b398d8bb160767cdd975edecd62cbabd Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 9 Jan 2024 03:11:52 +0000 Subject: [PATCH] [ie/Viously] Add extractor (#8927) Replaces Turbo extractor Authored by: nbr23, seproDev Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com> --- yt_dlp/extractor/_extractors.py | 2 +- yt_dlp/extractor/turbo.py | 64 --------------------------------- yt_dlp/extractor/viously.py | 60 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 65 deletions(-) delete mode 100644 yt_dlp/extractor/turbo.py create mode 100644 yt_dlp/extractor/viously.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 6f7a1e4f1..557ff9447 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2019,7 +2019,6 @@ from .tunein import ( TuneInPodcastEpisodeIE, TuneInShortenerIE, ) -from .turbo import TurboIE from .tv2 import ( TV2IE, TV2ArticleIE, @@ -2223,6 +2222,7 @@ from .viki import ( VikiIE, VikiChannelIE, ) +from .viously import ViouslyIE from .viqeo import ViqeoIE from .viu import ( ViuIE, diff --git a/yt_dlp/extractor/turbo.py b/yt_dlp/extractor/turbo.py deleted file mode 100644 index cdb7dcff8..000000000 --- a/yt_dlp/extractor/turbo.py +++ /dev/null @@ -1,64 +0,0 @@ -import re - -from .common import InfoExtractor -from ..compat import compat_str -from ..utils import ( - ExtractorError, - int_or_none, - qualities, - xpath_text, -) - - -class TurboIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?turbo\.fr/videos-voiture/(?P[0-9]+)-' - _API_URL = 'http://www.turbo.fr/api/tv/xml.php?player_generique=player_generique&id={0:}' - _TEST = { - 'url': 'http://www.turbo.fr/videos-voiture/454443-turbo-du-07-09-2014-renault-twingo-3-bentley-continental-gt-speed-ces-guide-achat-dacia.html', - 'md5': '33f4b91099b36b5d5a91f84b5bcba600', - 'info_dict': { - 'id': '454443', - 'ext': 'mp4', - 'duration': 3715, - 'title': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia... ', - 'description': 'Turbo du 07/09/2014 : Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...', - 'thumbnail': r're:^https?://.*\.jpg$', - } - } - - def _real_extract(self, url): - mobj = self._match_valid_url(url) - video_id = mobj.group('id') - - webpage = self._download_webpage(url, video_id) - - playlist = self._download_xml(self._API_URL.format(video_id), video_id) - item = playlist.find('./channel/item') - if item is None: - raise ExtractorError('Playlist item was not found', expected=True) - - title = xpath_text(item, './title', 'title') - duration = int_or_none(xpath_text(item, './durate', 'duration')) - thumbnail = xpath_text(item, './visuel_clip', 'thumbnail') - description = self._html_search_meta('description', webpage) - - formats = [] - get_quality = qualities(['3g', 'sd', 'hq']) - for child in item: - m = re.search(r'url_video_(?P.+)', child.tag) - if m: - quality = compat_str(m.group('quality')) - formats.append({ - 'format_id': quality, - 'url': child.text, - 'quality': get_quality(quality), - }) - - return { - 'id': video_id, - 'title': title, - 'duration': duration, - 'thumbnail': thumbnail, - 'description': description, - 'formats': formats, - } diff --git a/yt_dlp/extractor/viously.py b/yt_dlp/extractor/viously.py new file mode 100644 index 000000000..9ec7ed35f --- /dev/null +++ b/yt_dlp/extractor/viously.py @@ -0,0 +1,60 @@ +import base64 +import re + +from .common import InfoExtractor +from ..utils import ( + extract_attributes, + int_or_none, + parse_iso8601, +) +from ..utils.traversal import traverse_obj + + +class ViouslyIE(InfoExtractor): + _VALID_URL = False + _WEBPAGE_TESTS = [{ + 'url': 'http://www.turbo.fr/videos-voiture/454443-turbo-du-07-09-2014-renault-twingo-3-bentley-continental-gt-speed-ces-guide-achat-dacia.html', + 'md5': '37a6c3381599381ff53a7e1e0575c0bc', + 'info_dict': { + 'id': 'F_xQzS2jwb3', + 'ext': 'mp4', + 'title': 'Turbo du 07/09/2014\xa0: Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...', + 'description': 'Turbo du 07/09/2014\xa0: Renault Twingo 3, Bentley Continental GT Speed, CES, Guide Achat Dacia...', + 'age_limit': 0, + 'upload_date': '20230328', + 'timestamp': 1680037507, + 'duration': 3716, + 'categories': ['motors'], + } + }] + + def _extract_from_webpage(self, url, webpage): + viously_players = re.findall(r']*class="(?:[^"]*\s)?v(?:iou)?sly-player(?:\s[^"]*)?"[^>]*>', webpage) + if not viously_players: + return + + def custom_decode(text): + STANDARD_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=' + CUSTOM_ALPHABET = 'VIOUSLYABCDEFGHJKMNPQRTWXZviouslyabcdefghjkmnpqrtwxz9876543210+/=' + data = base64.b64decode(text.translate(str.maketrans(CUSTOM_ALPHABET, STANDARD_ALPHABET))) + return data.decode('utf-8').strip('\x00') + + for video_id in traverse_obj(viously_players, (..., {extract_attributes}, 'id')): + formats = self._extract_m3u8_formats( + f'https://www.viously.com/video/hls/{video_id}/index.m3u8', video_id, fatal=False) + if not formats: + continue + data = self._download_json( + f'https://www.viously.com/export/json/{video_id}', video_id, + transform_source=custom_decode, fatal=False) + yield { + 'id': video_id, + 'formats': formats, + **traverse_obj(data, ('video', { + 'title': ('title', {str}), + 'description': ('description', {str}), + 'duration': ('duration', {int_or_none}), + 'timestamp': ('iso_date', {parse_iso8601}), + 'categories': ('category', 'name', {str}, {lambda x: [x] if x else None}), + })), + }