Mirrors
/
yt-dlp
mirror of https://github.com/yt-dlp/yt-dlp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Gettr] Improve extractor (#2920) 

Authored by: i6t
This commit is contained in:
i6t 2022-03-04 22:53:43 +09:00 committed by GitHub
parent 97bef011ee
commit b90dbe6c19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 55 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
yt_dlp/extractor/gettr.py
View File

@ -8,8 +8,8 @@ from ..utils import (
dict_get, dict_get,
float_or_none, float_or_none,
int_or_none, int_or_none,
remove_end,
str_or_none, str_or_none,
traverse_obj,
try_get, try_get,
url_or_none, url_or_none,
urljoin, urljoin,
@ -36,8 +36,9 @@ class GettrIE(GettrBaseIE):
'ext': 'mp4', 'ext': 'mp4',
'uploader': 'EpochTV', 'uploader': 'EpochTV',
'uploader_id': 'epochtv', 'uploader_id': 'epochtv',
'upload_date': '20210927',
'thumbnail': r're:^https?://.+/out\.jpg', 'thumbnail': r're:^https?://.+/out\.jpg',
'timestamp': 1632782451058, 'timestamp': 1632782451.058,
'duration': 58.5585, 'duration': 58.5585,
'tags': ['hornofafrica', 'explorations'], 'tags': ['hornofafrica', 'explorations'],
} }
@ -50,43 +51,69 @@ class GettrIE(GettrBaseIE):
'ext': 'mp4', 'ext': 'mp4',
'uploader': 'Neues Forum Freiheit', 'uploader': 'Neues Forum Freiheit',
'uploader_id': 'nf_freiheit', 'uploader_id': 'nf_freiheit',
'upload_date': '20210718',
'thumbnail': r're:^https?://.+/out\.jpg', 'thumbnail': r're:^https?://.+/out\.jpg',
'timestamp': 1626594455017, 'timestamp': 1626594455.017,
'duration': 23, 'duration': 23,
'tags': 'count:12', 'tags': 'count:12',
} }
}, {
# quote post
'url': 'https://gettr.com/post/pxn5b743a9',
'only_matching': True,
}, {
# quote with video
'url': 'https://gettr.com/post/pxtiiz5ca2',
'only_matching': True,
}, {
# streaming embed
'url': 'https://gettr.com/post/pxlu8p3b13',
'only_matching': True,
}, {
# youtube embed
'url': 'https://gettr.com/post/pv6wp9e24c',
'only_matching': True,
'add_ie': ['Youtube'],
}] }]
def _real_extract(self, url): def _real_extract(self, url):
post_id = self._match_id(url) post_id = self._match_id(url)
webpage = self._download_webpage(url, post_id) webpage = self._download_webpage(url, post_id)
api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id) api_data = self._call_api('post/%s?incl="poststats|userinfo"' % post_id, post_id)
post_data = api_data.get('data') post_data = api_data.get('data')
user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']]) or {} user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']], dict) or {}
if post_data.get('nfound'): vid = post_data.get('vid')
raise ExtractorError(post_data.get('txt'), expected=True) ovid = post_data.get('ovid')
if post_data.get('p_type') == 'stream':
return self.url_result(f'https://gettr.com/streaming/{post_id}', ie='GettrStreaming', video_id=post_id)
if not (ovid or vid):
embed_url = url_or_none(post_data.get('prevsrc'))
shared_post_id = traverse_obj(api_data, ('aux', 'shrdpst', '_id'), ('data', 'rpstIds', 0), expected_type=str)
if embed_url:
return self.url_result(embed_url)
elif shared_post_id:
return self.url_result(f'https://gettr.com/post/{shared_post_id}', ie='Gettr', video_id=shared_post_id)
else:
raise ExtractorError('There\'s no video in this post.')
title = description = str_or_none( title = description = str_or_none(
post_data.get('txt') or self._og_search_description(webpage)) post_data.get('txt') or self._og_search_description(webpage))
uploader = str_or_none( uploader = str_or_none(
user_data.get('nickname') user_data.get('nickname')
or remove_end(self._og_search_title(webpage), ' on GETTR')) or self._search_regex(r'^(.+?) on GETTR', self._og_search_title(webpage, default=''), 'uploader', fatal=False))
if uploader: if uploader:
title = '%s - %s' % (uploader, title) title = '%s - %s' % (uploader, title)
if not dict_get(post_data, ['vid', 'ovid']): formats, subtitles = self._extract_m3u8_formats_and_subtitles(
raise ExtractorError('There\'s no video in this post.')
vid = post_data.get('vid')
ovid = post_data.get('ovid')
formats = self._extract_m3u8_formats(
urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4', urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else [] entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else ([], {})
if ovid: if ovid:
formats.append({ formats.append({
@ -103,15 +130,16 @@ class GettrIE(GettrBaseIE):
'id': post_id, 'id': post_id,
'title': title, 'title': title,
'description': description, 'description': description,
'thumbnail': url_or_none( 'formats': formats,
urljoin(self._MEDIA_BASE_URL, post_data.get('main')) 'subtitles': subtitles,
or self._og_search_thumbnail(webpage)), 'uploader': uploader,
'timestamp': int_or_none(post_data.get('cdate')),
'uploader_id': str_or_none( 'uploader_id': str_or_none(
dict_get(user_data, ['_id', 'username']) dict_get(user_data, ['_id', 'username'])
or post_data.get('uid')), or post_data.get('uid')),
'uploader': uploader, 'thumbnail': url_or_none(
'formats': formats, urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
or self._html_search_meta(['og:image', 'image'], webpage, 'thumbnail', fatal=False)),
'timestamp': float_or_none(dict_get(post_data, ['cdate', 'udate']), scale=1000),
'duration': float_or_none(post_data.get('vid_dur')), 'duration': float_or_none(post_data.get('vid_dur')),
'tags': post_data.get('htgs'), 'tags': post_data.get('htgs'),
} }
@ -165,19 +193,19 @@ class GettrStreamingIE(GettrBaseIE):
thumbnails = [{ thumbnails = [{
'url': urljoin(self._MEDIA_BASE_URL, thumbnail), 'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
} for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []] } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs'], list) or []]
self._sort_formats(formats) self._sort_formats(formats)
return { return {
'id': video_id, 'id': video_id,
'title': try_get(video_info, lambda x: x['postData']['ttl']), 'title': try_get(video_info, lambda x: x['postData']['ttl'], str),
'description': try_get(video_info, lambda x: x['postData']['dsc']), 'description': try_get(video_info, lambda x: x['postData']['dsc'], str),
'formats': formats, 'formats': formats,
'subtitles': subtitles, 'subtitles': subtitles,
'thumbnails': thumbnails, 'thumbnails': thumbnails,
'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']), 'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname'], str),
'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']), 'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id'], str),
'view_count': int_or_none(live_info.get('viewsCount')), 'view_count': int_or_none(live_info.get('viewsCount')),
'timestamp': float_or_none(live_info.get('startAt'), scale=1000), 'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
'duration': float_or_none(live_info.get('duration'), scale=1000), 'duration': float_or_none(live_info.get('duration'), scale=1000),