[ie/cnbc] Overhaul extractors (#8741)

Closes #5871, Closes #8378
Authored by: gonzalezjo, Noor-5, zhijinwuu, ruiminggu, seproDev

Co-authored-by: Noor Mostafa <93787875+Noor-5@users.noreply.github.com>
Co-authored-by: zhijinwuu <zhijinw@andrew.cmu.edu>
Co-authored-by: ruiminggu <ruimingg@andrew.cmu.edu>
Co-authored-by: sepro <4618135+seproDev@users.noreply.github.com>

yt_dlp/extractor/_extractors.py

@@ -379,7 +379,6 @@ from .clubic import ClubicIE
 from .clyp import ClypIE
 from .cmt import CMTIE
 from .cnbc import (
-    CNBCIE,
     CNBCVideoIE,
 )
 from .cnn import (

yt_dlp/extractor/cnbc.py

@@ -1,68 +1,97 @@
 from .common import InfoExtractor
-from ..utils import smuggle_url
+from ..utils import int_or_none, parse_iso8601, str_or_none, url_or_none
-
+from ..utils.traversal import traverse_obj
-
-class CNBCIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.cnbc\.com/gallery/\?video=(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://video.cnbc.com/gallery/?video=3000503714',
-        'info_dict': {
-            'id': '3000503714',
-            'ext': 'mp4',
-            'title': 'Fighting zombies is big business',
-            'description': 'md5:0c100d8e1a7947bd2feec9a5550e519e',
-            'timestamp': 1459332000,
-            'upload_date': '20160330',
-            'uploader': 'NBCU-CNBC',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        'skip': 'Dead link',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': smuggle_url(
-                'http://link.theplatform.com/s/gZWlPC/media/guid/2408950221/%s?mbr=true&manifest=m3u' % video_id,
-                {'force_smil_url': True}),
-            'id': video_id,
-        }
 
 
 class CNBCVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cnbc\.com(?P<path>/video/(?:[^/]+/)+(?P<id>[^./?#&]+)\.html)'
+    _VALID_URL = r'https?://(?:www\.)?cnbc\.com/video/(?:[^/?#]+/)+(?P<id>[^./?#&]+)\.html'
-    _TEST = {
+
-        'url': 'https://www.cnbc.com/video/2018/07/19/trump-i-dont-necessarily-agree-with-raising-rates.html',
+    _TESTS = [{
+        'url': 'https://www.cnbc.com/video/2023/12/07/mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand.html',
         'info_dict': {
-            'id': '7000031301',
             'ext': 'mp4',
-            'title': "Trump: I don't necessarily agree with raising rates",
+            'id': '107344774',
-            'description': 'md5:878d8f0b4ebb5bb1dda3514b91b49de3',
+            'display_id': 'mcdonalds-just-unveiled-cosmcsits-new-spinoff-brand',
-            'timestamp': 1531958400,
+            'modified_timestamp': 1702053483,
-            'upload_date': '20180719',
+            'timestamp': 1701977810,
-            'uploader': 'NBCU-CNBC',
+            'channel': 'News Videos',
+            'upload_date': '20231207',
+            'description': 'md5:882c001d85cb43d7579b514307b3e78b',
+            'release_timestamp': 1701977375,
+            'modified_date': '20231208',
+            'release_date': '20231207',
+            'duration': 65,
+            'author': 'Sean Conlon',
+            'title': 'Here\'s a first look at McDonald\'s new spinoff brand, CosMc\'s',
+            'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107344192-1701894812493-CosMcsskyHero_2336x1040_hero-desktop.jpg?v=1701894855',
         },
-        'params': {
+        'expected_warnings': ['Unable to download f4m manifest'],
-            'skip_download': True,
+    }, {
+        'url': 'https://www.cnbc.com/video/2023/12/08/jim-cramer-shares-his-take-on-seattles-tech-scene.html',
+        'info_dict': {
+            'author': 'Jim Cramer',
+            'channel': 'Mad Money with Jim Cramer',
+            'description': 'md5:72925be21b952e95eba51178dddf4e3e',
+            'duration': 299.0,
+            'ext': 'mp4',
+            'id': '107345451',
+            'display_id': 'jim-cramer-shares-his-take-on-seattles-tech-scene',
+            'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345481-1702079431MM-B-120823.jpg?v=1702079430',
+            'timestamp': 1702080139,
+            'title': 'Jim Cramer shares his take on Seattle\'s tech scene',
+            'release_date': '20231208',
+            'upload_date': '20231209',
+            'modified_timestamp': 1702080139,
+            'modified_date': '20231209',
+            'release_timestamp': 1702073551,
         },
-        'skip': 'Dead link',
+        'expected_warnings': ['Unable to download f4m manifest'],
-    }
+    }, {
+        'url': 'https://www.cnbc.com/video/2023/12/08/the-epicenter-of-ai-is-in-seattle-says-jim-cramer.html',
+        'info_dict': {
+            'author': 'Jim Cramer',
+            'channel': 'Mad Money with Jim Cramer',
+            'description': 'md5:72925be21b952e95eba51178dddf4e3e',
+            'duration': 113.0,
+            'ext': 'mp4',
+            'id': '107345474',
+            'display_id': 'the-epicenter-of-ai-is-in-seattle-says-jim-cramer',
+            'thumbnail': 'https://image.cnbcfm.com/api/v1/image/107345486-Screenshot_2023-12-08_at_70339_PM.png?v=1702080248',
+            'timestamp': 1702080535,
+            'title': 'The epicenter of AI is in Seattle, says Jim Cramer',
+            'release_timestamp': 1702077347,
+            'modified_timestamp': 1702080535,
+            'release_date': '20231208',
+            'upload_date': '20231209',
+            'modified_date': '20231209',
+        },
+        'expected_warnings': ['Unable to download f4m manifest'],
+    }]
 
     def _real_extract(self, url):
-        path, display_id = self._match_valid_url(url).groups()
+        display_id = self._match_id(url)
-        video_id = self._download_json(
+        webpage = self._download_webpage(url, display_id)
-            'https://webql-redesign.cnbcfm.com/graphql', display_id, query={
+        data = self._search_json(r'window\.__s_data=', webpage, 'video data', display_id)
-                'query': '''{
+
-  page(path: "%s") {
+        player_data = traverse_obj(data, (
-    vcpsId
+            'page', 'page', 'layout', ..., 'columns', ..., 'modules',
-  }
+            lambda _, v: v['name'] == 'clipPlayer', 'data', {dict}), get_all=False)
-}''' % path,
+
-            })['data']['page']['vcpsId']
+        return {
-        return self.url_result(
+            'id': display_id,
-            'http://video.cnbc.com/gallery/?video=%d' % video_id,
+            'display_id': display_id,
-            CNBCIE.ie_key())
+            'formats': self._extract_akamai_formats(player_data['playbackURL'], display_id),
+            **self._search_json_ld(webpage, display_id, fatal=False),
+            **traverse_obj(player_data, {
+                'id': ('id', {str_or_none}),
+                'title': ('title', {str}),
+                'description': ('description', {str}),
+                'author': ('author', ..., 'name', {str}),
+                'timestamp': ('datePublished', {parse_iso8601}),
+                'release_timestamp': ('uploadDate', {parse_iso8601}),
+                'modified_timestamp': ('dateLastPublished', {parse_iso8601}),
+                'thumbnail': ('thumbnail', {url_or_none}),
+                'duration': ('duration', {int_or_none}),
+                'channel': ('section', 'title', {str}),
+            }, get_all=False),
+        }