[cookies] Report progress when importing cookies

yt_dlp/YoutubeDL.py

@@ -643,6 +643,11 @@ class YoutubeDL(object):
                 else:
                     raise
 
+        if auto_init:
+            if auto_init != 'no_verbose_header':
+                self.print_debug_header()
+            self.add_default_info_extractors()
+
         if (sys.platform != 'win32'
                 and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
                 and not self.params.get('restrictfilenames', False)):
@@ -664,13 +669,6 @@ class YoutubeDL(object):
         # Set http_headers defaults according to std_headers
         self.params['http_headers'] = merge_headers(std_headers, self.params.get('http_headers', {}))
 
-        self._setup_opener()
-
-        if auto_init:
-            if auto_init != 'no_verbose_header':
-                self.print_debug_header()
-            self.add_default_info_extractors()
-
         hooks = {
             'post_hooks': self.add_post_hook,
             'progress_hooks': self.add_progress_hook,
@@ -687,6 +685,7 @@ class YoutubeDL(object):
                 get_postprocessor(pp_def.pop('key'))(self, **compat_kwargs(pp_def)),
                 when=when)
 
+        self._setup_opener()
         register_socks_protocols()
 
         def preload_download_archive(fn):
@@ -3698,6 +3697,7 @@ class YoutubeDL(object):
             delim=', ') or 'none'
         write_debug('Optional libraries: %s' % lib_str)
 
+        self._setup_opener()
         proxy_map = {}
         for handler in self._opener.handlers:
             if hasattr(handler, 'proxies'):
@@ -3717,6 +3717,8 @@ class YoutubeDL(object):
                     latest_version)
 
     def _setup_opener(self):
+        if hasattr(self, '_opener'):
+            return
         timeout_val = self.params.get('socket_timeout')
         self._socket_timeout = 20 if timeout_val is None else float(timeout_val)
 

yt_dlp/cookies.py

@@ -20,6 +20,7 @@ from .compat import (
     compat_b64decode,
     compat_cookiejar_Cookie,
 )
+from .minicurses import MultilinePrinter, QuietMultilinePrinter
 from .utils import (
     error_to_str,
     expand_path,
@@ -73,6 +74,32 @@ class YDLLogger:
         if self._ydl:
             self._ydl.report_error(message)
 
+    def progress_bar(self):
+        """Return a context manager with a print method. (Optional)"""
+        # Do not print to files/pipes, loggers, or when --no-progress is used
+        if not self._ydl or self._ydl.params.get('noprogress') or self._ydl.params.get('logger'):
+            return
+        file = self._ydl._out_files['error']
+        try:
+            if not file.isatty():
+                return
+        except BaseException:
+            return
+
+        printer = MultilinePrinter(file, preserve_output=False)
+        printer.print = lambda message: printer.print_at_line(f'[Cookies] {message}', 0)
+        return printer
+
+
+def _create_progress_bar(logger):
+    if hasattr(logger, 'progress_bar'):
+        printer = logger.progress_bar()
+        if printer:
+            return printer
+    printer = QuietMultilinePrinter()
+    printer.print = lambda _: None
+    return printer
+
 
 def load_cookies(cookie_file, browser_specification, ydl):
     cookie_jars = []
@@ -115,7 +142,7 @@ def _extract_firefox_cookies(profile, logger):
     else:
         search_root = os.path.join(_firefox_browser_dir(), profile)
 
-    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite')
+    cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite', logger)
     if cookie_database_path is None:
         raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root))
     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
@@ -126,13 +153,17 @@ def _extract_firefox_cookies(profile, logger):
             cursor = _open_database_copy(cookie_database_path, tmpdir)
             cursor.execute('SELECT host, name, value, path, expiry, isSecure FROM moz_cookies')
             jar = YoutubeDLCookieJar()
-            for host, name, value, path, expiry, is_secure in cursor.fetchall():
+            with _create_progress_bar(logger) as progress_bar:
-                cookie = compat_cookiejar_Cookie(
+                table = cursor.fetchall()
-                    version=0, name=name, value=value, port=None, port_specified=False,
+                total_cookie_count = len(table)
-                    domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
+                for i, (host, name, value, path, expiry, is_secure) in enumerate(table):
-                    path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
+                    progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
-                    comment=None, comment_url=None, rest={})
+                    cookie = compat_cookiejar_Cookie(
-                jar.set_cookie(cookie)
+                        version=0, name=name, value=value, port=None, port_specified=False,
+                        domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
+                        path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
+                        comment=None, comment_url=None, rest={})
+                    jar.set_cookie(cookie)
             logger.info('Extracted {} cookies from firefox'.format(len(jar)))
             return jar
         finally:
@@ -232,7 +263,7 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
             logger.error('{} does not support profiles'.format(browser_name))
             search_root = config['browser_dir']
 
-    cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies')
+    cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies', logger)
     if cookie_database_path is None:
         raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root))
     logger.debug('Extracting cookies from: "{}"'.format(cookie_database_path))
@@ -251,26 +282,18 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
             jar = YoutubeDLCookieJar()
             failed_cookies = 0
             unencrypted_cookies = 0
-            for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall():
+            with _create_progress_bar(logger) as progress_bar:
-                host_key = host_key.decode('utf-8')
+                table = cursor.fetchall()
-                name = name.decode('utf-8')
+                total_cookie_count = len(table)
-                value = value.decode('utf-8')
+                for i, line in enumerate(table):
-                path = path.decode('utf-8')
+                    progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
-
+                    is_encrypted, cookie = _process_chrome_cookie(decryptor, *line)
-                if not value and encrypted_value:
+                    if not cookie:
-                    value = decryptor.decrypt(encrypted_value)
-                    if value is None:
                         failed_cookies += 1
                         continue
-                else:
+                    elif not is_encrypted:
-                    unencrypted_cookies += 1
+                        unencrypted_cookies += 1
-
+                    jar.set_cookie(cookie)
-                cookie = compat_cookiejar_Cookie(
-                    version=0, name=name, value=value, port=None, port_specified=False,
-                    domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
-                    path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
-                    comment=None, comment_url=None, rest={})
-                jar.set_cookie(cookie)
             if failed_cookies > 0:
                 failed_message = ' ({} could not be decrypted)'.format(failed_cookies)
             else:
@@ -285,6 +308,25 @@ def _extract_chrome_cookies(browser_name, profile, keyring, logger):
                 cursor.connection.close()
 
 
+def _process_chrome_cookie(decryptor, host_key, name, value, encrypted_value, path, expires_utc, is_secure):
+    host_key = host_key.decode('utf-8')
+    name = name.decode('utf-8')
+    value = value.decode('utf-8')
+    path = path.decode('utf-8')
+    is_encrypted = not value and encrypted_value
+
+    if is_encrypted:
+        value = decryptor.decrypt(encrypted_value)
+        if value is None:
+            return is_encrypted, None
+
+    return is_encrypted, compat_cookiejar_Cookie(
+        version=0, name=name, value=value, port=None, port_specified=False,
+        domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
+        path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
+        comment=None, comment_url=None, rest={})
+
+
 class ChromeCookieDecryptor:
     """
     Overview:
@@ -547,10 +589,12 @@ def _parse_safari_cookies_page(data, jar, logger):
 
     p.skip_to(record_offsets[0], 'unknown page header field')
 
-    for record_offset in record_offsets:
+    with _create_progress_bar(logger) as progress_bar:
-        p.skip_to(record_offset, 'space between records')
+        for i, record_offset in enumerate(record_offsets):
-        record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
+            progress_bar.print(f'Loading cookie {i: 6d}/{number_of_cookies: 6d}')
-        p.read_bytes(record_length)
+            p.skip_to(record_offset, 'space between records')
+            record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger)
+            p.read_bytes(record_length)
     p.skip_to_end('space in between pages')
 
 
@@ -830,10 +874,11 @@ def _get_mac_keyring_password(browser_keyring_name, logger):
 
 
 def _get_windows_v10_key(browser_root, logger):
-    path = _find_most_recently_used_file(browser_root, 'Local State')
+    path = _find_most_recently_used_file(browser_root, 'Local State', logger)
     if path is None:
         logger.error('could not find local state file')
         return None
+    logger.debug(f'Found local state file at "{path}"')
     with open(path, 'r', encoding='utf8') as f:
         data = json.load(f)
     try:
@@ -925,13 +970,16 @@ def _get_column_names(cursor, table_name):
     return [row[1].decode('utf-8') for row in table_info]
 
 
-def _find_most_recently_used_file(root, filename):
+def _find_most_recently_used_file(root, filename, logger):
     # if there are multiple browser profiles, take the most recently used one
-    paths = []
+    i, paths = 0, []
-    for root, dirs, files in os.walk(root):
+    with _create_progress_bar(logger) as progress_bar:
-        for file in files:
+        for curr_root, dirs, files in os.walk(root):
-            if file == filename:
+            for file in files:
-                paths.append(os.path.join(root, file))
+                i += 1
+                progress_bar.print(f'Searching for "{filename}": {i: 6d} files searched')
+                if file == filename:
+                    paths.append(os.path.join(curr_root, file))
     return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime)
 
 

yt_dlp/minicurses.py

@@ -178,4 +178,4 @@ class MultilinePrinter(MultilinePrinterBase):
                 *text, CONTROL_SEQUENCES['ERASE_LINE'],
                 f'{CONTROL_SEQUENCES["UP"]}{CONTROL_SEQUENCES["ERASE_LINE"]}' * self.maximum)
         else:
-            self.write(*text, ' ' * self._lastlength)
+            self.write('\r', ' ' * self._lastlength, '\r')