From 0263617a83689c7b5dc7a3255253821fdd541eb8 Mon Sep 17 00:00:00 2001 From: Leon Date: Fri, 29 Jan 2021 03:30:01 +0100 Subject: [PATCH 1/8] disable auto-cleanup if custom user-data-dir is used - prevent custom data dir to be automatically removed. - remove the driver executable cleanup function, it is deleted at initialization anyway so it was double trouble. - reverted back to having chromedriver in the CWD instead of a temp folder @untested todo: test --- undetected_chromedriver/v2.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 131c0da..54ad79f 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -121,10 +121,14 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): user_data_dir=None, factor=0.5, delay=1, - ): - + ): + self._data_dir_default = True + if user_data_dir: + self._data_dir_default = False + p = Patcher(target_path=executable_path) p.auto(False) + self._patcher = p self.factor = factor self.delay = delay @@ -251,7 +255,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): except Exception: # noqa pass try: - shutil.rmtree(self.user_data_dir, ignore_errors=False) + if self._data_dir_default: + shutil.rmtree(self.user_data_dir, ignore_errors=False) except PermissionError: time.sleep(1) self.quit() @@ -274,10 +279,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): class Patcher(object): url_repo = "https://chromedriver.storage.googleapis.com" - def __init__(self, target_path=None, force=False, version_main: int = 0): - - if not target_path: - target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') + def __init__(self, target_path='./chromedriver', force=False, version_main: int = 0): + # if not target_path: + # target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') if not IS_POSIX: if not target_path[-4:] == ".exe": target_path += ".exe" @@ -445,10 +449,6 @@ class Patcher(object): fh.write(newline) linect += 1 return linect - - def __del__(self): - shutil.rmtree(os.path.dirname(self.target_path), ignore_errors=True) - - + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): pass From 33b796ddf7510ef2f27c34a5e4fa5273df8cb2c1 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 2 Feb 2021 07:56:13 +0100 Subject: [PATCH 2/8] - removed enable_console_log - reverted back to storing chromedriver binary in current workdir after several reports of users entire project folder being deleted (sorry for that btw). - some code cleanup - added a fix for useragent in headless mode in v2 (which reported 'Headless'). still headless mode in v2 is under construction and not fully functional. - added a keyword argument to the Chrome constructor: 'emulate_touch' which, when set to True will report presense of a touch(screen/device). This is mainly for bet365 detections. Otherwise i would not recommend setting it. credits to @boganfoo for this excellent find! --- undetected_chromedriver/__init__.py | 38 +++++---- undetected_chromedriver/v2.py | 116 +++++++++++++++++++--------- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/undetected_chromedriver/__init__.py b/undetected_chromedriver/__init__.py index a56ff4a..baa7141 100644 --- a/undetected_chromedriver/__init__.py +++ b/undetected_chromedriver/__init__.py @@ -19,11 +19,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam) import io import logging import os +import random import re +import string import sys import zipfile -import string -import random from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -32,12 +32,11 @@ from selenium.webdriver import ChromeOptions as _ChromeOptions logger = logging.getLogger(__name__) - TARGET_VERSION = 0 class Chrome: - def __new__(cls, *args, enable_console_log=False, **kwargs): + def __new__(cls, *args, emulate_touch=False, **kwargs): if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() @@ -73,11 +72,6 @@ class Chrome: }) }); """ - + ( - "console.log = console.dir = console.error = function(){};" - if not enable_console_log - else "" - ) }, ) return instance._orig_get(*args, **kwargs) @@ -89,14 +83,31 @@ class Chrome: ) instance.execute_cdp_cmd( "Network.setUserAgentOverride", - {"userAgent": original_user_agent_string.replace("Headless", ""),}, + { + "userAgent": original_user_agent_string.replace("Headless", ""), + }, ) + if emulate_touch: + instance.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})") return instance class ChromeOptions: + __doc__ = _ChromeOptions.__doc__ + def __new__(cls, *args, **kwargs): + __doc__ = _ChromeOptions.__new__.__doc__ + if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() if not ChromeDriverManager.selenium_patched: @@ -111,7 +122,6 @@ class ChromeOptions: class ChromeDriverManager(object): - installed = False selenium_patched = False target_version = None @@ -224,10 +234,10 @@ class ChromeDriverManager(object): @staticmethod def random_cdc(): cdc = random.choices(string.ascii_lowercase, k=26) - cdc[-6: -4] = map(str.upper, cdc[-6: -4]) + cdc[-6:-4] = map(str.upper, cdc[-6:-4]) cdc[2] = cdc[0] - cdc[3] = '_' - return ''.join(cdc).encode() + cdc[3] = "_" + return "".join(cdc).encode() def patch_binary(self): """ diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 54ad79f..ad5f8a0 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -46,8 +46,6 @@ import tempfile import threading import time import zipfile -import atexit -import contextlib from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -76,8 +74,10 @@ def find_chrome_executable(): for item in os.environ.get("PATH").split(os.pathsep): for subitem in ("google-chrome", "chromium", "chromium-browser"): candidates.add(os.sep.join((item, subitem))) - if 'darwin' in sys.platform: - candidates.update(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]) + if "darwin" in sys.platform: + candidates.update( + ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] + ) else: for item in map( os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA") @@ -94,16 +94,18 @@ def find_chrome_executable(): class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): - - __doc__ = """\ - -------------------------------------------------------------------------- - NOTE: - Chrome has everything included to work out of the box. - it does not `need` customizations. - any customizations MAY lead to trigger bot migitation systems. - - -------------------------------------------------------------------------- - """ + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + __doc__ = ( + """\ + -------------------------------------------------------------------------- + NOTE: + Chrome has everything included to work out of the box. + it does not `need` customizations. + any customizations MAY lead to trigger bot migitation systems. + + -------------------------------------------------------------------------- + """ + + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + ) _instances = set() @@ -121,14 +123,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): user_data_dir=None, factor=0.5, delay=1, - ): + emulate_touch=False, + ): self._data_dir_default = True if user_data_dir: self._data_dir_default = False - + p = Patcher(target_path=executable_path) p.auto(False) - + self._patcher = p self.factor = factor self.delay = delay @@ -169,6 +172,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): extra_args = [] if options.headless: extra_args.append("--headless") + options.add_argument("start-maximized") self.browser_args = [ find_chrome_executable(), @@ -199,16 +203,63 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): keep_alive=keep_alive, ) + if options.headless: + + orig_get = self.get + + def get_wrapped(*args, **kwargs): + if self.execute_script("return navigator.webdriver"): + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(window, 'navigator', { + value: new Proxy(navigator, { + has: (target, key) => (key === 'webdriver' ? false : key in target), + get: (target, key) => + key === 'webdriver' + ? undefined + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key] + }) + }); + """ + }, + ) + + self.execute_cdp_cmd( + "Network.setUserAgentOverride", + { + "userAgent": self.execute_script( + "return navigator.userAgent" + ).replace("Headless", "") + }, + ) + return orig_get(*args, **kwargs) + + self.get = get_wrapped + + if emulate_touch: + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + def start_session(self, capabilities=None, browser_profile=None): if not capabilities: capabilities = self.options.to_capabilities() super().start_session(capabilities, browser_profile) - def get_in(self, url: str, delay=2.5, factor=1): + def get_in(self, url: str, delay=2.5): """ :param url: str - :param delay: int - :param factor: disconnect seconds after .get() + :param delay: disconnect seconds after .get() too low will disconnect before get() fired. ================================================= @@ -238,8 +289,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): self.get(url) finally: self.close() - # threading.Timer(factor or self.factor, self.close).start() - time.sleep(delay or self.delay) + threading.Timer(delay, self.close).start() + time.sleep(delay) self.start_session() def quit(self): @@ -279,9 +330,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): class Patcher(object): url_repo = "https://chromedriver.storage.googleapis.com" - def __init__(self, target_path='./chromedriver', force=False, version_main: int = 0): - # if not target_path: - # target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') + def __init__( + self, target_path="./chromedriver", force=False, version_main: int = 0 + ): if not IS_POSIX: if not target_path[-4:] == ".exe": target_path += ".exe" @@ -327,14 +378,7 @@ class Patcher(object): :return: version string :rtype: LooseVersion """ - path = ( - "/" - + ( - "latest_release" - if not self.version_main - else f"latest_release_{self.version_main}" - ).upper() - ) + path = ("/latest_release" if not self.version_main else f"/latest_release_{self.version_main}").upper() logger.debug("getting release number from %s" % path) return LooseVersion(urlopen(self.url_repo + path).read().decode()) @@ -367,7 +411,7 @@ class Patcher(object): os.makedirs(os.path.dirname(self.target_path), mode=0o755) except OSError: pass - with zipfile.ZipFile(self.zipname, mode='r') as zf: + with zipfile.ZipFile(self.zipname, mode="r") as zf: zf.extract(self.exename) os.rename(self.exename, self.target_path) os.remove(self.zipname) @@ -436,7 +480,6 @@ class Patcher(object): :return: False on failure, binary name on success """ - logger.info("patching driver executable %s" % self.target_path) linect = 0 @@ -449,6 +492,7 @@ class Patcher(object): fh.write(newline) linect += 1 return linect - + + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): pass From b792eaf2422455d3b2bcda34112350ae17a7fe4d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 2 Feb 2021 07:56:13 +0100 Subject: [PATCH 3/8] - removed enable_console_log - reverted back to storing chromedriver binary in current workdir after several reports of users entire project folder being deleted (sorry for that btw). - some code cleanup - added a fix for useragent in headless mode in v2 (which reported 'Headless'). still headless mode in v2 is under construction and not fully functional. - added a keyword argument to the Chrome constructor: 'emulate_touch' which, when set to True will report presense of a touch(screen/device). This is mainly for bet365 detections. Otherwise i would not recommend setting it. credits to @boganfoo for this excellent find! --- undetected_chromedriver/__init__.py | 38 +++++---- undetected_chromedriver/v2.py | 116 +++++++++++++++++++--------- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/undetected_chromedriver/__init__.py b/undetected_chromedriver/__init__.py index a56ff4a..baa7141 100644 --- a/undetected_chromedriver/__init__.py +++ b/undetected_chromedriver/__init__.py @@ -19,11 +19,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam) import io import logging import os +import random import re +import string import sys import zipfile -import string -import random from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -32,12 +32,11 @@ from selenium.webdriver import ChromeOptions as _ChromeOptions logger = logging.getLogger(__name__) - TARGET_VERSION = 0 class Chrome: - def __new__(cls, *args, enable_console_log=False, **kwargs): + def __new__(cls, *args, emulate_touch=False, **kwargs): if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() @@ -73,11 +72,6 @@ class Chrome: }) }); """ - + ( - "console.log = console.dir = console.error = function(){};" - if not enable_console_log - else "" - ) }, ) return instance._orig_get(*args, **kwargs) @@ -89,14 +83,31 @@ class Chrome: ) instance.execute_cdp_cmd( "Network.setUserAgentOverride", - {"userAgent": original_user_agent_string.replace("Headless", ""),}, + { + "userAgent": original_user_agent_string.replace("Headless", ""), + }, ) + if emulate_touch: + instance.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})") return instance class ChromeOptions: + __doc__ = _ChromeOptions.__doc__ + def __new__(cls, *args, **kwargs): + __doc__ = _ChromeOptions.__new__.__doc__ + if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() if not ChromeDriverManager.selenium_patched: @@ -111,7 +122,6 @@ class ChromeOptions: class ChromeDriverManager(object): - installed = False selenium_patched = False target_version = None @@ -224,10 +234,10 @@ class ChromeDriverManager(object): @staticmethod def random_cdc(): cdc = random.choices(string.ascii_lowercase, k=26) - cdc[-6: -4] = map(str.upper, cdc[-6: -4]) + cdc[-6:-4] = map(str.upper, cdc[-6:-4]) cdc[2] = cdc[0] - cdc[3] = '_' - return ''.join(cdc).encode() + cdc[3] = "_" + return "".join(cdc).encode() def patch_binary(self): """ diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 54ad79f..ad5f8a0 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -46,8 +46,6 @@ import tempfile import threading import time import zipfile -import atexit -import contextlib from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -76,8 +74,10 @@ def find_chrome_executable(): for item in os.environ.get("PATH").split(os.pathsep): for subitem in ("google-chrome", "chromium", "chromium-browser"): candidates.add(os.sep.join((item, subitem))) - if 'darwin' in sys.platform: - candidates.update(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]) + if "darwin" in sys.platform: + candidates.update( + ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] + ) else: for item in map( os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA") @@ -94,16 +94,18 @@ def find_chrome_executable(): class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): - - __doc__ = """\ - -------------------------------------------------------------------------- - NOTE: - Chrome has everything included to work out of the box. - it does not `need` customizations. - any customizations MAY lead to trigger bot migitation systems. - - -------------------------------------------------------------------------- - """ + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + __doc__ = ( + """\ + -------------------------------------------------------------------------- + NOTE: + Chrome has everything included to work out of the box. + it does not `need` customizations. + any customizations MAY lead to trigger bot migitation systems. + + -------------------------------------------------------------------------- + """ + + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + ) _instances = set() @@ -121,14 +123,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): user_data_dir=None, factor=0.5, delay=1, - ): + emulate_touch=False, + ): self._data_dir_default = True if user_data_dir: self._data_dir_default = False - + p = Patcher(target_path=executable_path) p.auto(False) - + self._patcher = p self.factor = factor self.delay = delay @@ -169,6 +172,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): extra_args = [] if options.headless: extra_args.append("--headless") + options.add_argument("start-maximized") self.browser_args = [ find_chrome_executable(), @@ -199,16 +203,63 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): keep_alive=keep_alive, ) + if options.headless: + + orig_get = self.get + + def get_wrapped(*args, **kwargs): + if self.execute_script("return navigator.webdriver"): + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(window, 'navigator', { + value: new Proxy(navigator, { + has: (target, key) => (key === 'webdriver' ? false : key in target), + get: (target, key) => + key === 'webdriver' + ? undefined + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key] + }) + }); + """ + }, + ) + + self.execute_cdp_cmd( + "Network.setUserAgentOverride", + { + "userAgent": self.execute_script( + "return navigator.userAgent" + ).replace("Headless", "") + }, + ) + return orig_get(*args, **kwargs) + + self.get = get_wrapped + + if emulate_touch: + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + def start_session(self, capabilities=None, browser_profile=None): if not capabilities: capabilities = self.options.to_capabilities() super().start_session(capabilities, browser_profile) - def get_in(self, url: str, delay=2.5, factor=1): + def get_in(self, url: str, delay=2.5): """ :param url: str - :param delay: int - :param factor: disconnect seconds after .get() + :param delay: disconnect seconds after .get() too low will disconnect before get() fired. ================================================= @@ -238,8 +289,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): self.get(url) finally: self.close() - # threading.Timer(factor or self.factor, self.close).start() - time.sleep(delay or self.delay) + threading.Timer(delay, self.close).start() + time.sleep(delay) self.start_session() def quit(self): @@ -279,9 +330,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): class Patcher(object): url_repo = "https://chromedriver.storage.googleapis.com" - def __init__(self, target_path='./chromedriver', force=False, version_main: int = 0): - # if not target_path: - # target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') + def __init__( + self, target_path="./chromedriver", force=False, version_main: int = 0 + ): if not IS_POSIX: if not target_path[-4:] == ".exe": target_path += ".exe" @@ -327,14 +378,7 @@ class Patcher(object): :return: version string :rtype: LooseVersion """ - path = ( - "/" - + ( - "latest_release" - if not self.version_main - else f"latest_release_{self.version_main}" - ).upper() - ) + path = ("/latest_release" if not self.version_main else f"/latest_release_{self.version_main}").upper() logger.debug("getting release number from %s" % path) return LooseVersion(urlopen(self.url_repo + path).read().decode()) @@ -367,7 +411,7 @@ class Patcher(object): os.makedirs(os.path.dirname(self.target_path), mode=0o755) except OSError: pass - with zipfile.ZipFile(self.zipname, mode='r') as zf: + with zipfile.ZipFile(self.zipname, mode="r") as zf: zf.extract(self.exename) os.rename(self.exename, self.target_path) os.remove(self.zipname) @@ -436,7 +480,6 @@ class Patcher(object): :return: False on failure, binary name on success """ - logger.info("patching driver executable %s" % self.target_path) linect = 0 @@ -449,6 +492,7 @@ class Patcher(object): fh.write(newline) linect += 1 return linect - + + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): pass From a50362f9e8934f6629d6f7679e13c829217cafe5 Mon Sep 17 00:00:00 2001 From: UltrafunkAmsterdam Date: Tue, 2 Feb 2021 08:55:58 +0100 Subject: [PATCH 4/8] - added a keyword argument to the Chrome constructor: `emulate_touch` which, when set to True will report presense of a touch(screen/device). This is mainly for bet365 detections. Otherwise i would not recommend setting it. credits to @boganfoo for this excellent find! - removed `enable_console_log` - reverted back to storing chromedriver binary in current workdir after several reports of users entire project folder being deleted (sorry for that btw). - some code cleanup - added a fix for useragent in headless mode in v2 (which reported 'Headless'). still headless mode in v2 is under construction and not fully functional. --- undetected_chromedriver/__init__.py | 52 +++++++++++---------- undetected_chromedriver/v2.py | 70 +++++++++++++++-------------- 2 files changed, 62 insertions(+), 60 deletions(-) diff --git a/undetected_chromedriver/__init__.py b/undetected_chromedriver/__init__.py index baa7141..a735fe5 100644 --- a/undetected_chromedriver/__init__.py +++ b/undetected_chromedriver/__init__.py @@ -19,11 +19,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam) import io import logging import os -import random import re -import string import sys import zipfile +import string +import random from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -32,6 +32,7 @@ from selenium.webdriver import ChromeOptions as _ChromeOptions logger = logging.getLogger(__name__) + TARGET_VERSION = 0 @@ -60,54 +61,50 @@ class Chrome: { "source": """ - Object.defineProperty(window, 'navigator', { - value: new Proxy(navigator, { - has: (target, key) => (key === 'webdriver' ? false : key in target), - get: (target, key) => - key === 'webdriver' - ? undefined - : typeof target[key] === 'function' - ? target[key].bind(target) - : target[key] - }) - }); - """ + Object.defineProperty(window, 'navigator', { + value: new Proxy(navigator, { + has: (target, key) => (key === 'webdriver' ? false : key in target), + get: (target, key) => + key === 'webdriver' + ? undefined + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key] + }) + }); + """ }, ) return instance._orig_get(*args, **kwargs) instance.get = _get_wrapped + instance.get = _get_wrapped + instance.get = _get_wrapped original_user_agent_string = instance.execute_script( "return navigator.userAgent" ) instance.execute_cdp_cmd( "Network.setUserAgentOverride", - { - "userAgent": original_user_agent_string.replace("Headless", ""), - }, + {"userAgent": original_user_agent_string.replace("Headless", ""),}, ) if emulate_touch: instance.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", { "source": """ - Object.defineProperty(navigator, 'maxTouchPoints', { - get: () => 1 - })""" + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" }, ) - logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})") return instance class ChromeOptions: - __doc__ = _ChromeOptions.__doc__ def __new__(cls, *args, **kwargs): - __doc__ = _ChromeOptions.__new__.__doc__ - if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() if not ChromeDriverManager.selenium_patched: @@ -122,6 +119,7 @@ class ChromeOptions: class ChromeDriverManager(object): + installed = False selenium_patched = False target_version = None @@ -234,10 +232,10 @@ class ChromeDriverManager(object): @staticmethod def random_cdc(): cdc = random.choices(string.ascii_lowercase, k=26) - cdc[-6:-4] = map(str.upper, cdc[-6:-4]) + cdc[-6: -4] = map(str.upper, cdc[-6: -4]) cdc[2] = cdc[0] - cdc[3] = "_" - return "".join(cdc).encode() + cdc[3] = '_' + return ''.join(cdc).encode() def patch_binary(self): """ diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index ad5f8a0..26af959 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -46,6 +46,8 @@ import tempfile import threading import time import zipfile +import atexit +import contextlib from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -74,10 +76,8 @@ def find_chrome_executable(): for item in os.environ.get("PATH").split(os.pathsep): for subitem in ("google-chrome", "chromium", "chromium-browser"): candidates.add(os.sep.join((item, subitem))) - if "darwin" in sys.platform: - candidates.update( - ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] - ) + if 'darwin' in sys.platform: + candidates.update(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]) else: for item in map( os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA") @@ -94,18 +94,16 @@ def find_chrome_executable(): class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): - __doc__ = ( - """\ - -------------------------------------------------------------------------- - NOTE: - Chrome has everything included to work out of the box. - it does not `need` customizations. - any customizations MAY lead to trigger bot migitation systems. - - -------------------------------------------------------------------------- - """ - + selenium.webdriver.remote.webdriver.WebDriver.__doc__ - ) + + __doc__ = """\ + -------------------------------------------------------------------------- + NOTE: + Chrome has everything included to work out of the box. + it does not `need` customizations. + any customizations MAY lead to trigger bot migitation systems. + + -------------------------------------------------------------------------- + """ + selenium.webdriver.remote.webdriver.WebDriver.__doc__ _instances = set() @@ -125,13 +123,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): delay=1, emulate_touch=False, ): - self._data_dir_default = True - if user_data_dir: - self._data_dir_default = False p = Patcher(target_path=executable_path) p.auto(False) - self._patcher = p self.factor = factor self.delay = delay @@ -172,7 +166,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): extra_args = [] if options.headless: extra_args.append("--headless") - options.add_argument("start-maximized") self.browser_args = [ find_chrome_executable(), @@ -245,9 +238,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): "Page.addScriptToEvaluateOnNewDocument", { "source": """ - Object.defineProperty(navigator, 'maxTouchPoints', { - get: () => 1 - })""" + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" }, ) @@ -256,10 +249,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): capabilities = self.options.to_capabilities() super().start_session(capabilities, browser_profile) - def get_in(self, url: str, delay=2.5): + def get_in(self, url: str, delay=2.5, factor=1): """ :param url: str - :param delay: disconnect seconds after .get() + :param delay: int + :param factor: disconnect seconds after .get() too low will disconnect before get() fired. ================================================= @@ -289,8 +283,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): self.get(url) finally: self.close() - threading.Timer(delay, self.close).start() - time.sleep(delay) + # threading.Timer(factor or self.factor, self.close).start() + time.sleep(delay or self.delay) self.start_session() def quit(self): @@ -306,8 +300,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): except Exception: # noqa pass try: - if self._data_dir_default: - shutil.rmtree(self.user_data_dir, ignore_errors=False) + shutil.rmtree(self.user_data_dir, ignore_errors=False) except PermissionError: time.sleep(1) self.quit() @@ -331,7 +324,7 @@ class Patcher(object): url_repo = "https://chromedriver.storage.googleapis.com" def __init__( - self, target_path="./chromedriver", force=False, version_main: int = 0 + self, target_path="./chromedriver", force=False, version_main: int = 0 ): if not IS_POSIX: if not target_path[-4:] == ".exe": @@ -378,7 +371,14 @@ class Patcher(object): :return: version string :rtype: LooseVersion """ - path = ("/latest_release" if not self.version_main else f"/latest_release_{self.version_main}").upper() + path = ( + "/" + + ( + "latest_release" + if not self.version_main + else f"latest_release_{self.version_main}" + ).upper() + ) logger.debug("getting release number from %s" % path) return LooseVersion(urlopen(self.url_repo + path).read().decode()) @@ -411,7 +411,7 @@ class Patcher(object): os.makedirs(os.path.dirname(self.target_path), mode=0o755) except OSError: pass - with zipfile.ZipFile(self.zipname, mode="r") as zf: + with zipfile.ZipFile(self.zipname, mode='r') as zf: zf.extract(self.exename) os.rename(self.exename, self.target_path) os.remove(self.zipname) @@ -480,6 +480,7 @@ class Patcher(object): :return: False on failure, binary name on success """ + logger.info("patching driver executable %s" % self.target_path) linect = 0 @@ -493,6 +494,9 @@ class Patcher(object): linect += 1 return linect + def __del__(self): + shutil.rmtree(os.path.dirname(self.target_path), ignore_errors=True) + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): pass From 38b3eb2ec893c4e45d92cbf9083d356b2e21c085 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 2 Feb 2021 09:30:42 +0100 Subject: [PATCH 5/8] Update v2.py --- undetected_chromedriver/v2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 26af959..16c41f6 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -494,8 +494,8 @@ class Patcher(object): linect += 1 return linect - def __del__(self): - shutil.rmtree(os.path.dirname(self.target_path), ignore_errors=True) + + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): From db2214071dee171a0b6d66325f66f23ae472e876 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 3 Feb 2021 09:29:41 +0100 Subject: [PATCH 6/8] Update v2.py --- undetected_chromedriver/v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 16c41f6..2718b7d 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -166,6 +166,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): extra_args = [] if options.headless: extra_args.append("--headless") + extra_args.append("--window-size=1920,1080") self.browser_args = [ find_chrome_executable(), From fd91a8992176c028d4c8850187f39e1697f9f80f Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 4 Feb 2021 11:49:06 +0100 Subject: [PATCH 7/8] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ed2a93..9eccee9 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ from setuptools import setup setup( name="undetected-chromedriver", - version="2.0.2", + version="2.1.0", packages=["undetected_chromedriver"], install_requires=["selenium",], url="https://github.com/ultrafunkamsterdam/undetected-chromedriver", From 0fa88770a237acd1c9bfbb1a9431aa526b1cbdb3 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 4 Feb 2021 11:57:45 +0100 Subject: [PATCH 8/8] fix closing browser window when using as context manager --- undetected_chromedriver/v2.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 2718b7d..86a40d6 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -119,8 +119,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): keep_alive=True, debug_addr=None, user_data_dir=None, - factor=0.5, - delay=1, + factor=1, + delay=2, emulate_touch=False, ): @@ -250,7 +250,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): capabilities = self.options.to_capabilities() super().start_session(capabilities, browser_profile) - def get_in(self, url: str, delay=2.5, factor=1): + def get_in(self, url: str, delay=2, factor=1): """ :param url: str :param delay: int @@ -283,10 +283,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): try: self.get(url) finally: - self.close() + self.service.stop() # threading.Timer(factor or self.factor, self.close).start() time.sleep(delay or self.delay) - self.start_session() + self.service.start() + # self.start_session() def quit(self): try: @@ -313,9 +314,10 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - threading.Timer(self.factor, self.start_session).start() + self.service.stop() + #threading.Timer(self.factor, self.service.start).start() time.sleep(self.delay) + self.service.start() def __hash__(self): return hash(self.options.debugger_address)