From b792eaf2422455d3b2bcda34112350ae17a7fe4d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 2 Feb 2021 07:56:13 +0100 Subject: [PATCH] - removed enable_console_log - reverted back to storing chromedriver binary in current workdir after several reports of users entire project folder being deleted (sorry for that btw). - some code cleanup - added a fix for useragent in headless mode in v2 (which reported 'Headless'). still headless mode in v2 is under construction and not fully functional. - added a keyword argument to the Chrome constructor: 'emulate_touch' which, when set to True will report presense of a touch(screen/device). This is mainly for bet365 detections. Otherwise i would not recommend setting it. credits to @boganfoo for this excellent find! --- undetected_chromedriver/__init__.py | 38 +++++---- undetected_chromedriver/v2.py | 116 +++++++++++++++++++--------- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/undetected_chromedriver/__init__.py b/undetected_chromedriver/__init__.py index a56ff4a..baa7141 100644 --- a/undetected_chromedriver/__init__.py +++ b/undetected_chromedriver/__init__.py @@ -19,11 +19,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam) import io import logging import os +import random import re +import string import sys import zipfile -import string -import random from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -32,12 +32,11 @@ from selenium.webdriver import ChromeOptions as _ChromeOptions logger = logging.getLogger(__name__) - TARGET_VERSION = 0 class Chrome: - def __new__(cls, *args, enable_console_log=False, **kwargs): + def __new__(cls, *args, emulate_touch=False, **kwargs): if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() @@ -73,11 +72,6 @@ class Chrome: }) }); """ - + ( - "console.log = console.dir = console.error = function(){};" - if not enable_console_log - else "" - ) }, ) return instance._orig_get(*args, **kwargs) @@ -89,14 +83,31 @@ class Chrome: ) instance.execute_cdp_cmd( "Network.setUserAgentOverride", - {"userAgent": original_user_agent_string.replace("Headless", ""),}, + { + "userAgent": original_user_agent_string.replace("Headless", ""), + }, ) + if emulate_touch: + instance.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})") return instance class ChromeOptions: + __doc__ = _ChromeOptions.__doc__ + def __new__(cls, *args, **kwargs): + __doc__ = _ChromeOptions.__new__.__doc__ + if not ChromeDriverManager.installed: ChromeDriverManager(*args, **kwargs).install() if not ChromeDriverManager.selenium_patched: @@ -111,7 +122,6 @@ class ChromeOptions: class ChromeDriverManager(object): - installed = False selenium_patched = False target_version = None @@ -224,10 +234,10 @@ class ChromeDriverManager(object): @staticmethod def random_cdc(): cdc = random.choices(string.ascii_lowercase, k=26) - cdc[-6: -4] = map(str.upper, cdc[-6: -4]) + cdc[-6:-4] = map(str.upper, cdc[-6:-4]) cdc[2] = cdc[0] - cdc[3] = '_' - return ''.join(cdc).encode() + cdc[3] = "_" + return "".join(cdc).encode() def patch_binary(self): """ diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py index 54ad79f..ad5f8a0 100644 --- a/undetected_chromedriver/v2.py +++ b/undetected_chromedriver/v2.py @@ -46,8 +46,6 @@ import tempfile import threading import time import zipfile -import atexit -import contextlib from distutils.version import LooseVersion from urllib.request import urlopen, urlretrieve @@ -76,8 +74,10 @@ def find_chrome_executable(): for item in os.environ.get("PATH").split(os.pathsep): for subitem in ("google-chrome", "chromium", "chromium-browser"): candidates.add(os.sep.join((item, subitem))) - if 'darwin' in sys.platform: - candidates.update(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]) + if "darwin" in sys.platform: + candidates.update( + ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"] + ) else: for item in map( os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA") @@ -94,16 +94,18 @@ def find_chrome_executable(): class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): - - __doc__ = """\ - -------------------------------------------------------------------------- - NOTE: - Chrome has everything included to work out of the box. - it does not `need` customizations. - any customizations MAY lead to trigger bot migitation systems. - - -------------------------------------------------------------------------- - """ + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + __doc__ = ( + """\ + -------------------------------------------------------------------------- + NOTE: + Chrome has everything included to work out of the box. + it does not `need` customizations. + any customizations MAY lead to trigger bot migitation systems. + + -------------------------------------------------------------------------- + """ + + selenium.webdriver.remote.webdriver.WebDriver.__doc__ + ) _instances = set() @@ -121,14 +123,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): user_data_dir=None, factor=0.5, delay=1, - ): + emulate_touch=False, + ): self._data_dir_default = True if user_data_dir: self._data_dir_default = False - + p = Patcher(target_path=executable_path) p.auto(False) - + self._patcher = p self.factor = factor self.delay = delay @@ -169,6 +172,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): extra_args = [] if options.headless: extra_args.append("--headless") + options.add_argument("start-maximized") self.browser_args = [ find_chrome_executable(), @@ -199,16 +203,63 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): keep_alive=keep_alive, ) + if options.headless: + + orig_get = self.get + + def get_wrapped(*args, **kwargs): + if self.execute_script("return navigator.webdriver"): + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(window, 'navigator', { + value: new Proxy(navigator, { + has: (target, key) => (key === 'webdriver' ? false : key in target), + get: (target, key) => + key === 'webdriver' + ? undefined + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key] + }) + }); + """ + }, + ) + + self.execute_cdp_cmd( + "Network.setUserAgentOverride", + { + "userAgent": self.execute_script( + "return navigator.userAgent" + ).replace("Headless", "") + }, + ) + return orig_get(*args, **kwargs) + + self.get = get_wrapped + + if emulate_touch: + self.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", + { + "source": """ + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 1 + })""" + }, + ) + def start_session(self, capabilities=None, browser_profile=None): if not capabilities: capabilities = self.options.to_capabilities() super().start_session(capabilities, browser_profile) - def get_in(self, url: str, delay=2.5, factor=1): + def get_in(self, url: str, delay=2.5): """ :param url: str - :param delay: int - :param factor: disconnect seconds after .get() + :param delay: disconnect seconds after .get() too low will disconnect before get() fired. ================================================= @@ -238,8 +289,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): self.get(url) finally: self.close() - # threading.Timer(factor or self.factor, self.close).start() - time.sleep(delay or self.delay) + threading.Timer(delay, self.close).start() + time.sleep(delay) self.start_session() def quit(self): @@ -279,9 +330,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): class Patcher(object): url_repo = "https://chromedriver.storage.googleapis.com" - def __init__(self, target_path='./chromedriver', force=False, version_main: int = 0): - # if not target_path: - # target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') + def __init__( + self, target_path="./chromedriver", force=False, version_main: int = 0 + ): if not IS_POSIX: if not target_path[-4:] == ".exe": target_path += ".exe" @@ -327,14 +378,7 @@ class Patcher(object): :return: version string :rtype: LooseVersion """ - path = ( - "/" - + ( - "latest_release" - if not self.version_main - else f"latest_release_{self.version_main}" - ).upper() - ) + path = ("/latest_release" if not self.version_main else f"/latest_release_{self.version_main}").upper() logger.debug("getting release number from %s" % path) return LooseVersion(urlopen(self.url_repo + path).read().decode()) @@ -367,7 +411,7 @@ class Patcher(object): os.makedirs(os.path.dirname(self.target_path), mode=0o755) except OSError: pass - with zipfile.ZipFile(self.zipname, mode='r') as zf: + with zipfile.ZipFile(self.zipname, mode="r") as zf: zf.extract(self.exename) os.rename(self.exename, self.target_path) os.remove(self.zipname) @@ -436,7 +480,6 @@ class Patcher(object): :return: False on failure, binary name on success """ - logger.info("patching driver executable %s" % self.target_path) linect = 0 @@ -449,6 +492,7 @@ class Patcher(object): fh.write(newline) linect += 1 return linect - + + class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): pass