- removed enable_console_log

- reverted back to storing chromedriver binary in current workdir after several reports of users entire project folder being deleted (sorry for that btw).

- some code cleanup

- added a fix for useragent in headless mode in v2 (which reported 'Headless'). still headless mode in v2 is under construction and not fully functional.

- added a keyword argument to the Chrome constructor: 'emulate_touch' which, when set to True will report presense of a touch(screen/device). This is mainly for bet365 detections. Otherwise i would not recommend setting it.

 credits to @boganfoo for this excellent find!
This commit is contained in:
unknown 2021-02-02 07:56:13 +01:00
parent 0263617a83
commit b792eaf242
2 changed files with 104 additions and 50 deletions

View File

@ -19,11 +19,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
import io import io
import logging import logging
import os import os
import random
import re import re
import string
import sys import sys
import zipfile import zipfile
import string
import random
from distutils.version import LooseVersion from distutils.version import LooseVersion
from urllib.request import urlopen, urlretrieve from urllib.request import urlopen, urlretrieve
@ -32,12 +32,11 @@ from selenium.webdriver import ChromeOptions as _ChromeOptions
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
TARGET_VERSION = 0 TARGET_VERSION = 0
class Chrome: class Chrome:
def __new__(cls, *args, enable_console_log=False, **kwargs): def __new__(cls, *args, emulate_touch=False, **kwargs):
if not ChromeDriverManager.installed: if not ChromeDriverManager.installed:
ChromeDriverManager(*args, **kwargs).install() ChromeDriverManager(*args, **kwargs).install()
@ -73,11 +72,6 @@ class Chrome:
}) })
}); });
""" """
+ (
"console.log = console.dir = console.error = function(){};"
if not enable_console_log
else ""
)
}, },
) )
return instance._orig_get(*args, **kwargs) return instance._orig_get(*args, **kwargs)
@ -89,14 +83,31 @@ class Chrome:
) )
instance.execute_cdp_cmd( instance.execute_cdp_cmd(
"Network.setUserAgentOverride", "Network.setUserAgentOverride",
{"userAgent": original_user_agent_string.replace("Headless", ""),}, {
"userAgent": original_user_agent_string.replace("Headless", ""),
},
) )
if emulate_touch:
instance.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument",
{
"source": """
Object.defineProperty(navigator, 'maxTouchPoints', {
get: () => 1
})"""
},
)
logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})") logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})")
return instance return instance
class ChromeOptions: class ChromeOptions:
__doc__ = _ChromeOptions.__doc__
def __new__(cls, *args, **kwargs): def __new__(cls, *args, **kwargs):
__doc__ = _ChromeOptions.__new__.__doc__
if not ChromeDriverManager.installed: if not ChromeDriverManager.installed:
ChromeDriverManager(*args, **kwargs).install() ChromeDriverManager(*args, **kwargs).install()
if not ChromeDriverManager.selenium_patched: if not ChromeDriverManager.selenium_patched:
@ -111,7 +122,6 @@ class ChromeOptions:
class ChromeDriverManager(object): class ChromeDriverManager(object):
installed = False installed = False
selenium_patched = False selenium_patched = False
target_version = None target_version = None
@ -224,10 +234,10 @@ class ChromeDriverManager(object):
@staticmethod @staticmethod
def random_cdc(): def random_cdc():
cdc = random.choices(string.ascii_lowercase, k=26) cdc = random.choices(string.ascii_lowercase, k=26)
cdc[-6: -4] = map(str.upper, cdc[-6: -4]) cdc[-6:-4] = map(str.upper, cdc[-6:-4])
cdc[2] = cdc[0] cdc[2] = cdc[0]
cdc[3] = '_' cdc[3] = "_"
return ''.join(cdc).encode() return "".join(cdc).encode()
def patch_binary(self): def patch_binary(self):
""" """

View File

@ -46,8 +46,6 @@ import tempfile
import threading import threading
import time import time
import zipfile import zipfile
import atexit
import contextlib
from distutils.version import LooseVersion from distutils.version import LooseVersion
from urllib.request import urlopen, urlretrieve from urllib.request import urlopen, urlretrieve
@ -76,8 +74,10 @@ def find_chrome_executable():
for item in os.environ.get("PATH").split(os.pathsep): for item in os.environ.get("PATH").split(os.pathsep):
for subitem in ("google-chrome", "chromium", "chromium-browser"): for subitem in ("google-chrome", "chromium", "chromium-browser"):
candidates.add(os.sep.join((item, subitem))) candidates.add(os.sep.join((item, subitem)))
if 'darwin' in sys.platform: if "darwin" in sys.platform:
candidates.update(["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]) candidates.update(
["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"]
)
else: else:
for item in map( for item in map(
os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA") os.environ.get, ("PROGRAMFILES", "PROGRAMFILES(X86)", "LOCALAPPDATA")
@ -94,16 +94,18 @@ def find_chrome_executable():
class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
__doc__ = (
__doc__ = """\ """\
-------------------------------------------------------------------------- --------------------------------------------------------------------------
NOTE: NOTE:
Chrome has everything included to work out of the box. Chrome has everything included to work out of the box.
it does not `need` customizations. it does not `need` customizations.
any customizations MAY lead to trigger bot migitation systems. any customizations MAY lead to trigger bot migitation systems.
-------------------------------------------------------------------------- --------------------------------------------------------------------------
""" + selenium.webdriver.remote.webdriver.WebDriver.__doc__ """
+ selenium.webdriver.remote.webdriver.WebDriver.__doc__
)
_instances = set() _instances = set()
@ -121,14 +123,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
user_data_dir=None, user_data_dir=None,
factor=0.5, factor=0.5,
delay=1, delay=1,
): emulate_touch=False,
):
self._data_dir_default = True self._data_dir_default = True
if user_data_dir: if user_data_dir:
self._data_dir_default = False self._data_dir_default = False
p = Patcher(target_path=executable_path) p = Patcher(target_path=executable_path)
p.auto(False) p.auto(False)
self._patcher = p self._patcher = p
self.factor = factor self.factor = factor
self.delay = delay self.delay = delay
@ -169,6 +172,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
extra_args = [] extra_args = []
if options.headless: if options.headless:
extra_args.append("--headless") extra_args.append("--headless")
options.add_argument("start-maximized")
self.browser_args = [ self.browser_args = [
find_chrome_executable(), find_chrome_executable(),
@ -199,16 +203,63 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
keep_alive=keep_alive, keep_alive=keep_alive,
) )
if options.headless:
orig_get = self.get
def get_wrapped(*args, **kwargs):
if self.execute_script("return navigator.webdriver"):
self.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument",
{
"source": """
Object.defineProperty(window, 'navigator', {
value: new Proxy(navigator, {
has: (target, key) => (key === 'webdriver' ? false : key in target),
get: (target, key) =>
key === 'webdriver'
? undefined
: typeof target[key] === 'function'
? target[key].bind(target)
: target[key]
})
});
"""
},
)
self.execute_cdp_cmd(
"Network.setUserAgentOverride",
{
"userAgent": self.execute_script(
"return navigator.userAgent"
).replace("Headless", "")
},
)
return orig_get(*args, **kwargs)
self.get = get_wrapped
if emulate_touch:
self.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument",
{
"source": """
Object.defineProperty(navigator, 'maxTouchPoints', {
get: () => 1
})"""
},
)
def start_session(self, capabilities=None, browser_profile=None): def start_session(self, capabilities=None, browser_profile=None):
if not capabilities: if not capabilities:
capabilities = self.options.to_capabilities() capabilities = self.options.to_capabilities()
super().start_session(capabilities, browser_profile) super().start_session(capabilities, browser_profile)
def get_in(self, url: str, delay=2.5, factor=1): def get_in(self, url: str, delay=2.5):
""" """
:param url: str :param url: str
:param delay: int :param delay: disconnect <delay> seconds after .get()
:param factor: disconnect <factor> seconds after .get()
too low will disconnect before get() fired. too low will disconnect before get() fired.
================================================= =================================================
@ -238,8 +289,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
self.get(url) self.get(url)
finally: finally:
self.close() self.close()
# threading.Timer(factor or self.factor, self.close).start() threading.Timer(delay, self.close).start()
time.sleep(delay or self.delay) time.sleep(delay)
self.start_session() self.start_session()
def quit(self): def quit(self):
@ -279,9 +330,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
class Patcher(object): class Patcher(object):
url_repo = "https://chromedriver.storage.googleapis.com" url_repo = "https://chromedriver.storage.googleapis.com"
def __init__(self, target_path='./chromedriver', force=False, version_main: int = 0): def __init__(
# if not target_path: self, target_path="./chromedriver", force=False, version_main: int = 0
# target_path = os.path.join(tempfile.gettempdir(), 'undetected_chromedriver', 'chromedriver') ):
if not IS_POSIX: if not IS_POSIX:
if not target_path[-4:] == ".exe": if not target_path[-4:] == ".exe":
target_path += ".exe" target_path += ".exe"
@ -327,14 +378,7 @@ class Patcher(object):
:return: version string :return: version string
:rtype: LooseVersion :rtype: LooseVersion
""" """
path = ( path = ("/latest_release" if not self.version_main else f"/latest_release_{self.version_main}").upper()
"/"
+ (
"latest_release"
if not self.version_main
else f"latest_release_{self.version_main}"
).upper()
)
logger.debug("getting release number from %s" % path) logger.debug("getting release number from %s" % path)
return LooseVersion(urlopen(self.url_repo + path).read().decode()) return LooseVersion(urlopen(self.url_repo + path).read().decode())
@ -367,7 +411,7 @@ class Patcher(object):
os.makedirs(os.path.dirname(self.target_path), mode=0o755) os.makedirs(os.path.dirname(self.target_path), mode=0o755)
except OSError: except OSError:
pass pass
with zipfile.ZipFile(self.zipname, mode='r') as zf: with zipfile.ZipFile(self.zipname, mode="r") as zf:
zf.extract(self.exename) zf.extract(self.exename)
os.rename(self.exename, self.target_path) os.rename(self.exename, self.target_path)
os.remove(self.zipname) os.remove(self.zipname)
@ -436,7 +480,6 @@ class Patcher(object):
:return: False on failure, binary name on success :return: False on failure, binary name on success
""" """
logger.info("patching driver executable %s" % self.target_path) logger.info("patching driver executable %s" % self.target_path)
linect = 0 linect = 0
@ -449,6 +492,7 @@ class Patcher(object):
fh.write(newline) fh.write(newline)
linect += 1 linect += 1
return linect return linect
class ChromeOptions(selenium.webdriver.chrome.webdriver.Options): class ChromeOptions(selenium.webdriver.chrome.webdriver.Options):
pass pass