diff --git a/README.md b/README.md
index 7c6132c..9901099 100644
--- a/README.md
+++ b/README.md
@@ -1,138 +1,1106 @@
-# undetected_chromedriver #
-
-https://github.com/ultrafunkamsterdam/undetected-chromedriver
-
-Optimized Selenium Chromedriver patch which does not trigger anti-bot services like Distill Network / Imperva / DataDome / Botprotect.io
-Automatically downloads the driver binary and patches it.
-
-* **Tested until current chrome beta versions**
-* **Works also on Brave Browser and many other Chromium based browsers**
-* **Python 3.6++**
-
-## Installation ##
-```
-pip install undetected-chromedriver
-```
-
-## Usage ##
-
-To prevent unnecessary hair-pulling and issue-raising, please mind the **[important note at the end of this document](#important-note) .**
-
-
-
-#### The Version 2 way ####
-Literally, this is all you have to do. Settings are included and your browser executable found automagically.
-
-```python
-import undetected_chromedriver.v2 as uc
-driver = uc.Chrome()
-with driver:
- driver.get('https://coinfaucet.eu') # known url using cloudflare's "under attack mode"
-```
-
-
-
-
-
-
-
-#### the easy way (v1 old stuff) ####
-```python
-import undetected_chromedriver as uc
-driver = uc.Chrome()
-driver.get('https://distilnetworks.com')
-```
-
-
-
-#### target specific chrome version (v1 old stuff) ####
-```python
-import undetected_chromedriver as uc
-uc.TARGET_VERSION = 85
-driver = uc.Chrome()
-```
-
-
-#### monkeypatch mode (v1 old stuff) ####
-Needs to be done before importing from selenium package
-
-```python
-import undetected_chromedriver as uc
-uc.install()
-
-from selenium.webdriver import Chrome
-driver = Chrome()
-driver.get('https://distilnetworks.com')
-
-```
-
-
-#### the customized way (v1 old stuff) ####
-```python
-import undetected_chromedriver as uc
-
-#specify chromedriver version to download and patch
-uc.TARGET_VERSION = 78
-
-# or specify your own chromedriver binary (why you would need this, i don't know)
-
-uc.install(
- executable_path='c:/users/user1/chromedriver.exe',
-)
-
-opts = uc.ChromeOptions()
-opts.add_argument(f'--proxy-server=socks5://127.0.0.1:9050')
-driver = uc.Chrome(options=opts)
-driver.get('https://distilnetworks.com')
-```
-
-
-#### datadome.co example (v1 old stuff) ####
-These guys have actually a powerful product, and a link to this repo, which makes me wanna test their product.
-Make sure you use a "clean" ip for this one.
-```python
-#
-# STANDARD selenium Chromedriver
-#
-from selenium import webdriver
-chrome = webdriver.Chrome()
-chrome.get('https://datadome.co/customers-stories/toppreise-ends-web-scraping-and-content-theft-with-datadome/')
-chrome.save_screenshot('datadome_regular_webdriver.png')
-True # it caused my ip to be flagged, unfortunately
-
-
-#
-# UNDETECTED chromedriver (headless,even)
-#
-import undetected_chromedriver as uc
-options = uc.ChromeOptions()
-options.headless=True
-options.add_argument('--headless')
-chrome = uc.Chrome(options=options)
-chrome.get('https://datadome.co/customers-stories/toppreise-ends-web-scraping-and-content-theft-with-datadome/')
-chrome.save_screenshot('datadome_undetected_webddriver.png')
-
-```
-**Check both saved screenhots [here](https://imgur.com/a/fEmqadP)**
-
-
-
-## important note (v1 old stuff) ####
-
-Due to the inner workings of the module, it is needed to browse programmatically (ie: using .get(url) ). Never use the gui to navigate. Using your keybord and mouse for navigation causes possible detection! New Tabs: same story. If you really need multi-tabs, then open the tab with the blank page (hint: url is `data:,` including comma, and yes, driver accepts it) and do your thing as usual. If you follow these "rules" (actually its default behaviour), then you will have a great time for now.
-
-TL;DR and for the visual-minded:
-
-```python
-In [1]: import undetected_chromedriver as uc
-In [2]: driver = uc.Chrome()
-In [3]: driver.execute_script('return navigator.webdriver')
-Out[3]: True # Detectable
-In [4]: driver.get('https://distilnetworks.com') # starts magic
-In [4]: driver.execute_script('return navigator.webdriver')
-In [5]: None # Undetectable!
-```
-## end important note ##
-
-
-
+
+# undetected_chromedriver #
+
+https://github.com/ultrafunkamsterdam/undetected-chromedriver
+
+Optimized Selenium Chromedriver patch which does not trigger anti-bot services like Distill Network / Imperva / DataDome / Botprotect.io
+Automatically downloads the driver binary and patches it.
+
+* **Tested until current chrome beta versions**
+* **Works also on Brave Browser and many other Chromium based browsers**
+* **Python 3.6++**
+
+## Installation ##
+```
+pip install undetected-chromedriver
+```
+
+## Usage ##
+
+To prevent unnecessary hair-pulling and issue-raising, please mind the **[important note at the end of this document](#important-note) .**
+
+
+
+### The Version 2 way ###
+Literally, this is all you have to do.
+Settings are included and your browser executable is found automagically.
+This is also the snippet i recommend using in case you experience an issue.
+```python
+import undetected_chromedriver.v2 as uc
+driver = uc.Chrome()
+with driver:
+ driver.get('https://nowsecure.nl') # known url using cloudflare's "under attack mode"
+```
+
+### The Version 2 more advanced way, including setting profie folder ###
+Literally, this is all you have to do.
+If a specified folder does not exist, a NEW profile is created.
+Data dirs which are specified like this will not be autoremoved on exit.
+
+
+```python
+import undetected_chromedriver.v2 as uc
+options = uc.ChromeOptions()
+
+# setting profile
+options.user_data_dir = "c:\\temp\\profile"
+
+# another way to set profile is the below (which takes precedence if both variants are used
+options.add_argument('--user-data-dir=c:\\temp\\profile2')
+
+# just some options passing in to skip annoying popups
+options.add_argument('--no-first-run --no-service-autorun --password-store=basic')
+driver = uc.Chrome(options=options)
+
+with driver:
+ driver.get('https://nowsecure.nl') # known url using cloudflare's "under attack mode"
+
+```
+
+
+### The Version 2 expert mode, including Devtool/Wire events! ###
+Literally, this is all you have to do.
+You can now listen and subscribe to the low level devtools-protocol.
+I just recently found out that is also on planning for future release of the official chromedriver.
+However i implemented my own for now. Since i needed it myself for investigation.
+
+
+```python
+
+import undetected_chromedriver.v2 as uc
+from pprint import pformat
+
+driver = uc.Chrome(enable_cdp_event=True)
+
+def mylousyprintfunction(eventdata):
+ print(pformat(eventdata))
+
+# set the callback to Network.dataReceived to print (yeah not much original)
+driver.add_cdp_listener("Network.dataReceived", mylousyprintfunction)
+driver.get('https://nowsecure.nl') # known url using cloudflare's "under attack mode"
+
+
+def mylousyprintfunction(message):
+ print(pformat(message))
+
+
+# for more inspiration checkout the link below
+# https://chromedevtools.github.io/devtools-protocol/1-3/Network/
+
+# and of couse 2 lousy examples
+driver.add_cdp_listener('Network.requestWillBeSent', mylousyprintfunction)
+driver.add_cdp_listener('Network.dataReceived', mylousyprintfunction)
+
+# hint: a wildcard captures all events!
+# driver.add_cdp_listener('*', mylousyprintfunction)
+
+# now all these events will be printed in my console
+
+with driver:
+ driver.get('https://nowsecure.nl')
+
+
+{'method': 'Network.requestWillBeSent',
+ 'params': {'documentURL': 'https://nowsecure.nl/',
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'hasUserGesture': False,
+ 'initiator': {'type': 'other'},
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'request': {'headers': {'Upgrade-Insecure-Requests': '1',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT '
+ '10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0'},
+ 'initialPriority': 'VeryHigh',
+ 'method': 'GET',
+ 'mixedContentType': 'none',
+ 'referrerPolicy': 'strict-origin-when-cross-origin',
+ 'url': 'https://nowsecure.nl/'},
+ 'requestId': '449906A5C736D819123288133F2797E6',
+ 'timestamp': 190010.996717,
+ 'type': 'Document',
+ 'wallTime': 1621835932.112026}}
+{'method': 'Network.requestWillBeSentExtraInfo',
+ 'params': {'associatedCookies': [],
+ 'headers': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/',
+ ':scheme': 'https',
+ 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+ 'accept-encoding': 'gzip, deflate, br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'document',
+ 'sec-fetch-mode': 'navigate',
+ 'sec-fetch-site': 'none',
+ 'sec-fetch-user': '?1',
+ 'upgrade-insecure-requests': '1',
+ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+ 'x64) AppleWebKit/537.36 (KHTML, like '
+ 'Gecko) Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'requestId': '449906A5C736D819123288133F2797E6'}}
+{'method': 'Network.responseReceivedExtraInfo',
+ 'params': {'blockedCookies': [],
+ 'headers': {'alt-svc': 'h3-27=":443"; ma=86400, h3-28=":443"; '
+ 'ma=86400, h3-29=":443"; ma=86400',
+ 'cache-control': 'private, max-age=0, no-store, '
+ 'no-cache, must-revalidate, '
+ 'post-check=0, pre-check=0',
+ 'cf-ray': '65444b779ae6546f-LHR',
+ 'cf-request-id': '0a3e8d7eba0000546ffd3fa000000001',
+ 'content-type': 'text/html; charset=UTF-8',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'expect-ct': 'max-age=604800, '
+ 'report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+ 'expires': 'Thu, 01 Jan 1970 00:00:01 GMT',
+ 'nel': '{"report_to":"cf-nel","max_age":604800}',
+ 'permissions-policy': 'accelerometer=(),autoplay=(),camera=(),clipboard-read=(),clipboard-write=(),fullscreen=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-xhr=(),usb=()',
+ 'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report?s=CAfobYlmWImQ90e%2B4BFBhpPYL%2FyGyBvkcWAj%2B%2FVOLoEq0NVrD5jU9m5pi%2BKI%2BOAnINLPXOCoX2psLphA5Z38aZzWNr3eW%2BDTIK%2FQidc%3D"}],"group":"cf-nel","max_age":604800}',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-frame-options': 'SAMEORIGIN'},
+ 'requestId': '449906A5C736D819123288133F2797E6',
+ 'resourceIPAddressSpace': 'Public'}}
+{'method': 'Network.responseReceived',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'requestId': '449906A5C736D819123288133F2797E6',
+ 'response': {'connectionId': 158,
+ 'connectionReused': False,
+ 'encodedDataLength': 851,
+ 'fromDiskCache': False,
+ 'fromPrefetchCache': False,
+ 'fromServiceWorker': False,
+ 'headers': {'alt-svc': 'h3-27=":443"; ma=86400, '
+ 'h3-28=":443"; ma=86400, '
+ 'h3-29=":443"; ma=86400',
+ 'cache-control': 'private, max-age=0, '
+ 'no-store, no-cache, '
+ 'must-revalidate, '
+ 'post-check=0, '
+ 'pre-check=0',
+ 'cf-ray': '65444b779ae6546f-LHR',
+ 'cf-request-id': '0a3e8d7eba0000546ffd3fa000000001',
+ 'content-type': 'text/html; charset=UTF-8',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'expect-ct': 'max-age=604800, '
+ 'report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+ 'expires': 'Thu, 01 Jan 1970 00:00:01 GMT',
+ 'nel': '{"report_to":"cf-nel","max_age":604800}',
+ 'permissions-policy': 'accelerometer=(),autoplay=(),camera=(),clipboard-read=(),clipboard-write=(),fullscreen=(),geolocation=(),gyroscope=(),hid=(),interest-cohort=(),magnetometer=(),microphone=(),payment=(),publickey-credentials-get=(),screen-wake-lock=(),serial=(),sync-xhr=(),usb=()',
+ 'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report?s=CAfobYlmWImQ90e%2B4BFBhpPYL%2FyGyBvkcWAj%2B%2FVOLoEq0NVrD5jU9m5pi%2BKI%2BOAnINLPXOCoX2psLphA5Z38aZzWNr3eW%2BDTIK%2FQidc%3D"}],"group":"cf-nel","max_age":604800}',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-frame-options': 'SAMEORIGIN'},
+ 'mimeType': 'text/html',
+ 'protocol': 'h2',
+ 'remoteIPAddress': '104.21.5.197',
+ 'remotePort': 443,
+ 'requestHeaders': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/',
+ ':scheme': 'https',
+ 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+ 'accept-encoding': 'gzip, deflate, '
+ 'br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'sec-ch-ua': '" Not '
+ 'A;Brand";v="99", '
+ '"Chromium";v="90", '
+ '"Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'document',
+ 'sec-fetch-mode': 'navigate',
+ 'sec-fetch-site': 'none',
+ 'sec-fetch-user': '?1',
+ 'upgrade-insecure-requests': '1',
+ 'user-agent': 'Mozilla/5.0 '
+ '(Windows NT 10.0; '
+ 'Win64; x64) '
+ 'AppleWebKit/537.36 '
+ '(KHTML, like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'responseTime': 1621835932177.923,
+ 'securityDetails': {'certificateId': 0,
+ 'certificateTransparencyCompliance': 'compliant',
+ 'cipher': 'AES_128_GCM',
+ 'issuer': 'Cloudflare Inc ECC '
+ 'CA-3',
+ 'keyExchange': '',
+ 'keyExchangeGroup': 'X25519',
+ 'protocol': 'TLS 1.3',
+ 'sanList': ['sni.cloudflaressl.com',
+ '*.nowsecure.nl',
+ 'nowsecure.nl'],
+ 'signedCertificateTimestampList': [{'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'Google '
+ "'Argon2021' "
+ 'log',
+ 'logId': 'F65C942FD1773022145418083094568EE34D131933BFDF0C2F200BCC4EF164E3',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '30450221008A25458182A6E7F608FE1492086762A367381E94137952FFD621BA2E60F7E2F702203BCDEBCE1C544DECF0A113DE12B33E299319E6240426F38F08DFC04EF2E42825',
+ 'status': 'Verified',
+ 'timestamp': 1598706372839.0},
+ {'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'DigiCert '
+ 'Yeti2021 '
+ 'Log',
+ 'logId': '5CDC4392FEE6AB4544B15E9AD456E61037FBD5FA47DCA17394B25EE6F6C70ECA',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '3046022100A95A49C7435DBFC73406AC409062C27269E6E69F443A2213F3A085E3BCBD234A022100DEA878296F8A1DB43546DC1865A4C5AD2B90664A243AE0A3A6D4925802EE68A8',
+ 'status': 'Verified',
+ 'timestamp': 1598706372823.0}],
+ 'subjectName': 'sni.cloudflaressl.com',
+ 'validFrom': 1598659200,
+ 'validTo': 1630238400},
+ 'securityState': 'secure',
+ 'status': 503,
+ 'statusText': '',
+ 'timing': {'connectEnd': 40.414,
+ 'connectStart': 0,
+ 'dnsEnd': 0,
+ 'dnsStart': 0,
+ 'proxyEnd': -1,
+ 'proxyStart': -1,
+ 'pushEnd': 0,
+ 'pushStart': 0,
+ 'receiveHeadersEnd': 60.361,
+ 'requestTime': 190011.002239,
+ 'sendEnd': 41.348,
+ 'sendStart': 41.19,
+ 'sslEnd': 40.405,
+ 'sslStart': 10.853,
+ 'workerFetchStart': -1,
+ 'workerReady': -1,
+ 'workerRespondWithSettled': -1,
+ 'workerStart': -1},
+ 'url': 'https://nowsecure.nl/'},
+ 'timestamp': 190011.06449,
+ 'type': 'Document'}}
+{'method': 'Page.frameStartedLoading',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700'}}
+{'method': 'Page.frameNavigated',
+ 'params': {'frame': {'adFrameType': 'none',
+ 'crossOriginIsolatedContextType': 'NotIsolated',
+ 'domainAndRegistry': 'nowsecure.nl',
+ 'gatedAPIFeatures': ['SharedArrayBuffers',
+ 'SharedArrayBuffersTransferAllowed'],
+ 'id': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'mimeType': 'text/html',
+ 'secureContextType': 'Secure',
+ 'securityOrigin': 'https://nowsecure.nl',
+ 'url': 'https://nowsecure.nl/'}}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 9835,
+ 'encodedDataLength': 0,
+ 'requestId': '449906A5C736D819123288133F2797E6',
+ 'timestamp': 190011.093343}}
+{'method': 'Network.loadingFinished',
+ 'params': {'encodedDataLength': 10713,
+ 'requestId': '449906A5C736D819123288133F2797E6',
+ 'shouldReportCorbBlocking': False,
+ 'timestamp': 190011.064011}}
+{'method': 'Network.requestWillBeSent',
+ 'params': {'documentURL': 'https://nowsecure.nl/',
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'hasUserGesture': False,
+ 'initiator': {'stack': {'callFrames': [{'columnNumber': 51,
+ 'functionName': '',
+ 'lineNumber': 114,
+ 'scriptId': '8',
+ 'url': 'https://nowsecure.nl/'},
+ {'columnNumber': 9,
+ 'functionName': '',
+ 'lineNumber': 115,
+ 'scriptId': '8',
+ 'url': 'https://nowsecure.nl/'}]},
+ 'type': 'script'},
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'request': {'headers': {'Referer': 'https://nowsecure.nl/',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT '
+ '10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0'},
+ 'initialPriority': 'Low',
+ 'method': 'GET',
+ 'mixedContentType': 'none',
+ 'referrerPolicy': 'strict-origin-when-cross-origin',
+ 'url': 'https://nowsecure.nl/cdn-cgi/challenge-platform/h/b/orchestrate/jsch/v1?ray=65444b779ae6546f'},
+ 'requestId': '17180.2',
+ 'timestamp': 190011.106133,
+ 'type': 'Script',
+ 'wallTime': 1621835932.221325}}
+{'method': 'Network.requestWillBeSent',
+ 'params': {'documentURL': 'https://nowsecure.nl/',
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'hasUserGesture': False,
+ 'initiator': {'columnNumber': 13,
+ 'lineNumber': 117,
+ 'type': 'parser',
+ 'url': 'https://nowsecure.nl/'},
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'request': {'headers': {'Referer': 'https://nowsecure.nl/',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT '
+ '10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0'},
+ 'initialPriority': 'Low',
+ 'method': 'GET',
+ 'mixedContentType': 'none',
+ 'referrerPolicy': 'strict-origin-when-cross-origin',
+ 'url': 'https://nowsecure.nl/cdn-cgi/images/trace/jschal/js/transparent.gif?ray=65444b779ae6546f'},
+ 'requestId': '17180.3',
+ 'timestamp': 190011.106911,
+ 'type': 'Image',
+ 'wallTime': 1621835932.222102}}
+{'method': 'Network.requestWillBeSent',
+ 'params': {'documentURL': 'https://nowsecure.nl/',
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'hasUserGesture': False,
+ 'initiator': {'type': 'parser', 'url': 'https://nowsecure.nl/'},
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'request': {'headers': {'Referer': 'https://nowsecure.nl/',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT '
+ '10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0'},
+ 'initialPriority': 'Low',
+ 'method': 'GET',
+ 'mixedContentType': 'none',
+ 'referrerPolicy': 'strict-origin-when-cross-origin',
+ 'url': 'https://nowsecure.nl/cdn-cgi/images/trace/jschal/nojs/transparent.gif?ray=65444b779ae6546f'},
+ 'requestId': '17180.4',
+ 'timestamp': 190011.109527,
+ 'type': 'Image',
+ 'wallTime': 1621835932.224719}}
+{'method': 'Page.domContentEventFired', 'params': {'timestamp': 190011.110345}}
+{'method': 'Network.requestWillBeSentExtraInfo',
+ 'params': {'associatedCookies': [],
+ 'clientSecurityState': {'initiatorIPAddressSpace': 'Public',
+ 'initiatorIsSecureContext': True,
+ 'privateNetworkRequestPolicy': 'WarnFromInsecureToMorePrivate'},
+ 'headers': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/images/trace/jschal/js/transparent.gif?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+ 'accept-encoding': 'gzip, deflate, br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'image',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+ 'x64) AppleWebKit/537.36 (KHTML, like '
+ 'Gecko) Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'requestId': '17180.3'}}
+{'method': 'Network.requestWillBeSentExtraInfo',
+ 'params': {'associatedCookies': [],
+ 'clientSecurityState': {'initiatorIPAddressSpace': 'Public',
+ 'initiatorIsSecureContext': True,
+ 'privateNetworkRequestPolicy': 'WarnFromInsecureToMorePrivate'},
+ 'headers': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/challenge-platform/h/b/orchestrate/jsch/v1?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': '*/*',
+ 'accept-encoding': 'gzip, deflate, br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'script',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+ 'x64) AppleWebKit/537.36 (KHTML, like '
+ 'Gecko) Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'requestId': '17180.2'}}
+{'method': 'Network.requestWillBeSentExtraInfo',
+ 'params': {'associatedCookies': [],
+ 'clientSecurityState': {'initiatorIPAddressSpace': 'Public',
+ 'initiatorIsSecureContext': True,
+ 'privateNetworkRequestPolicy': 'WarnFromInsecureToMorePrivate'},
+ 'headers': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/images/trace/jschal/nojs/transparent.gif?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+ 'accept-encoding': 'gzip, deflate, br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'image',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+ 'x64) AppleWebKit/537.36 (KHTML, like '
+ 'Gecko) Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'requestId': '17180.4'}}
+{'method': 'Network.responseReceivedExtraInfo',
+ 'params': {'blockedCookies': [],
+ 'headers': {'accept-ranges': 'bytes',
+ 'cache-control': 'max-age=7200\npublic',
+ 'cf-ray': '65444b781d1de604-LHR',
+ 'content-length': '42',
+ 'content-type': 'image/gif',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'etag': '"60a4d856-2a"',
+ 'expires': 'Mon, 24 May 2021 07:58:53 GMT',
+ 'last-modified': 'Wed, 19 May 2021 09:20:22 GMT',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-content-type-options': 'nosniff',
+ 'x-frame-options': 'DENY'},
+ 'requestId': '17180.3',
+ 'resourceIPAddressSpace': 'Public'}}
+{'method': 'Network.responseReceivedExtraInfo',
+ 'params': {'blockedCookies': [],
+ 'headers': {'accept-ranges': 'bytes',
+ 'cache-control': 'max-age=7200\npublic',
+ 'cf-ray': '65444b781d1fe604-LHR',
+ 'content-length': '42',
+ 'content-type': 'image/gif',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'etag': '"60a4d856-2a"',
+ 'expires': 'Mon, 24 May 2021 07:58:53 GMT',
+ 'last-modified': 'Wed, 19 May 2021 09:20:22 GMT',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-content-type-options': 'nosniff',
+ 'x-frame-options': 'DENY'},
+ 'requestId': '17180.4',
+ 'resourceIPAddressSpace': 'Public'}}
+{'method': 'Network.resourceChangedPriority',
+ 'params': {'newPriority': 'High',
+ 'requestId': '17180.4',
+ 'timestamp': 190011.171057}}
+{'method': 'Network.responseReceived',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'requestId': '17180.3',
+ 'response': {'connectionId': 0,
+ 'connectionReused': False,
+ 'encodedDataLength': 214,
+ 'fromDiskCache': False,
+ 'fromPrefetchCache': False,
+ 'fromServiceWorker': False,
+ 'headers': {'accept-ranges': 'bytes',
+ 'cache-control': 'max-age=7200\npublic',
+ 'cf-ray': '65444b781d1de604-LHR',
+ 'content-length': '42',
+ 'content-type': 'image/gif',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'etag': '"60a4d856-2a"',
+ 'expires': 'Mon, 24 May 2021 07:58:53 GMT',
+ 'last-modified': 'Wed, 19 May 2021 '
+ '09:20:22 GMT',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-content-type-options': 'nosniff',
+ 'x-frame-options': 'DENY'},
+ 'mimeType': 'image/gif',
+ 'protocol': 'h3-29',
+ 'remoteIPAddress': '104.21.5.197',
+ 'remotePort': 443,
+ 'requestHeaders': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/images/trace/jschal/js/transparent.gif?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+ 'accept-encoding': 'gzip, deflate, '
+ 'br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not '
+ 'A;Brand";v="99", '
+ '"Chromium";v="90", '
+ '"Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'image',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 '
+ '(Windows NT 10.0; '
+ 'Win64; x64) '
+ 'AppleWebKit/537.36 '
+ '(KHTML, like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'responseTime': 1621835932265.169,
+ 'securityDetails': {'certificateId': 0,
+ 'certificateTransparencyCompliance': 'compliant',
+ 'cipher': 'AES_128_GCM',
+ 'issuer': 'Cloudflare Inc ECC '
+ 'CA-3',
+ 'keyExchange': '',
+ 'keyExchangeGroup': 'X25519',
+ 'protocol': 'QUIC',
+ 'sanList': ['sni.cloudflaressl.com',
+ '*.nowsecure.nl',
+ 'nowsecure.nl'],
+ 'signedCertificateTimestampList': [{'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'Google '
+ "'Argon2021' "
+ 'log',
+ 'logId': 'F65C942FD1773022145418083094568EE34D131933BFDF0C2F200BCC4EF164E3',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '30450221008A25458182A6E7F608FE1492086762A367381E94137952FFD621BA2E60F7E2F702203BCDEBCE1C544DECF0A113DE12B33E299319E6240426F38F08DFC04EF2E42825',
+ 'status': 'Verified',
+ 'timestamp': 1598706372839.0},
+ {'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'DigiCert '
+ 'Yeti2021 '
+ 'Log',
+ 'logId': '5CDC4392FEE6AB4544B15E9AD456E61037FBD5FA47DCA17394B25EE6F6C70ECA',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '3046022100A95A49C7435DBFC73406AC409062C27269E6E69F443A2213F3A085E3BCBD234A022100DEA878296F8A1DB43546DC1865A4C5AD2B90664A243AE0A3A6D4925802EE68A8',
+ 'status': 'Verified',
+ 'timestamp': 1598706372823.0}],
+ 'subjectName': 'sni.cloudflaressl.com',
+ 'validFrom': 1598659200,
+ 'validTo': 1630238400},
+ 'securityState': 'secure',
+ 'status': 200,
+ 'statusText': '',
+ 'timing': {'connectEnd': 26.087,
+ 'connectStart': 0,
+ 'dnsEnd': 0,
+ 'dnsStart': 0,
+ 'proxyEnd': -1,
+ 'proxyStart': -1,
+ 'pushEnd': 0,
+ 'pushStart': 0,
+ 'receiveHeadersEnd': 40.709,
+ 'requestTime': 190011.109386,
+ 'sendEnd': 26.346,
+ 'sendStart': 26.182,
+ 'sslEnd': 26.087,
+ 'sslStart': 0,
+ 'workerFetchStart': -1,
+ 'workerReady': -1,
+ 'workerRespondWithSettled': -1,
+ 'workerStart': -1},
+ 'url': 'https://nowsecure.nl/cdn-cgi/images/trace/jschal/js/transparent.gif?ray=65444b779ae6546f'},
+ 'timestamp': 190011.174536,
+ 'type': 'Image'}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 42,
+ 'encodedDataLength': 0,
+ 'requestId': '17180.3',
+ 'timestamp': 190011.174737}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 0,
+ 'encodedDataLength': 44,
+ 'requestId': '17180.3',
+ 'timestamp': 190011.17524}}
+{'method': 'Network.loadingFinished',
+ 'params': {'encodedDataLength': 258,
+ 'requestId': '17180.3',
+ 'shouldReportCorbBlocking': False,
+ 'timestamp': 190011.152073}}
+{'method': 'Network.responseReceived',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'requestId': '17180.4',
+ 'response': {'connectionId': 0,
+ 'connectionReused': True,
+ 'encodedDataLength': 178,
+ 'fromDiskCache': False,
+ 'fromPrefetchCache': False,
+ 'fromServiceWorker': False,
+ 'headers': {'accept-ranges': 'bytes',
+ 'cache-control': 'max-age=7200\npublic',
+ 'cf-ray': '65444b781d1fe604-LHR',
+ 'content-length': '42',
+ 'content-type': 'image/gif',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'etag': '"60a4d856-2a"',
+ 'expires': 'Mon, 24 May 2021 07:58:53 GMT',
+ 'last-modified': 'Wed, 19 May 2021 '
+ '09:20:22 GMT',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding',
+ 'x-content-type-options': 'nosniff',
+ 'x-frame-options': 'DENY'},
+ 'mimeType': 'image/gif',
+ 'protocol': 'h3-29',
+ 'remoteIPAddress': '104.21.5.197',
+ 'remotePort': 443,
+ 'requestHeaders': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/images/trace/jschal/nojs/transparent.gif?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+ 'accept-encoding': 'gzip, deflate, '
+ 'br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not '
+ 'A;Brand";v="99", '
+ '"Chromium";v="90", '
+ '"Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'image',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 '
+ '(Windows NT 10.0; '
+ 'Win64; x64) '
+ 'AppleWebKit/537.36 '
+ '(KHTML, like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'responseTime': 1621835932268.067,
+ 'securityDetails': {'certificateId': 0,
+ 'certificateTransparencyCompliance': 'compliant',
+ 'cipher': 'AES_128_GCM',
+ 'issuer': 'Cloudflare Inc ECC '
+ 'CA-3',
+ 'keyExchange': '',
+ 'keyExchangeGroup': 'X25519',
+ 'protocol': 'QUIC',
+ 'sanList': ['sni.cloudflaressl.com',
+ '*.nowsecure.nl',
+ 'nowsecure.nl'],
+ 'signedCertificateTimestampList': [{'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'Google '
+ "'Argon2021' "
+ 'log',
+ 'logId': 'F65C942FD1773022145418083094568EE34D131933BFDF0C2F200BCC4EF164E3',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '30450221008A25458182A6E7F608FE1492086762A367381E94137952FFD621BA2E60F7E2F702203BCDEBCE1C544DECF0A113DE12B33E299319E6240426F38F08DFC04EF2E42825',
+ 'status': 'Verified',
+ 'timestamp': 1598706372839.0},
+ {'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'DigiCert '
+ 'Yeti2021 '
+ 'Log',
+ 'logId': '5CDC4392FEE6AB4544B15E9AD456E61037FBD5FA47DCA17394B25EE6F6C70ECA',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '3046022100A95A49C7435DBFC73406AC409062C27269E6E69F443A2213F3A085E3BCBD234A022100DEA878296F8A1DB43546DC1865A4C5AD2B90664A243AE0A3A6D4925802EE68A8',
+ 'status': 'Verified',
+ 'timestamp': 1598706372823.0}],
+ 'subjectName': 'sni.cloudflaressl.com',
+ 'validFrom': 1598659200,
+ 'validTo': 1630238400},
+ 'securityState': 'secure',
+ 'status': 200,
+ 'statusText': '',
+ 'timing': {'connectEnd': -1,
+ 'connectStart': -1,
+ 'dnsEnd': -1,
+ 'dnsStart': -1,
+ 'proxyEnd': -1,
+ 'proxyStart': -1,
+ 'pushEnd': 0,
+ 'pushStart': 0,
+ 'receiveHeadersEnd': 42.415,
+ 'requestTime': 190011.110341,
+ 'sendEnd': 25.713,
+ 'sendStart': 25.609,
+ 'sslEnd': -1,
+ 'sslStart': -1,
+ 'workerFetchStart': -1,
+ 'workerReady': -1,
+ 'workerRespondWithSettled': -1,
+ 'workerStart': -1},
+ 'url': 'https://nowsecure.nl/cdn-cgi/images/trace/jschal/nojs/transparent.gif?ray=65444b779ae6546f'},
+ 'timestamp': 190011.175727,
+ 'type': 'Image'}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 42,
+ 'encodedDataLength': 0,
+ 'requestId': '17180.4',
+ 'timestamp': 190011.175856}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 0,
+ 'encodedDataLength': 44,
+ 'requestId': '17180.4',
+ 'timestamp': 190011.176133}}
+{'method': 'Network.loadingFinished',
+ 'params': {'encodedDataLength': 222,
+ 'requestId': '17180.4',
+ 'shouldReportCorbBlocking': False,
+ 'timestamp': 190011.153335}}
+{'method': 'Network.responseReceivedExtraInfo',
+ 'params': {'blockedCookies': [],
+ 'headers': {'alt-svc': 'h3-27=":443"; ma=86400, h3-28=":443"; '
+ 'ma=86400, h3-29=":443"; ma=86400',
+ 'cache-control': 'max-age=0, must-revalidate',
+ 'cf-ray': '65444b781d1ee604-LHR',
+ 'cf-request-id': '0a3e8d7f140000e60496387000000001',
+ 'content-encoding': 'br',
+ 'content-type': 'text/javascript',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'expect-ct': 'max-age=604800, '
+ 'report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+ 'nel': '{"report_to":"cf-nel","max_age":604800}',
+ 'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report?s=ZtI%2Bx8B7DpI8%2FsDA72maecFVCPvIsfBOyJjT8weyiqfmrHrmcBYpRhc%2FI%2F6JmIlnxW%2F%2BBohxLi1F8mpjAUabJ0kXLYnmjGKp2Ndio9M%3D"}],"group":"cf-nel","max_age":604800}',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding'},
+ 'requestId': '17180.2',
+ 'resourceIPAddressSpace': 'Public'}}
+{'method': 'Network.responseReceived',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'requestId': '17180.2',
+ 'response': {'connectionId': 0,
+ 'connectionReused': True,
+ 'encodedDataLength': 510,
+ 'fromDiskCache': False,
+ 'fromPrefetchCache': False,
+ 'fromServiceWorker': False,
+ 'headers': {'alt-svc': 'h3-27=":443"; ma=86400, '
+ 'h3-28=":443"; ma=86400, '
+ 'h3-29=":443"; ma=86400',
+ 'cache-control': 'max-age=0, '
+ 'must-revalidate',
+ 'cf-ray': '65444b781d1ee604-LHR',
+ 'cf-request-id': '0a3e8d7f140000e60496387000000001',
+ 'content-encoding': 'br',
+ 'content-type': 'text/javascript',
+ 'date': 'Mon, 24 May 2021 05:58:53 GMT',
+ 'expect-ct': 'max-age=604800, '
+ 'report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"',
+ 'nel': '{"report_to":"cf-nel","max_age":604800}',
+ 'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report?s=ZtI%2Bx8B7DpI8%2FsDA72maecFVCPvIsfBOyJjT8weyiqfmrHrmcBYpRhc%2FI%2F6JmIlnxW%2F%2BBohxLi1F8mpjAUabJ0kXLYnmjGKp2Ndio9M%3D"}],"group":"cf-nel","max_age":604800}',
+ 'server': 'cloudflare',
+ 'vary': 'Accept-Encoding'},
+ 'mimeType': 'text/javascript',
+ 'protocol': 'h3-29',
+ 'remoteIPAddress': '104.21.5.197',
+ 'remotePort': 443,
+ 'requestHeaders': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/cdn-cgi/challenge-platform/h/b/orchestrate/jsch/v1?ray=65444b779ae6546f',
+ ':scheme': 'https',
+ 'accept': '*/*',
+ 'accept-encoding': 'gzip, deflate, '
+ 'br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not '
+ 'A;Brand";v="99", '
+ '"Chromium";v="90", '
+ '"Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'script',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 '
+ '(Windows NT 10.0; '
+ 'Win64; x64) '
+ 'AppleWebKit/537.36 '
+ '(KHTML, like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+ 'responseTime': 1621835932301.817,
+ 'securityDetails': {'certificateId': 0,
+ 'certificateTransparencyCompliance': 'compliant',
+ 'cipher': 'AES_128_GCM',
+ 'issuer': 'Cloudflare Inc ECC '
+ 'CA-3',
+ 'keyExchange': '',
+ 'keyExchangeGroup': 'X25519',
+ 'protocol': 'QUIC',
+ 'sanList': ['sni.cloudflaressl.com',
+ '*.nowsecure.nl',
+ 'nowsecure.nl'],
+ 'signedCertificateTimestampList': [{'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'Google '
+ "'Argon2021' "
+ 'log',
+ 'logId': 'F65C942FD1773022145418083094568EE34D131933BFDF0C2F200BCC4EF164E3',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '30450221008A25458182A6E7F608FE1492086762A367381E94137952FFD621BA2E60F7E2F702203BCDEBCE1C544DECF0A113DE12B33E299319E6240426F38F08DFC04EF2E42825',
+ 'status': 'Verified',
+ 'timestamp': 1598706372839.0},
+ {'hashAlgorithm': 'SHA-256',
+ 'logDescription': 'DigiCert '
+ 'Yeti2021 '
+ 'Log',
+ 'logId': '5CDC4392FEE6AB4544B15E9AD456E61037FBD5FA47DCA17394B25EE6F6C70ECA',
+ 'origin': 'Embedded '
+ 'in '
+ 'certificate',
+ 'signatureAlgorithm': 'ECDSA',
+ 'signatureData': '3046022100A95A49C7435DBFC73406AC409062C27269E6E69F443A2213F3A085E3BCBD234A022100DEA878296F8A1DB43546DC1865A4C5AD2B90664A243AE0A3A6D4925802EE68A8',
+ 'status': 'Verified',
+ 'timestamp': 1598706372823.0}],
+ 'subjectName': 'sni.cloudflaressl.com',
+ 'validFrom': 1598659200,
+ 'validTo': 1630238400},
+ 'securityState': 'secure',
+ 'status': 200,
+ 'statusText': '',
+ 'timing': {'connectEnd': -1,
+ 'connectStart': -1,
+ 'dnsEnd': -1,
+ 'dnsStart': -1,
+ 'proxyEnd': -1,
+ 'proxyStart': -1,
+ 'pushEnd': 0,
+ 'pushStart': 0,
+ 'receiveHeadersEnd': 78.885,
+ 'requestTime': 190011.107975,
+ 'sendEnd': 27.934,
+ 'sendStart': 27.809,
+ 'sslEnd': -1,
+ 'sslStart': -1,
+ 'workerFetchStart': -1,
+ 'workerReady': -1,
+ 'workerRespondWithSettled': -1,
+ 'workerStart': -1},
+ 'url': 'https://nowsecure.nl/cdn-cgi/challenge-platform/h/b/orchestrate/jsch/v1?ray=65444b779ae6546f'},
+ 'timestamp': 190011.188468,
+ 'type': 'Script'}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 31556,
+ 'encodedDataLength': 0,
+ 'requestId': '17180.2',
+ 'timestamp': 190011.188663}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 6737,
+ 'encodedDataLength': 11251,
+ 'requestId': '17180.2',
+ 'timestamp': 190011.198249}}
+{'method': 'Network.dataReceived',
+ 'params': {'dataLength': 0,
+ 'encodedDataLength': 2049,
+ 'requestId': '17180.2',
+ 'timestamp': 190011.200943}}
+{'method': 'Network.loadingFinished',
+ 'params': {'encodedDataLength': 13810,
+ 'requestId': '17180.2',
+ 'shouldReportCorbBlocking': False,
+ 'timestamp': 190011.198142}}
+{'method': 'Page.loadEventFired', 'params': {'timestamp': 190011.204711}}
+{'method': 'Page.frameScheduledNavigation',
+ 'params': {'delay': 12,
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'reason': 'metaTagRefresh',
+ 'url': 'https://nowsecure.nl/'}}
+{'method': 'Page.frameStoppedLoading',
+ 'params': {'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700'}}
+{'method': 'Network.requestWillBeSent',
+ 'params': {'documentURL': 'https://nowsecure.nl/',
+ 'frameId': 'F42BAE4BDD4E428EE2503CB5A7B4F700',
+ 'hasUserGesture': False,
+ 'initiator': {'type': 'other'},
+ 'loaderId': '449906A5C736D819123288133F2797E6',
+ 'request': {'headers': {'Referer': 'https://nowsecure.nl/',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT '
+ '10.0; Win64; x64) '
+ 'AppleWebKit/537.36 (KHTML, '
+ 'like Gecko) '
+ 'Chrome/90.0.4430.212 '
+ 'Safari/537.36',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0'},
+ 'initialPriority': 'High',
+ 'method': 'GET',
+ 'mixedContentType': 'none',
+ 'referrerPolicy': 'strict-origin-when-cross-origin',
+ 'url': 'https://nowsecure.nl/favicon.ico'},
+ 'requestId': '17180.5',
+ 'timestamp': 190011.210491,
+ 'type': 'Other',
+ 'wallTime': 1621835932.325683}}
+{'method': 'Network.requestWillBeSentExtraInfo',
+ 'params': {'associatedCookies': [{'blockedReasons': [],
+ 'cookie': {'domain': 'nowsecure.nl',
+ 'expires': 1621839532,
+ 'httpOnly': False,
+ 'name': 'cf_chl_prog',
+ 'path': '/',
+ 'priority': 'Medium',
+ 'sameParty': False,
+ 'secure': False,
+ 'session': False,
+ 'size': 12,
+ 'sourcePort': 443,
+ 'sourceScheme': 'Secure',
+ 'value': 'e'}}],
+ 'clientSecurityState': {'initiatorIPAddressSpace': 'Public',
+ 'initiatorIsSecureContext': True,
+ 'privateNetworkRequestPolicy': 'WarnFromInsecureToMorePrivate'},
+ 'headers': {':authority': 'nowsecure.nl',
+ ':method': 'GET',
+ ':path': '/favicon.ico',
+ ':scheme': 'https',
+ 'accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+ 'accept-encoding': 'gzip, deflate, br',
+ 'accept-language': 'en-US,en;q=0.9',
+ 'cookie': 'cf_chl_prog=e',
+ 'referer': 'https://nowsecure.nl/',
+ 'sec-ch-ua': '" Not A;Brand";v="99", '
+ '"Chromium";v="90", "Google '
+ 'Chrome";v="90"',
+ 'sec-ch-ua-mobile': '?0',
+ 'sec-fetch-dest': 'image',
+ 'sec-fetch-mode': 'no-cors',
+ 'sec-fetch-site': 'same-origin',
+ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; '
+ 'x64) AppleWebKit/537.36 (KHTML, like '
+ 'Gecko) Chrome/90.0.4430.212 '
+ 'Safari/537.36'},
+
+# hopefullly you get the idea.
+```
+
+
+
+
+
+
+
+
+#### the easy way (v1 old stuff) ####
+```python
+import undetected_chromedriver as uc
+driver = uc.Chrome()
+driver.get('https://distilnetworks.com')
+```
+
+
+
+
+#### target specific chrome version (v1 old stuff) ####
+```python
+import undetected_chromedriver as uc
+uc.TARGET_VERSION = 85
+driver = uc.Chrome()
+```
+
+
+#### monkeypatch mode (v1 old stuff) ####
+Needs to be done before importing from selenium package
+
+```python
+import undetected_chromedriver as uc
+uc.install()
+
+from selenium.webdriver import Chrome
+driver = Chrome()
+driver.get('https://distilnetworks.com')
+
+```
+
+
+#### the customized way (v1 old stuff) ####
+```python
+import undetected_chromedriver as uc
+
+#specify chromedriver version to download and patch
+uc.TARGET_VERSION = 78
+
+# or specify your own chromedriver binary (why you would need this, i don't know)
+
+uc.install(
+ executable_path='c:/users/user1/chromedriver.exe',
+)
+
+opts = uc.ChromeOptions()
+opts.add_argument(f'--proxy-server=socks5://127.0.0.1:9050')
+driver = uc.Chrome(options=opts)
+driver.get('https://distilnetworks.com')
+```
+
+
+#### datadome.co example (v1 old stuff) ####
+These guys have actually a powerful product, and a link to this repo, which makes me wanna test their product.
+Make sure you use a "clean" ip for this one.
+```python
+#
+# STANDARD selenium Chromedriver
+#
+from selenium import webdriver
+chrome = webdriver.Chrome()
+chrome.get('https://datadome.co/customers-stories/toppreise-ends-web-scraping-and-content-theft-with-datadome/')
+chrome.save_screenshot('datadome_regular_webdriver.png')
+True # it caused my ip to be flagged, unfortunately
+
+
+#
+# UNDETECTED chromedriver (headless,even)
+#
+import undetected_chromedriver as uc
+options = uc.ChromeOptions()
+options.headless=True
+options.add_argument('--headless')
+chrome = uc.Chrome(options=options)
+chrome.get('https://datadome.co/customers-stories/toppreise-ends-web-scraping-and-content-theft-with-datadome/')
+chrome.save_screenshot('datadome_undetected_webddriver.png')
+
+```
+**Check both saved screenhots [here](https://imgur.com/a/fEmqadP)**
diff --git a/setup.py b/setup.py
index 193158f..c4fbf73 100644
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,8 @@ setup(
packages=["undetected_chromedriver"],
install_requires=[
"selenium",
+ "requests",
+ "websockets",
],
url="https://github.com/ultrafunkamsterdam/undetected-chromedriver",
license="GPL-3.0",
diff --git a/undetected_chromedriver/__init__.py b/undetected_chromedriver/__init__.py
index 20cb328..661cc0f 100644
--- a/undetected_chromedriver/__init__.py
+++ b/undetected_chromedriver/__init__.py
@@ -19,19 +19,19 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
import io
import logging
import os
+import random
import re
+import string
import sys
import zipfile
-import string
-import random
from distutils.version import LooseVersion
from urllib.request import urlopen, urlretrieve
-from selenium.webdriver import Chrome as _Chrome
-from selenium.webdriver import ChromeOptions as _ChromeOptions
+from selenium.webdriver import Chrome as _Chrome, ChromeOptions as _ChromeOptions
logger = logging.getLogger(__name__)
-__version__ = "2.2.7"
+__version__ = "3.0.0"
+
TARGET_VERSION = 0
@@ -72,7 +72,9 @@ class Chrome:
: target[key]
})
});
- """
+
+
+ """
},
)
return instance._orig_get(*args, **kwargs)
@@ -120,7 +122,6 @@ class ChromeOptions:
class ChromeDriverManager(object):
-
installed = False
selenium_patched = False
target_version = None
diff --git a/undetected_chromedriver/cdp.py b/undetected_chromedriver/cdp.py
new file mode 100644
index 0000000..7def6ca
--- /dev/null
+++ b/undetected_chromedriver/cdp.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import json
+import logging
+from collections import Mapping, Sequence
+
+import requests
+import websockets
+
+log = logging.getLogger(__name__)
+
+
+class CDPObjectBase(dict):
+ def __init__(self, *a, **kw):
+ super().__init__(**kw)
+ for k in self:
+ if isinstance(self[k], Mapping):
+ self[k] = self.__class__(self[k]) # noqa
+ elif isinstance(self[k], Sequence) and not isinstance(
+ self[k], (str, bytes)
+ ):
+ self[k] = self[k].__class__(self.__class__(i) for i in self[k])
+ else:
+ self[k] = self[k]
+
+ def __repr__(self):
+ tpl = f"{self.__class__.__name__}(\n\t{{}}\n\t)"
+ return tpl.format("\n ".join(f"{k} = {v}" for k, v in self.items()))
+
+
+class PageElement(CDPObjectBase):
+ pass
+
+
+class CDP:
+ log = logging.getLogger("CDP")
+
+ endpoints = {
+ "json": "/json",
+ "protocol": "/json/protocol",
+ "list": "/json/list",
+ "new": "/json/new?{url}",
+ "activate": "/json/activate/{id}",
+ "close": "/json/close/{id}",
+ }
+
+ def __init__(self, options: "ChromeOptions"):
+ self.server_addr = "http://{0}:{1}".format(*options.debugger_address.split(":"))
+
+ self._reqid = 0
+ self._session = requests.Session()
+ self._last_resp = None
+ self._last_json = None
+
+ resp = self.get(self.endpoints["json"])
+ self.sessionId = resp[0]["id"]
+ self.wsurl = resp[0]["webSocketDebuggerUrl"]
+
+ def tab_activate(self, id):
+ return self.post(self.endpoints["activate"].format(id=id))
+
+ def tab_list(self):
+ retval = self.post(self.endpoints["list"])
+ return [PageElement(o) for o in retval]
+
+ def tab_new(self, url):
+ return self.post(self.endpoints["new"].format(url=url))
+
+ def tab_close_last_opened(self):
+ sessions = self.tab_list()
+ opentabs = [s for s in sessions if s["type"] == "page"]
+ return self.post(self.endpoints["close"].format(id=opentabs[-1]["id"]))
+
+ async def send(self, method: str, params: dict):
+ self._reqid += 1
+ async with websockets.connect(self.wsurl) as ws:
+ await ws.send(
+ json.dumps({"method": method, "params": params, "id": self._reqid})
+ )
+ self._last_resp = await ws.recv()
+ self._last_json = json.loads(self._last_resp)
+ self.log.info(self._last_json)
+
+ def get(self, uri):
+ resp = self._session.get(self.server_addr + uri)
+ try:
+ self._last_resp = resp
+ self._last_json = resp.json()
+ except Exception:
+ return
+ else:
+ return self._last_json
+
+ def post(self, uri):
+ resp = self._session.post(self.server_addr + uri)
+ try:
+ self._last_resp = resp
+ self._last_json = resp.json()
+ except Exception:
+ return self._last_resp
+
+ @property
+ def last_json(self):
+ return self._last_json
diff --git a/undetected_chromedriver/options.py b/undetected_chromedriver/options.py
new file mode 100644
index 0000000..8801d31
--- /dev/null
+++ b/undetected_chromedriver/options.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import base64
+import os
+
+from selenium.webdriver.chrome.options import Options as _ChromeOptions
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+
+
+class ChromeOptions(_ChromeOptions):
+ KEY = "goog:chromeOptions"
+
+ session = None
+ emulate_touch = True
+ mock_permissions = True
+ mock_chrome_global = False
+ mock_canvas_fp = True
+ _user_data_dir = None
+
+ def __init__(self):
+ super().__init__()
+ self._arguments = []
+ self._binary_location = ""
+ self._extension_files = []
+ self._extensions = []
+ self._experimental_options = {}
+ self._debugger_address = None
+ self._caps = self.default_capabilities
+ self.mobile_options = None
+ self.set_capability("pageLoadStrategy", "normal")
+
+ @property
+ def user_data_dir(self):
+ return self._user_data_dir
+
+ @user_data_dir.setter
+ def user_data_dir(self, path: str):
+ """
+ Sets the browser profile folder to use, or creates a new profile
+ at given .
+
+ Parameters
+ ----------
+ path: str
+ the path to a chrome profile folder
+ if it does not exist, a new profile will be created at given location
+ """
+ apath = os.path.abspath(path)
+ self._user_data_dir = os.path.normpath(apath)
+
+ @property
+ def arguments(self):
+ """
+ :Returns: A list of arguments needed for the browser
+ """
+ return self._arguments
+
+ @property
+ def binary_location(self) -> str:
+ """
+ :Returns: The location of the binary, otherwise an empty string
+ """
+ return self._binary_location
+
+ @binary_location.setter
+ def binary_location(self, value: str):
+ """
+ Allows you to set where the chromium binary lives
+ :Args:
+ - value: path to the Chromium binary
+ """
+ self._binary_location = value
+
+ @property
+ def debugger_address(self) -> str:
+ """
+ :Returns: The address of the remote devtools instance
+ """
+ return self._debugger_address
+
+ @debugger_address.setter
+ def debugger_address(self, value: str):
+ """
+ Allows you to set the address of the remote devtools instance
+ that the ChromeDriver instance will try to connect to during an
+ active wait.
+ :Args:
+ - value: address of remote devtools instance if any (hostname[:port])
+ """
+ self._debugger_address = value
+
+ @property
+ def extensions(self):
+ """
+ :Returns: A list of encoded extensions that will be loaded
+ """
+ encoded_extensions = []
+ for ext in self._extension_files:
+ file_ = open(ext, "rb")
+ # Should not use base64.encodestring() which inserts newlines every
+ # 76 characters (per RFC 1521). Chromedriver has to remove those
+ # unnecessary newlines before decoding, causing performance hit.
+ encoded_extensions.append(base64.b64encode(file_.read()).decode("UTF-8"))
+ file_.close()
+ return encoded_extensions + self._extensions
+
+ def add_extension(self, extension: str):
+ """
+ Adds the path to the extension to a list that will be used to extract it
+ to the ChromeDriver
+ :Args:
+ - extension: path to the \\*.crx file
+ """
+ if extension:
+ extension_to_add = os.path.abspath(os.path.expanduser(extension))
+ if os.path.exists(extension_to_add):
+ self._extension_files.append(extension_to_add)
+ else:
+ raise IOError("Path to the extension doesn't exist")
+ else:
+ raise ValueError("argument can not be null")
+
+ def add_encoded_extension(self, extension: str):
+ """
+ Adds Base64 encoded string with extension data to a list that will be used to extract it
+ to the ChromeDriver
+ :Args:
+ - extension: Base64 encoded string with extension data
+ """
+ if extension:
+ self._extensions.append(extension)
+ else:
+ raise ValueError("argument can not be null")
+
+ @property
+ def experimental_options(self) -> dict:
+ """
+ :Returns: A dictionary of experimental options for chromium
+ """
+ return self._experimental_options
+
+ def add_experimental_option(self, name: str, value: dict):
+ """
+ Adds an experimental option which is passed to chromium.
+ :Args:
+ name: The experimental option name.
+ value: The option value.
+ """
+ self._experimental_options[name] = value
+
+ @property
+ def headless(self) -> bool:
+ """
+ :Returns: True if the headless argument is set, else False
+ """
+ return "--headless" in self._arguments
+
+ @headless.setter
+ def headless(self, value: bool):
+ """
+ Sets the headless argument
+ :Args:
+ value: boolean value indicating to set the headless option
+ """
+ args = {"--headless"}
+ if value is True:
+ self._arguments.extend(args)
+ else:
+ self._arguments = list(set(self._arguments) - args)
+
+ @property
+ def page_load_strategy(self) -> str:
+ return self._caps["pageLoadStrategy"]
+
+ @page_load_strategy.setter
+ def page_load_strategy(self, strategy: str):
+ if strategy in ["normal", "eager", "none"]:
+ self.set_capability("pageLoadStrategy", strategy)
+ else:
+ raise ValueError(
+ "Strategy can only be one of the following: normal, eager, none"
+ )
+
+ @property
+ def capabilities(self):
+ return self._caps
+
+ def set_capability(self, name, value):
+ """ Sets a capability """
+ self._caps[name] = value
+
+ def to_capabilities(self) -> dict:
+ """
+ Creates a capabilities with all the options that have been set
+ :Returns: A dictionary with everything
+ """
+ caps = self._caps
+ chrome_options = self.experimental_options.copy()
+ if self.mobile_options:
+ chrome_options.update(self.mobile_options)
+ chrome_options["extensions"] = self.extensions
+ if self.binary_location:
+ chrome_options["binary"] = self.binary_location
+ chrome_options["args"] = self._arguments
+ if self.debugger_address:
+ chrome_options["debuggerAddress"] = self.debugger_address
+
+ caps[self.KEY] = chrome_options
+
+ return caps
+
+ def ignore_local_proxy_environment_variables(self):
+ """
+ By calling this you will ignore HTTP_PROXY and HTTPS_PROXY from being picked up and used.
+ """
+ self._ignore_local_proxy = True
+
+ @property
+ def default_capabilities(self) -> dict:
+ return DesiredCapabilities.CHROME.copy()
+
+ def enable_mobile(
+ self,
+ android_package: str = None,
+ android_activity: str = None,
+ device_serial: str = None,
+ ):
+ """
+ Enables mobile browser use for browsers that support it
+ :Args:
+ android_activity: The name of the android package to start
+ """
+ if not android_package:
+ raise AttributeError("android_package must be passed in")
+ self.mobile_options = {"androidPackage": android_package}
+ if android_activity:
+ self.mobile_options["androidActivity"] = android_activity
+ if device_serial:
+ self.mobile_options["androidDeviceSerial"] = device_serial
+
+ def add_argument(self, argument):
+ """
+ Adds an argument to the list
+ :Args:
+ - Sets the arguments
+ """
+ if argument:
+ self._arguments.append(argument)
+ else:
+ raise ValueError("argument can not be null")
+
+ @classmethod
+ def from_options(cls, options):
+ o = cls()
+ o.__dict__.update(options.__dict__)
+ return o
diff --git a/undetected_chromedriver/patcher.py b/undetected_chromedriver/patcher.py
new file mode 100644
index 0000000..621e07f
--- /dev/null
+++ b/undetected_chromedriver/patcher.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import io
+import logging
+import os
+import random
+import re
+import string
+import sys
+import zipfile
+from distutils.version import LooseVersion
+from urllib.request import urlopen, urlretrieve
+
+logger = logging.getLogger(__name__)
+
+IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux"))
+
+
+class Patcher(object):
+ url_repo = "https://chromedriver.storage.googleapis.com"
+ zip_name = "chromedriver_%s.zip"
+ exe_name = "chromedriver%s"
+
+ platform = sys.platform
+ if platform.endswith("win32"):
+ zip_name %= "win32"
+ exe_name %= ".exe"
+ if platform.endswith("linux"):
+ zip_name %= "linux64"
+ exe_name %= ""
+ if platform.endswith("darwin"):
+ zip_name %= "mac64"
+ exe_name %= ""
+
+ if platform.endswith("win32"):
+ d = "~/appdata/roaming/undetected_chromedriver"
+ elif platform.startswith("linux"):
+ d = "~/.local/share/undetected_chromedriver"
+ elif platform.endswith("darwin"):
+ d = "~/Library/Application Support/undetected_chromedriver"
+ else:
+ d = "~/.undetected_chromedriver"
+ data_path = os.path.abspath(os.path.expanduser(d))
+
+ def __init__(self, executable_path=None, force=False, version_main: int = 0):
+ """
+
+ Args:
+ executable_path: None = automatic
+ a full file path to the chromedriver executable
+ force: False
+ terminate processes which are holding lock
+ version_main: 0 = auto
+ specify main chrome version (rounded, ex: 82)
+ """
+
+ self.force = force
+
+ if not executable_path:
+ executable_path = os.path.join(self.data_path, self.exe_name)
+
+ if not IS_POSIX:
+ if not executable_path[-4:] == ".exe":
+ executable_path += ".exe"
+
+ self.zip_path = os.path.join(self.data_path, self.zip_name)
+
+ self.executable_path = os.path.abspath(os.path.join(".", executable_path))
+
+ self.version_main = version_main
+ self.version_full = None
+
+ @classmethod
+ def auto(cls, executable_path=None, force=False):
+ """
+
+ Args:
+ force:
+
+ Returns:
+
+ """
+ i = cls(executable_path, force=force)
+ try:
+ os.unlink(i.executable_path)
+ except PermissionError:
+ if i.force:
+ cls.force_kill_instances(i.executable_path)
+ return i.auto(force=False)
+ try:
+ if i.is_binary_patched():
+ # assumes already running AND patched
+ return True
+ except PermissionError:
+ pass
+ # return False
+ except FileNotFoundError:
+ pass
+
+ release = i.fetch_release_number()
+ i.version_main = release.version[0]
+ i.version_full = release
+ i.unzip_package(i.fetch_package())
+ i.patch()
+ return i
+
+ def patch(self):
+ self.patch_exe()
+ return self.is_binary_patched()
+
+ def fetch_release_number(self):
+ """
+ Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
+ :return: version string
+ :rtype: LooseVersion
+ """
+ path = "/latest_release"
+ if self.version_main:
+ path += f"_{self.version_main}"
+ path = path.upper()
+ logger.debug("getting release number from %s" % path)
+ return LooseVersion(urlopen(self.url_repo + path).read().decode())
+
+ def parse_exe_version(self):
+ with io.open(self.executable_path, "rb") as f:
+ for line in iter(lambda: f.readline(), b""):
+ match = re.search(br"platform_handle\x00content\x00([0-9.]*)", line)
+ if match:
+ return LooseVersion(match[1].decode())
+
+ def fetch_package(self):
+ """
+ Downloads ChromeDriver from source
+
+ :return: path to downloaded file
+ """
+ u = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, self.zip_name)
+ logger.debug("downloading from %s" % u)
+ # return urlretrieve(u, filename=self.data_path)[0]
+ return urlretrieve(u)[0]
+
+ def unzip_package(self, fp):
+ """
+ Does what it says
+
+ :return: path to unpacked executable
+ """
+ logger.debug("unzipping %s" % fp)
+ try:
+ os.unlink(self.zip_path)
+ except (FileNotFoundError, OSError):
+ pass
+
+ os.makedirs(self.data_path, mode=0o755, exist_ok=True)
+
+ with zipfile.ZipFile(fp, mode="r") as zf:
+ zf.extract(self.exe_name, os.path.dirname(self.executable_path))
+ os.remove(fp)
+ os.chmod(self.executable_path, 0o755)
+ return self.executable_path
+
+ @staticmethod
+ def force_kill_instances(exe_name):
+ """
+ kills running instances.
+ :param: executable name to kill, may be a path as well
+
+ :return: True on success else False
+ """
+ exe_name = os.path.basename(exe_name)
+ if IS_POSIX:
+ r = os.system("kill -f -9 $(pidof %s)" % exe_name)
+ else:
+ r = os.system("taskkill /f /im %s" % exe_name)
+ return not r
+
+ @staticmethod
+ def gen_random_cdc():
+ cdc = random.choices(string.ascii_lowercase, k=26)
+ cdc[-6:-4] = map(str.upper, cdc[-6:-4])
+ cdc[2] = cdc[0]
+ cdc[3] = "_"
+ return "".join(cdc).encode()
+
+ def is_binary_patched(self, executable_path=None):
+ """simple check if executable is patched.
+
+ :return: False if not patched, else True
+ """
+ executable_path = executable_path or self.executable_path
+ with io.open(executable_path, "rb") as fh:
+ for line in iter(lambda: fh.readline(), b""):
+ if b"cdc_" in line:
+ return False
+ else:
+ return True
+
+ def patch_exe(self):
+ """
+ Patches the ChromeDriver binary
+
+ :return: False on failure, binary name on success
+ """
+ logger.info("patching driver executable %s" % self.executable_path)
+
+ linect = 0
+ replacement = self.gen_random_cdc()
+ with io.open(self.executable_path, "r+b") as fh:
+ for line in iter(lambda: fh.readline(), b""):
+ if b"cdc_" in line:
+ fh.seek(-len(line), 1)
+ newline = re.sub(b"cdc_.{22}", replacement, line)
+ fh.write(newline)
+ linect += 1
+ return linect
+
+ def __repr__(self):
+ return "{0:s}({1:s})".format(
+ self.__class__.__name__,
+ self.executable_path,
+ )
diff --git a/undetected_chromedriver/reactor.py b/undetected_chromedriver/reactor.py
new file mode 100644
index 0000000..874ae19
--- /dev/null
+++ b/undetected_chromedriver/reactor.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# this module is part of undetected_chromedriver
+
+import asyncio
+import json
+import logging
+import threading
+
+logger = logging.getLogger(__name__)
+
+
+class Reactor(threading.Thread):
+ def __init__(self, driver: "Chrome"):
+ super().__init__()
+
+ self.driver = driver
+ self.loop = asyncio.new_event_loop()
+
+ self.lock = threading.Lock()
+ self.event = threading.Event()
+ self.daemon = True
+ self.handlers = {}
+
+ def add_event_handler(self, method_name, callback: callable):
+ """
+
+ Parameters
+ ----------
+ event_name: str
+ example "Network.responseReceived"
+
+ callback: callable
+ callable which accepts 1 parameter: the message object dictionary
+
+ Returns
+ -------
+
+ """
+ with self.lock:
+ self.handlers[method_name.lower()] = callback
+
+ @property
+ def running(self):
+ return not self.event.is_set()
+
+ def run(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ self.loop.run_until_complete(self.listen())
+ except Exception as e:
+ logger.warning("Reactor.run() => %s", e)
+
+ async def listen(self):
+
+ while self.running:
+
+ await asyncio.sleep(0)
+
+ try:
+ with self.lock:
+ log_entries = self.driver.get_log("performance")
+
+ for entry in log_entries:
+
+ try:
+
+ obj_serialized: str = entry.get("message")
+ obj = json.loads(obj_serialized)
+ message = obj.get("message")
+ method = message.get("method")
+
+ if "*" in self.handlers:
+ await self.loop.run_in_executor(
+ None, self.handlers["*"], message
+ )
+ elif method.lower() in self.handlers:
+ await self.loop.run_in_executor(
+ None, self.handlers[method.lower()], message
+ )
+
+ # print(type(message), message)
+ except Exception as e:
+ raise e from None
+
+ except Exception as e:
+ if "invalid session id" in str(e):
+ pass
+ else:
+ logging.debug("exception ignored :", e)
diff --git a/undetected_chromedriver/tests/fp.js b/undetected_chromedriver/tests/fp.js
new file mode 100644
index 0000000..814a663
--- /dev/null
+++ b/undetected_chromedriver/tests/fp.js
@@ -0,0 +1,319 @@
+(function (name, context, definition) {
+ if (typeof module !== 'undefined' && module.exports) {
+ module.exports = definition();
+ } else if (typeof define === 'function' && define.amd) {
+ define(definition);
+ } else {
+ context[name] = definition();
+ }
+})('Fingerprint', this, function () {
+ 'use strict';
+
+ var Fingerprint = function (options) {
+ var nativeForEach, nativeMap;
+ nativeForEach = Array.prototype.forEach;
+ nativeMap = Array.prototype.map;
+
+ this.each = function (obj, iterator, context) {
+ if (obj === null) {
+ return;
+ }
+ if (nativeForEach && obj.forEach === nativeForEach) {
+ obj.forEach(iterator, context);
+ } else if (obj.length === +obj.length) {
+ for (var i = 0, l = obj.length; i < l; i++) {
+ if (iterator.call(context, obj[i], i, obj) === {}) return;
+ }
+ } else {
+ for (var key in obj) {
+ if (obj.hasOwnProperty(key)) {
+ if (iterator.call(context, obj[key], key, obj) === {}) return;
+ }
+ }
+ }
+ };
+
+ this.map = function (obj, iterator, context) {
+ var results = [];
+ // Not using strict equality so that this acts as a
+ // shortcut to checking for `null` and `undefined`.
+ if (obj == null) return results;
+ if (nativeMap && obj.map === nativeMap) return obj.map(iterator, context);
+ this.each(obj, function (value, index, list) {
+ results[results.length] = iterator.call(context, value, index, list);
+ });
+ return results;
+ };
+
+ if (typeof options == 'object') {
+ this.hasher = options.hasher;
+ this.screen_resolution = options.screen_resolution;
+ this.screen_orientation = options.screen_orientation;
+ this.canvas = options.canvas;
+ this.ie_activex = options.ie_activex;
+ } else if (typeof options == 'function') {
+ this.hasher = options;
+ }
+ };
+
+ Fingerprint.prototype = {
+ get: function () {
+ var keys = [];
+ keys.push(navigator.userAgent);
+ keys.push(navigator.language);
+ keys.push(screen.colorDepth);
+ if (this.screen_resolution) {
+ var resolution = this.getScreenResolution();
+ if (typeof resolution !== 'undefined') { // headless browsers, such as phantomjs
+ keys.push(resolution.join('x'));
+ }
+ }
+ keys.push(new Date().getTimezoneOffset());
+ keys.push(this.hasSessionStorage());
+ keys.push(this.hasLocalStorage());
+ keys.push(this.hasIndexDb());
+ //body might not be defined at this point or removed programmatically
+ if (document.body) {
+ keys.push(typeof (document.body.addBehavior));
+ } else {
+ keys.push(typeof undefined);
+ }
+ keys.push(typeof (window.openDatabase));
+ keys.push(navigator.cpuClass);
+ keys.push(navigator.platform);
+ keys.push(navigator.doNotTrack);
+ keys.push(this.getPluginsString());
+ if (this.canvas && this.isCanvasSupported()) {
+ keys.push(this.getCanvasFingerprint());
+ }
+ if (this.hasher) {
+ return this.hasher(keys.join('###'), 31);
+ } else {
+ return this.murmurhash3_32_gc(keys.join('###'), 31);
+ }
+ },
+
+ /**
+ * JS Implementation of MurmurHash3 (r136) (as of May 20, 2011)
+ *
+ * @author Gary Court
+ * @see http://github.com/garycourt/murmurhash-js
+ * @author Austin Appleby
+ * @see http://sites.google.com/site/murmurhash/
+ *
+ * @param {string} key ASCII only
+ * @param {number} seed Positive integer only
+ * @return {number} 32-bit positive integer hash
+ */
+
+ murmurhash3_32_gc: function (key, seed) {
+ var remainder, bytes, h1, h1b, c1, c2, k1, i;
+
+ remainder = key.length & 3; // key.length % 4
+ bytes = key.length - remainder;
+ h1 = seed;
+ c1 = 0xcc9e2d51;
+ c2 = 0x1b873593;
+ i = 0;
+
+ while (i < bytes) {
+ k1 =
+ ((key.charCodeAt(i) & 0xff)) |
+ ((key.charCodeAt(++i) & 0xff) << 8) |
+ ((key.charCodeAt(++i) & 0xff) << 16) |
+ ((key.charCodeAt(++i) & 0xff) << 24);
+ ++i;
+
+ k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff;
+ k1 = (k1 << 15) | (k1 >>> 17);
+ k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff;
+
+ h1 ^= k1;
+ h1 = (h1 << 13) | (h1 >>> 19);
+ h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff;
+ h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16));
+ }
+
+ k1 = 0;
+
+ switch (remainder) {
+ case 3:
+ k1 ^= (key.charCodeAt(i + 2) & 0xff) << 16;
+ case 2:
+ k1 ^= (key.charCodeAt(i + 1) & 0xff) << 8;
+ case 1:
+ k1 ^= (key.charCodeAt(i) & 0xff);
+
+ k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff;
+ k1 = (k1 << 15) | (k1 >>> 17);
+ k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff;
+ h1 ^= k1;
+ }
+
+ h1 ^= key.length;
+
+ h1 ^= h1 >>> 16;
+ h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff;
+ h1 ^= h1 >>> 13;
+ h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff;
+ h1 ^= h1 >>> 16;
+
+ return h1 >>> 0;
+ },
+
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=781447
+ hasLocalStorage: function () {
+ try {
+ return !!window.localStorage;
+ } catch (e) {
+ return true; // SecurityError when referencing it means it exists
+ }
+ },
+
+ hasSessionStorage: function () {
+ try {
+ return !!window.sessionStorage;
+ } catch (e) {
+ return true; // SecurityError when referencing it means it exists
+ }
+ },
+
+ hasIndexDb: function () {
+ try {
+ return !!window.indexedDB;
+ } catch (e) {
+ return true; // SecurityError when referencing it means it exists
+ }
+ },
+
+ isCanvasSupported: function () {
+ var elem = document.createElement('canvas');
+ return !!(elem.getContext && elem.getContext('2d'));
+ },
+
+ isIE: function () {
+ if (navigator.appName === 'Microsoft Internet Explorer') {
+ return true;
+ } else if (navigator.appName === 'Netscape' && /Trident/.test(navigator.userAgent)) {// IE 11
+ return true;
+ }
+ return false;
+ },
+
+ getPluginsString: function () {
+ if (this.isIE() && this.ie_activex) {
+ return this.getIEPluginsString();
+ } else {
+ return this.getRegularPluginsString();
+ }
+ },
+
+ getRegularPluginsString: function () {
+ return this.map(navigator.plugins, function (p) {
+ var mimeTypes = this.map(p, function (mt) {
+ return [mt.type, mt.suffixes].join('~');
+ }).join(',');
+ return [p.name, p.description, mimeTypes].join('::');
+ }, this).join(';');
+ },
+
+ getIEPluginsString: function () {
+ if (window.ActiveXObject) {
+ var names = ['ShockwaveFlash.ShockwaveFlash',//flash plugin
+ 'AcroPDF.PDF', // Adobe PDF reader 7+
+ 'PDF.PdfCtrl', // Adobe PDF reader 6 and earlier, brrr
+ 'QuickTime.QuickTime', // QuickTime
+ // 5 versions of real players
+ 'rmocx.RealPlayer G2 Control',
+ 'rmocx.RealPlayer G2 Control.1',
+ 'RealPlayer.RealPlayer(tm) ActiveX Control (32-bit)',
+ 'RealVideo.RealVideo(tm) ActiveX Control (32-bit)',
+ 'RealPlayer',
+ 'SWCtl.SWCtl', // ShockWave player
+ 'WMPlayer.OCX', // Windows media player
+ 'AgControl.AgControl', // Silverlight
+ 'Skype.Detection'];
+
+ // starting to detect plugins in IE
+ return this.map(names, function (name) {
+ try {
+ new ActiveXObject(name);
+ return name;
+ } catch (e) {
+ return null;
+ }
+ }).join(';');
+ } else {
+ return ""; // behavior prior version 0.5.0, not breaking backwards compat.
+ }
+ },
+
+ getScreenResolution: function () {
+ var resolution;
+ if (this.screen_orientation) {
+ resolution = (screen.height > screen.width) ? [screen.height, screen.width] : [screen.width, screen.height];
+ } else {
+ resolution = [screen.height, screen.width];
+ }
+ return resolution;
+ },
+
+ getCanvasFingerprint: function () {
+ var canvas = document.createElement('canvas');
+ var ctx = canvas.getContext('2d');
+ // https://www.browserleaks.com/canvas#how-does-it-work
+ var txt = 'http://valve.github.io';
+ ctx.textBaseline = "top";
+ ctx.font = "14px 'Arial'";
+ ctx.textBaseline = "alphabetic";
+ ctx.fillStyle = "#f60";
+ ctx.fillRect(125, 1, 62, 20);
+ ctx.fillStyle = "#069";
+ ctx.fillText(txt, 2, 15);
+ ctx.fillStyle = "rgba(102, 204, 0, 0.7)";
+ ctx.fillText(txt, 4, 17);
+ return canvas.toDataURL();
+ }
+ };
+ return Fingerprint;
+});
+
+
+new Fingerprint({canvas: true}).get();
+
+
+var inject = function () {
+ var overwrite = function (name) {
+ const OLD = HTMLCanvasElement.prototype[name];
+ Object.defineProperty(HTMLCanvasElement.prototype, name, {
+ "value": function () {
+ var shift = {
+ 'r': Math.floor(Math.random() * 10) - 5,
+ 'g': Math.floor(Math.random() * 10) - 5,
+ 'b': Math.floor(Math.random() * 10) - 5,
+ 'a': Math.floor(Math.random() * 10) - 5
+ };
+ var width = this.width, height = this.height, context = this.getContext("2d");
+ var imageData = context.getImageData(0, 0, width, height);
+ for (var i = 0; i < height; i++) {
+ for (var j = 0; j < width; j++) {
+ var n = ((i * (width * 4)) + (j * 4));
+ imageData.data[n + 0] = imageData.data[n + 0] + shift.r;
+ imageData.data[n + 1] = imageData.data[n + 1] + shift.g;
+ imageData.data[n + 2] = imageData.data[n + 2] + shift.b;
+ imageData.data[n + 3] = imageData.data[n + 3] + shift.a;
+ }
+ }
+ context.putImageData(imageData, 0, 0);
+ return OLD.apply(this, arguments);
+ }
+ });
+ };
+ overwrite('toBlob');
+ overwrite('toDataURL');
+};
+inject();
+
+
+new Fingerprint({canvas: true}).get();
+
diff --git a/undetected_chromedriver/tests/test_undetected_chromedriver.py b/undetected_chromedriver/tests/test_undetected_chromedriver.py
index 1669644..7491503 100644
--- a/undetected_chromedriver/tests/test_undetected_chromedriver.py
+++ b/undetected_chromedriver/tests/test_undetected_chromedriver.py
@@ -1,16 +1,12 @@
import logging
-import os
import sys
import time # noqa
-from ..v2 import *
-
logging.basicConfig(level=10)
logger = logging.getLogger("TEST")
logger.setLevel(20)
-
JS_SERIALIZE_FUNCTION = """
decycle=function(n,e){"use strict";var t=new WeakMap;return function n(o,r){var c,i;return void 0!==e&&(o=e(o)),"object"!=typeof o||null===o||o instanceof Boolean||o instanceof Date||o instanceof Number||o instanceof RegExp||o instanceof String?o:void 0!==(c=t.get(o))?{$ref:c}:(t.set(o,r),Array.isArray(o)?(i=[],o.forEach(function(e,t){i[t]=n(e,r+"["+t+"]")})):(i={},Object.keys(o).forEach(function(e){i[e]=n(o[e],r+"["+JSON.stringify(e)+"]")})),i)}(n,"$")};
function replacer(t){try{if(Array.prototype.splice.call(t).length<100){let e={};for(let r in t)e[r]=t[r];return e}}catch(t){}}
@@ -42,9 +38,10 @@ def test_undetected_chromedriver():
driver = uc.Chrome()
with driver:
- driver.get("https://coinfaucet.eu")
+
+ driver.get("https://nowsecure.nl")
time.sleep(4) # sleep only used for timing of screenshot
- driver.save_screenshot("coinfaucet.eu.png")
+ driver.save_screenshot("nowsecure.nl.png")
with driver:
driver.get("https://cia.gov")
diff --git a/undetected_chromedriver/tests/test_undetected_chromedriver_funstuff.py b/undetected_chromedriver/tests/test_undetected_chromedriver_funstuff.py
new file mode 100644
index 0000000..e5d514d
--- /dev/null
+++ b/undetected_chromedriver/tests/test_undetected_chromedriver_funstuff.py
@@ -0,0 +1,67 @@
+import asyncio
+import logging
+
+import cv2
+
+import undetected_chromedriver.v2 as uc
+
+logging.basicConfig(level=10)
+
+just_some_urls = [
+ "https://bing.com",
+ "http://www.google.com",
+ "https://codepen.io",
+ "https://",
+]
+
+
+class ChromeDriverCV2Streamer:
+ def __init__(self, driver):
+ super().__init__()
+ self.driver = driver
+ self.display = None
+ self.event = asyncio.Event()
+ self.daemon = True
+
+ def stop(self):
+ self.event.set()
+
+ def start(self):
+ asyncio.ensure_future(self._start_capture_loop())
+
+ async def _start_capture_loop(self):
+ executor = None
+ self.display = cv2.namedWindow("display")
+ while not self.event.is_set():
+ await asyncio.sleep(0.25)
+ try:
+ success = await loop.run_in_executor(
+ executor, self.driver.save_screenshot, "capture.tmp.png"
+ )
+ logging.getLogger().debug("got screenshot? %s", success)
+ frame = await loop.run_in_executor(
+ executor, cv2.imread, "capture.tmp.png"
+ )
+ logging.getLogger().debug("frame: %s", frame)
+ await loop.run_in_executor(executor, cv2.imshow, "display", frame)
+ await loop.run_in_executor(executor, cv2.waitKey, 1)
+ logging.getLogger().debug("waited key success")
+ except Exception as e:
+ print(e)
+
+
+async def main():
+ opts = uc.ChromeOptions()
+ opts.headless = True
+ driver = uc.Chrome(options=opts)
+
+ streamer = ChromeDriverCV2Streamer(driver)
+ streamer.start()
+ for url in just_some_urls:
+ # with driver:
+ driver.get("https://nu.nl")
+ await asyncio.sleep(3)
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())
diff --git a/undetected_chromedriver/tests/test_undetected_chromedriver_with_cdp_events.py b/undetected_chromedriver/tests/test_undetected_chromedriver_with_cdp_events.py
new file mode 100644
index 0000000..6f49420
--- /dev/null
+++ b/undetected_chromedriver/tests/test_undetected_chromedriver_with_cdp_events.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+import logging
+import os
+import sys
+
+import undetected_chromedriver.v2 as uc
+
+# it's not required to enable logging for cdp events to work
+# but as this is a test, it's good too it all
+logging.basicConfig(level=10)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("selenium.webdriver.remote.remote_connection").setLevel(logging.WARN)
+
+driver = uc.Chrome(enable_cdp_events=True)
+
+# set the callback to Network.dataReceived to print (yeah not much original)
+driver.add_cdp_listener("Network.dataReceived", print)
+
+# example of executing regular cdp commands
+driver.execute_cdp_cmd("Network.getAllCookies", {})
+
+# okay another one
+driver.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {"source": """ alert('another new document')"""},
+)
+
+# set the callback for ALL events (this may slow down execution)
+# driver.add_cdp_listener('*', print)
+
+
+with driver:
+ driver.get("https://nowsecure.nl")
+driver.save_screenshot("nowsecure.nl.headfull.png")
+try:
+ os.system("nowsecure.nl.headfull.png")
+except:
+ pass
+
+driver.quit()
+
+opts = uc.ChromeOptions()
+opts.headless = True
+driver = uc.Chrome(enable_cdp_events=True, options=opts)
+
+# okay another one
+driver.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {"source": """ alert('another new document')"""},
+)
+
+driver.add_cdp_listener("*", print)
+
+with driver:
+ driver.get("https://nowsecure.nl")
+ driver.save_screenshot("nowsecure.nl.headfull.png")
+try:
+ os.system("nowsecure.nl.headfull.png")
+except:
+ pass
+
+while True:
+ sys.stdin.read()
diff --git a/undetected_chromedriver/v2.py b/undetected_chromedriver/v2.py
index 061bef1..9c11c0a 100644
--- a/undetected_chromedriver/v2.py
+++ b/undetected_chromedriver/v2.py
@@ -1,67 +1,589 @@
#!/usr/bin/env python3
# this module is part of undetected_chromedriver
-"""
-V2 beta
-
-whats new:
-
- - currently this v2 module will be available as option.
- to use it / test it, you need to alter your imports by appending .v2
-
- - headless mode not (yet) supported in v2
-
- example:
-
- ```python
- import undetected_chromedriver.v2 as uc
- driver = uc.Chrome()
- driver.get('https://somewebsite.xyz')
-
- # if site is protected by hCaptcha/Cloudflare
- driver.get_in('https://cloudflareprotectedsite.xyz')
-
- # if site is protected by hCaptcha/Cloudflare
- # (different syntax, same function)
- with driver:
- driver.get('https://cloudflareprotectedsite.xyz')
- ```
-
- tests/example in ../tests/test_undetected_chromedriver.py
-
-"""
-
from __future__ import annotations
-import io
+import json
import logging
import os
-import random
import re
import shutil
-import string
import subprocess
import sys
import tempfile
import time
-import zipfile
-from distutils.version import LooseVersion
-from urllib.request import urlopen, urlretrieve
import selenium.webdriver.chrome.service
import selenium.webdriver.chrome.webdriver
import selenium.webdriver.common.service
import selenium.webdriver.remote.webdriver
-from selenium.webdriver.chrome.options import Options as _ChromeOptions
-__all__ = ("Chrome", "ChromeOptions", "Patcher", "find_chrome_executable")
+from .options import ChromeOptions
+from .patcher import IS_POSIX, Patcher
+from .reactor import Reactor
-IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux"))
+__all__ = ("Chrome", "ChromeOptions", "Patcher", "Reactor", "find_chrome_executable")
logger = logging.getLogger("uc")
logger.setLevel(logging.getLogger().getEffectiveLevel())
+class Chrome(selenium.webdriver.Chrome):
+ """
+
+ Controls the ChromeDriver and allows you to drive the browser.
+
+ The webdriver file will be downloaded by this module automatically,
+ you do not need to specify this. however, you may if you wish.
+
+ Attributes
+ ----------
+
+ Methods
+ -------
+
+ reconnect()
+
+ this can be useful in case of heavy detection methods
+ -stops the chromedriver service which runs in the background
+ -starts the chromedriver service which runs in the background
+ -recreate session
+
+
+ start_session(capabilities=None, browser_profile=None)
+
+ differentiates from the regular method in that it does not
+ require a capabilities argument. The capabilities are automatically
+ recreated from the options at creation time.
+
+ --------------------------------------------------------------------------
+ NOTE:
+ Chrome has everything included to work out of the box.
+ it does not `need` customizations.
+ any customizations MAY lead to trigger bot migitation systems.
+
+ --------------------------------------------------------------------------
+ """
+
+ _instances = set()
+
+ def __init__(
+ self,
+ executable_path=None,
+ port=0,
+ options=None,
+ enable_cdp_events=False,
+ service_args=None,
+ desired_capabilities=None,
+ service_log_path=None,
+ keep_alive=False,
+ log_level=0,
+ headless=False,
+ delay=5,
+ ):
+ """
+ Creates a new instance of the chrome driver.
+
+ Starts the service and then creates new instance of chrome driver.
+
+ Parameters
+ ----------
+ executable_path: str, optional, default: None - use find_chrome_executable
+ Path to the executable. If the default is used it assumes the executable is in the $PATH
+
+ port: int, optional, default: 0
+ port you would like the service to run, if left as 0, a free port will be found.
+
+ options: ChromeOptions, optional, default: None - automatic useful defaults
+ this takes an instance of ChromeOptions, mainly to customize browser behavior.
+ anything other dan the default, for example extensions or startup options
+ are not supported in case of failure, and can probably lowers your undetectability.
+
+ enable_cdp_events: bool, default: False
+ :: currently for chrome only
+ this enables the handling of wire messages
+ when enabled, you can subscribe to CDP events by using:
+
+ driver.on_cdp_event("Network.dataReceived", yourcallback)
+ # yourcallback is an callable which accepts exactly 1 dict as parameter
+
+ service_args: list of str, optional, default: None
+ arguments to pass to the driver service
+
+ desired_capabilities: dict, optional, default: None - auto from config
+ Dictionary object with non-browser specific capabilities only, such as "item" or "loggingPref".
+
+ service_log_path: str, optional, default: None
+ path to log information from the driver.
+
+ keep_alive: bool, optional, default: True
+ Whether to configure ChromeRemoteConnection to use HTTP keep-alive.
+
+ log_level: int, optional, default: adapts to python global log level
+
+ headless: bool, optional, default: False
+ can also be specified in the options instance.
+ Specify whether you want to use the browser in headless mode.
+ warning: this lowers undetectability and not fully supported.
+
+ emulate_touch: bool, optional, default: False
+ if set to True, patches window.maxTouchPoints to always return non-zero
+
+ delay: int, optional, default: 5
+ delay in seconds to wait before giving back control.
+ this is used only when using the context manager
+ (`with` statement) to bypass, for example CloudFlare.
+ 5 seconds is a foolproof value.
+
+ """
+
+ patcher = Patcher(executable_path=executable_path)
+ patcher.auto()
+
+ if not options:
+ options = ChromeOptions()
+
+ try:
+ if options.session and options.session is not None:
+ # prevent reuse of options,
+ # as it just appends arguments, not replace them
+ # you'll get conflicts starting chrome
+ raise RuntimeError("you cannot reuse the ChromeOptions object")
+ except AttributeError:
+ pass
+
+ options.session = self
+
+ debug_port = selenium.webdriver.common.service.utils.free_port()
+ debug_host = "127.0.0.1"
+
+ if not options.debugger_address:
+ options.debugger_address = "%s:%d" % (debug_host, debug_port)
+
+ if enable_cdp_events:
+ options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
+
+ options.add_argument("--remote-debugging-host=%s" % debug_host)
+ options.add_argument("--remote-debugging-port=%s" % debug_port)
+
+ user_data_dir, language, keep_user_data_dir = None, None, None
+
+ # see if a custom user profile is specified
+ for arg in options.arguments:
+
+ if "lang" in arg:
+ m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
+ try:
+ language = m[1]
+ except IndexError:
+ logger.debug("will set the language to en-US,en;q=0.9")
+ language = "en-US,en;q=0.9"
+
+ if "user-data-dir" in arg:
+ m = re.search("(?:--)?user-data-dir(?:[ =])?(.*)", arg)
+ try:
+ user_data_dir = m[1]
+ logger.debug(
+ "user-data-dir found in user argument %s => %s" % (arg, m[1])
+ )
+ keep_user_data_dir = True
+
+ except IndexError:
+ logger.debug(
+ "no user data dir could be extracted from supplied argument %s "
+ % arg
+ )
+
+ if not user_data_dir:
+
+ if options.user_data_dir:
+ options.add_argument("--user-data-dir=%s" % options.user_data_dir)
+ keep_user_data_dir = True
+ logger.debug(
+ "user_data_dir property found in options object: %s" % user_data_dir
+ )
+
+ else:
+ user_data_dir = os.path.normpath(tempfile.mkdtemp())
+ keep_user_data_dir = False
+ arg = "--user-data-dir=%s" % user_data_dir
+ options.add_argument(arg)
+ logger.debug(
+ "created a temporary folder in which the user-data (profile) will be stored during this\n"
+ "session, and added it to chrome startup arguments: %s" % arg
+ )
+
+ if not language:
+ try:
+ import locale
+
+ language = locale.getdefaultlocale()[0].replace("_", "-")
+ except Exception:
+ pass
+ if not language:
+ language = "en-US"
+
+ options.add_argument("--lang=%s" % language)
+
+ if not options.binary_location:
+ options.binary_location = find_chrome_executable()
+
+ self._delay = delay
+
+ self.user_data_dir = user_data_dir
+ self.keep_user_data_dir = keep_user_data_dir
+
+ if headless or options.headless:
+ options.headless = True
+ options.add_argument("--window-size=1920,1080")
+ options.add_argument("--start-maximized")
+
+ options.add_argument(
+ "--log-level=%d" % log_level
+ or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]
+ )
+
+ # fix exit_type flag to prevent tab-restore nag
+ try:
+ with open(
+ os.path.join(user_data_dir, "Default/Preferences"),
+ encoding="latin1",
+ mode="r+",
+ ) as fs:
+ config = json.load(fs)
+ if config["profile"]["exit_type"] is not None:
+ # fixing the restore-tabs-nag
+ config["profile"]["exit_type"] = None
+ fs.seek(0, 0)
+ fs.write(json.dumps(config, indent=4))
+ logger.debug("fixed exit_type flag")
+ except Exception as e:
+ logger.debug("did not find a bad exit_type flag ")
+
+ self.options = options
+
+ if not desired_capabilities:
+ desired_capabilities = options.to_capabilities()
+
+ self.browser = subprocess.Popen(
+ [options.binary_location, *options.arguments],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+ close_fds=True,
+ )
+
+ super().__init__(
+ executable_path=patcher.executable_path,
+ port=port,
+ options=options,
+ service_args=service_args,
+ desired_capabilities=desired_capabilities,
+ service_log_path=service_log_path,
+ keep_alive=keep_alive,
+ )
+
+ # self.webdriver = selenium.webdriver.chrome.webdriver.WebDriver(
+ # executable_path=patcher.executable_path,
+ # port=port,
+ # options=options,
+ # service_args=service_args,
+ # desired_capabilities=desired_capabilities,
+ # service_log_path=service_log_path,
+ # keep_alive=keep_alive,
+ # )
+
+ self.reactor = None
+ if enable_cdp_events:
+
+ if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+ logging.getLogger(
+ "selenium.webdriver.remote.remote_connection"
+ ).setLevel(20)
+
+ reactor = Reactor(self)
+ reactor.start()
+ self.reactor = reactor
+
+ if options.headless:
+ self._configure_headless()
+
+ def _configure_headless(self):
+
+ orig_get = self.get
+
+ logger.info("setting properties for headless")
+
+ def get_wrapped(*args, **kwargs):
+
+ if self.execute_script("return navigator.webdriver"):
+ logger.info("patch navigator.webdriver")
+ self.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {
+ "source": """
+
+ Object.defineProperty(window, 'navigator', {
+ value: new Proxy(navigator, {
+ has: (target, key) => (key === 'webdriver' ? false : key in target),
+ get: (target, key) =>
+ key === 'webdriver' ?
+ undefined :
+ typeof target[key] === 'function' ?
+ target[key].bind(target) :
+ target[key]
+ })
+ });
+
+ """
+ },
+ )
+
+ logger.info("patch user-agent string")
+ self.execute_cdp_cmd(
+ "Network.setUserAgentOverride",
+ {
+ "userAgent": self.execute_script(
+ "return navigator.userAgent"
+ ).replace("Headless", "")
+ },
+ )
+
+ if self.options.mock_permissions:
+ logger.info("patch permissions api")
+
+ self.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {
+ "source": """
+ // fix Notification permission in headless mode
+ Object.defineProperty(Notification, 'permission', { get: () => "default"});
+ """
+ },
+ )
+
+ if self.options.emulate_touch:
+ logger.info("patch emulate touch")
+
+ self.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {
+ "source": """
+ Object.defineProperty(navigator, 'maxTouchPoints', {
+ get: () => 1
+ })"""
+ },
+ )
+
+ if self.options.mock_canvas_fp:
+ logger.info("patch HTMLCanvasElement fingerprinting")
+
+ self.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {
+ "source": """
+ (function() {
+ const ORIGINAL_CANVAS = HTMLCanvasElement.prototype[name];
+ Object.defineProperty(HTMLCanvasElement.prototype, name, {
+ "value": function() {
+ var shift = {
+ 'r': Math.floor(Math.random() * 10) - 5,
+ 'g': Math.floor(Math.random() * 10) - 5,
+ 'b': Math.floor(Math.random() * 10) - 5,
+ 'a': Math.floor(Math.random() * 10) - 5
+ };
+ var width = this.width,
+ height = this.height,
+ context = this.getContext("2d");
+ var imageData = context.getImageData(0, 0, width, height);
+ for (var i = 0; i < height; i++) {
+ for (var j = 0; j < width; j++) {
+ var n = ((i * (width * 4)) + (j * 4));
+ imageData.data[n + 0] = imageData.data[n + 0] + shift.r;
+ imageData.data[n + 1] = imageData.data[n + 1] + shift.g;
+ imageData.data[n + 2] = imageData.data[n + 2] + shift.b;
+ imageData.data[n + 3] = imageData.data[n + 3] + shift.a;
+ }
+ }
+ context.putImageData(imageData, 0, 0);
+ return ORIGINAL_CANVAS.apply(this, arguments);
+ }
+ });
+ })(this)
+ """
+ },
+ )
+
+ if self.options.mock_chrome_global:
+ self.execute_cdp_cmd(
+ "Page.addScriptToEvaluateOnNewDocument",
+ {
+ "source": """
+
+ Object.defineProperty(window, 'chrome', {
+ value: new Proxy(window.chrome, {
+ has: (target, key) => true,
+ get: (target, key) => {
+ return {
+ app: {
+ isInstalled: false,
+ },
+ webstore: {
+ onInstallStageChanged: {},
+ onDownloadProgress: {},
+ },
+ runtime: {
+ PlatformOs: {
+ MAC: 'mac',
+ WIN: 'win',
+ ANDROID: 'android',
+ CROS: 'cros',
+ LINUX: 'linux',
+ OPENBSD: 'openbsd',
+ },
+ PlatformArch: {
+ ARM: 'arm',
+ X86_32: 'x86-32',
+ X86_64: 'x86-64',
+ },
+ PlatformNaclArch: {
+ ARM: 'arm',
+ X86_32: 'x86-32',
+ X86_64: 'x86-64',
+ },
+ RequestUpdateCheckStatus: {
+ THROTTLED: 'throttled',
+ NO_UPDATE: 'no_update',
+ UPDATE_AVAILABLE: 'update_available',
+ },
+ OnInstalledReason: {
+ INSTALL: 'install',
+ UPDATE: 'update',
+ CHROME_UPDATE: 'chrome_update',
+ SHARED_MODULE_UPDATE: 'shared_module_update',
+ },
+ OnRestartRequiredReason: {
+ APP_UPDATE: 'app_update',
+ OS_UPDATE: 'os_update',
+ PERIODIC: 'periodic',
+ },
+ },
+ }
+ }
+ })
+ });
+ """
+ },
+ )
+
+ return orig_get(*args, **kwargs)
+
+ self.get = get_wrapped
+
+ def __dir__(self):
+ return object.__dir__(self)
+
+ def add_cdp_listener(self, event_name, callback):
+ if (
+ self.reactor
+ and self.reactor is not None
+ and isinstance(self.reactor, Reactor)
+ ):
+ self.reactor.add_event_handler(event_name, callback)
+ return self.reactor.handlers
+ return False
+
+ def reconnect(self):
+ try:
+ self.service.stop()
+ except Exception as e:
+ logger.debug(e)
+
+ try:
+ self.service.start()
+ except Exception as e:
+ logger.debug(e)
+
+ try:
+ self.start_session()
+ except Exception as e:
+ logger.debug(e)
+
+ def start_session(self, capabilities=None, browser_profile=None):
+ if not capabilities:
+ capabilities = self.options.to_capabilities()
+ super().start_session(capabilities, browser_profile)
+
+ def quit(self):
+ logger.debug("closing webdriver")
+ try:
+ if self.reactor and isinstance(self.reactor, Reactor):
+ self.reactor.event.set()
+ super().quit()
+
+ except Exception: # noqa
+ pass
+ try:
+ logger.debug("killing browser")
+ self.browser.kill()
+ self.browser.wait(1)
+
+ except TimeoutError as e:
+ logger.debug(e, exc_info=True)
+ except Exception: # noqa
+ pass
+
+ if not self.keep_user_data_dir or self.keep_user_data_dir is False:
+ for _ in range(3):
+ try:
+ logger.debug("removing profile : %s" % self.user_data_dir)
+ shutil.rmtree(self.user_data_dir, ignore_errors=False)
+ except FileNotFoundError:
+ pass
+ except PermissionError:
+ logger.debug(
+ "permission error. files are still in use/locked. retying..."
+ )
+ else:
+ break
+ time.sleep(1)
+
+ def __del__(self):
+ logger.debug("Chrome.__del__")
+ self.quit()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.service.stop()
+ time.sleep(self._delay)
+ self.service.start()
+ self.start_session()
+
+ def __hash__(self):
+ return hash(self.options.debugger_address)
+
+ def find_elements_by_text(self, text: str):
+ for elem in self.find_elements_by_css_selector("*"):
+ try:
+ if text.lower() in elem.text.lower():
+ yield elem
+ except Exception as e:
+ logger.debug("find_elements_by_text: %s" % e)
+
+ def find_element_by_text(self, text: str, selector=None):
+ if not selector:
+ selector = "*"
+ for elem in self.find_elements_by_css_selector(selector):
+ try:
+ if text.lower() in elem.text.lower():
+ return elem
+ except Exception as e:
+ logger.debug("find_elements_by_text: {}".format(e))
+
+
def find_chrome_executable():
"""
Finds the chrome, chrome beta, chrome canary, chromium executable
@@ -94,628 +616,3 @@ def find_chrome_executable():
for candidate in candidates:
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
return os.path.normpath(candidate)
-
-
-class Chrome(object):
- """
- Controls the ChromeDriver and allows you to drive the browser.
-
- The webdriver file will be downloaded by this module automatically,
- you do not need to specify this. however, you may if you wish.
-
-
- Attributes
- ----------
-
-
- Methods
- -------
-
- reconnect()
-
- this can be useful in case of heavy detection methods
- -stops the chromedriver service which runs in the background
- -starts the chromedriver service which runs in the background
- -recreate session
-
-
- start_session(capabilities=None, browser_profile=None)
-
- differentiates from the regular method in that it does not
- require a capabilities argument. The capabilities are automatically
- recreated from the options at creation time.
-
-
- --------------------------------------------------------------------------
- NOTE:
- Chrome has everything included to work out of the box.
- it does not `need` customizations.
- any customizations MAY lead to trigger bot migitation systems.
-
- --------------------------------------------------------------------------
- """
-
- _instances = set()
-
- def __init__(
- self,
- executable_path=None,
- port=0,
- options=None,
- service_args=None,
- desired_capabilities=None,
- service_log_path=None,
- keep_alive=True,
- log_level=0,
- headless=False,
- emulate_touch=False,
- delay=5,
- ):
- """
- Creates a new instance of the chrome driver.
-
- Starts the service and then creates new instance of chrome driver.
-
-
- Parameters
- ----------
- executable_path: str, optional, default: None - use find_chrome_executable
- Path to the executable. If the default is used it assumes the executable is in the $PATH
-
- port: int, optional, default: 0
- port you would like the service to run, if left as 0, a free port will be found.
-
- options: ChromeOptions, optional, default: None - automatic useful defaults
- this takes an instance of ChromeOptions, mainly to customize browser behavior.
- anything other dan the default, for example extensions or startup options
- are not supported in case of failure, and can probably lowers your undetectability.
-
- service_args: list of str, optional, default: None
- arguments to pass to the driver service
-
- desired_capabilities: dict, optional, default: None - auto from config
- Dictionary object with non-browser specific capabilities only, such as "proxy" or "loggingPref".
-
- service_log_path: str, optional, default: None
- path to log information from the driver.
-
- keep_alive: bool, optional, default: True
- Whether to configure ChromeRemoteConnection to use HTTP keep-alive.
-
- log_level: int, optional, default: adapts to python global log level
-
- headless: bool, optional, default: False
- can also be specified in the options instance.
- Specify whether you want to use the browser in headless mode.
- warning: this lowers undetectability and not fully supported.
-
- emulate_touch: bool, optional, default: False
- if set to True, patches window.maxTouchPoints to always return non-zero
-
- delay: int, optional, default: 5
- delay in seconds to wait before giving back control.
- this is used only when using the context manager
- (`with` statement) to bypass, for example CloudFlare.
- 5 seconds is a foolproof value.
-
- """
-
- patcher = Patcher(executable_path=executable_path)
- patcher.auto()
-
- if not options:
- options = selenium.webdriver.chrome.webdriver.Options()
- try:
- if options.session and options.session is not None:
- # prevent reuse of options,
- # as it just appends arguments, not replace them
- # you'll get conflicts starting chrome
- raise RuntimeError("you cannot reuse the ChromeOptions object")
- except AttributeError:
- pass
-
- options.session = self
-
- debug_port = selenium.webdriver.common.service.utils.free_port()
- debug_host = "127.0.0.1"
- if not options.debugger_address:
- options.debugger_address = "%s:%d" % (debug_host, debug_port)
-
- options.add_argument("--remote-debugging-host=%s " % debug_host)
- options.add_argument("--remote-debugging-port=%s" % debug_port)
-
- # see if a custom user profile is specified
- for arg in options.arguments:
- if "user-data-dir" in arg:
- m = re.search("(?:--)?user-data-dir(?:[ =])?(.*)", arg)
- try:
- user_data_dir = m[1]
- logger.debug(
- "user-data-dir found in user argument %s => %s" % (arg, m[1])
- )
- keep_user_data_dir = True
- break
- except IndexError:
- logger.debug(
- "no user data dir could be extracted from supplied argument %s "
- % arg
- )
- else:
- user_data_dir = os.path.normpath(tempfile.mkdtemp())
- keep_user_data_dir = False
- arg = "--user-data-dir=%s" % user_data_dir
- options.add_argument(arg)
- logger.debug(
- "created a temporary folder in which the user-data (profile) will be stored during this\n"
- "session, and added it to chrome startup arguments: %s" % arg
- )
-
- if not options.binary_location:
- options.binary_location = find_chrome_executable()
-
- self._delay = delay
-
- self.user_data_dir = user_data_dir
- self.keep_user_data_dir = keep_user_data_dir
-
- if headless or options.headless:
- options.headless = True
- options.add_argument("--window-size=1920,1080")
- options.add_argument("--start-maximized")
-
- options.add_argument(
- "--log-level=%d" % log_level
- or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]
- )
-
- # fix exit_type flag to prevent tab-restore nag
- try:
- with open(
- os.path.join(user_data_dir, "Default/Preferences"),
- encoding="latin1",
- mode="r+",
- ) as fs:
- import json
-
- config = json.load(fs)
- if config["profile"]["exit_type"] is not None:
- # fixing the restore-tabs-nag
- config["profile"]["exit_type"] = None
- fs.seek(0, 0)
- fs.write(json.dumps(config, indent=4))
- logger.debug("fixed exit_type flag")
- except Exception as e:
- logger.debug("did not find a bad exit_type flag ")
-
- self.options = options
-
- if not desired_capabilities:
- desired_capabilities = options.to_capabilities()
-
- # unlock_port(debug_port)
-
- self.browser = subprocess.Popen(
- [options.binary_location, *options.arguments],
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- )
-
- self.webdriver = selenium.webdriver.chrome.webdriver.WebDriver(
- executable_path=patcher.executable_path,
- port=port,
- options=options,
- service_args=service_args,
- desired_capabilities=desired_capabilities,
- service_log_path=service_log_path,
- keep_alive=keep_alive,
- )
-
- self.__class__._instances.add((self, options))
- if options.headless:
- if emulate_touch:
- self.execute_cdp_cmd(
- "Page.addScriptToEvaluateOnNewDocument",
- {
- "source": """
- Object.defineProperty(navigator, 'maxTouchPoints', {
- get: () => 1
- })"""
- },
- )
-
- orig_get = self.webdriver.get
-
- logger.info("setting properties for headless")
-
- def get_wrapped(*args, **kwargs):
-
- if self.execute_script("return navigator.webdriver"):
- self.execute_cdp_cmd(
- "Page.addScriptToEvaluateOnNewDocument",
- {
- "source": """
- Object.defineProperty(window, 'navigator', {
- value: new Proxy(navigator, {
- has: (target, key) => (key === 'webdriver' ? false : key in target),
- get: (target, key) =>
- key === 'webdriver'
- ? undefined
- : typeof target[key] === 'function'
- ? target[key].bind(target)
- : target[key]
- })
- });
- """
- },
- )
-
- logger.info("removing headless from user-agent string")
-
- self.execute_cdp_cmd(
- "Network.setUserAgentOverride",
- {
- "userAgent": self.execute_script(
- "return navigator.userAgent"
- ).replace("Headless", "")
- },
- )
- self.execute_cdp_cmd(
- "Page.addScriptToEvaluateOnNewDocument",
- {
- "source": """
- // fix Notification permission in headless mode
- Object.defineProperty(Notification, 'permission', { get: () => "default"});
- """
- },
- )
-
- if emulate_touch:
- self.execute_cdp_cmd(
- "Page.addScriptToEvaluateOnNewDocument",
- {
- "source": """
- Object.defineProperty(navigator, 'maxTouchPoints', {
- get: () => 1
- })"""
- },
- )
- return orig_get(*args, **kwargs)
-
- self.webdriver.get = get_wrapped
-
- def __getattribute__(self, attr):
- try:
- return object.__getattribute__(self, attr)
- except AttributeError:
- try:
- return object.__getattribute__(self.webdriver, attr)
- except AttributeError:
- raise
-
- def __dir__(self):
- return object.__dir__(self) + object.__dir__(self.webdriver)
-
- def reconnect(self):
- try:
- self.service.stop()
- except Exception as e:
- logger.debug(e)
-
- try:
- self.service.start()
- except Exception as e:
- logger.debug(e)
-
- try:
- self.start_session()
- except Exception as e:
- logger.debug(e)
-
- def start_session(self, capabilities=None, browser_profile=None):
- if not capabilities:
- capabilities = self.options.to_capabilities()
- self.webdriver.start_session(capabilities, browser_profile)
-
- def quit(self):
- logger.debug("closing webdriver")
- try:
- self.webdriver.quit()
- except Exception: # noqa
- pass
- try:
- logger.debug("killing browser")
- self.browser.kill()
- self.browser.wait(1)
- except TimeoutError as e:
- logger.debug(e, exc_info=True)
- except Exception: # noqa
- pass
- if not self.keep_user_data_dir or self.keep_user_data_dir is False:
- for _ in range(3):
- try:
- logger.debug("removing profile : %s" % self.user_data_dir)
- shutil.rmtree(self.user_data_dir, ignore_errors=False)
- except FileNotFoundError:
- pass
- except PermissionError:
- logger.debug(
- "permission error. files are still in use/locked. retying..."
- )
- else:
- break
- time.sleep(1)
-
- def __del__(self):
- self.quit()
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.service.stop()
- time.sleep(self._delay)
- self.service.start()
- self.start_session()
-
- def __hash__(self):
- return hash(self.options.debugger_address)
-
- def find_elements_by_text(self, text: str):
- for elem in self.find_elements_by_css_selector("*"):
- try:
- if text.lower() in elem.text.lower():
- yield elem
- except Exception as e:
- logger.debug("find_elements_by_text: %s" % e)
-
- def find_element_by_text(self, text: str):
- for elem in self.find_elements_by_css_selector("*"):
- try:
- if text.lower() in elem.text.lower():
- return elem
- except Exception as e:
- logger.debug("find_elements_by_text: %s" % e)
-
-
-class Patcher(object):
- url_repo = "https://chromedriver.storage.googleapis.com"
- zip_name = "chromedriver_%s.zip"
- exe_name = "chromedriver%s"
-
- platform = sys.platform
- if platform.endswith("win32"):
- zip_name %= "win32"
- exe_name %= ".exe"
- if platform.endswith("linux"):
- zip_name %= "linux64"
- exe_name %= ""
- if platform.endswith("darwin"):
- zip_name %= "mac64"
- exe_name %= ""
-
- if platform.endswith("win32"):
- d = "~/appdata/roaming/undetected_chromedriver"
- elif platform.startswith("linux"):
- d = "~/.local/share/undetected_chromedriver"
- elif platform.endswith("darwin"):
- d = "~/Library/Application Support/undetected_chromedriver"
- else:
- d = "~/.undetected_chromedriver"
- data_path = os.path.abspath(os.path.expanduser(d))
-
- def __init__(self, executable_path=None, force=False, version_main: int = 0):
- """
-
- Args:
- executable_path: None = automatic
- a full file path to the chromedriver executable
- force: False
- terminate processes which are holding lock
- version_main: 0 = auto
- specify main chrome version (rounded, ex: 82)
- """
-
- self.force = force
-
- if not executable_path:
- executable_path = os.path.join(self.data_path, self.exe_name)
-
- if not IS_POSIX:
- if not executable_path[-4:] == ".exe":
- executable_path += ".exe"
-
- self.zip_path = os.path.join(self.data_path, self.zip_name)
-
- self.executable_path = os.path.abspath(os.path.join(".", executable_path))
-
- self.version_main = version_main
- self.version_full = None
-
- @classmethod
- def auto(cls, executable_path=None, force=False):
- """
-
- Args:
- force:
-
- Returns:
-
- """
- i = cls(executable_path, force=force)
- try:
- os.unlink(i.executable_path)
- except PermissionError:
- if i.force:
- cls.force_kill_instances(i.executable_path)
- return i.auto(force=False)
- try:
- if i.is_binary_patched():
- # assumes already running AND patched
- return True
- except PermissionError:
- pass
- # return False
- except FileNotFoundError:
- pass
-
- release = i.fetch_release_number()
- i.version_main = release.version[0]
- i.version_full = release
- i.unzip_package(i.fetch_package())
- i.patch()
- return i
-
- def patch(self):
- self.patch_exe()
- return self.is_binary_patched()
-
- def fetch_release_number(self):
- """
- Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
- :return: version string
- :rtype: LooseVersion
- """
- path = "/latest_release"
- if self.version_main:
- path += f"_{self.version_main}"
- path = path.upper()
- logger.debug("getting release number from %s" % path)
- return LooseVersion(urlopen(self.url_repo + path).read().decode())
-
- def parse_exe_version(self):
- with io.open(self.executable_path, "rb") as f:
- for line in iter(lambda: f.readline(), b""):
- match = re.search(br"platform_handle\x00content\x00([0-9.]*)", line)
- if match:
- return LooseVersion(match[1].decode())
-
- def fetch_package(self):
- """
- Downloads ChromeDriver from source
-
- :return: path to downloaded file
- """
- u = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, self.zip_name)
- logger.debug("downloading from %s" % u)
- # return urlretrieve(u, filename=self.data_path)[0]
- return urlretrieve(u)[0]
-
- def unzip_package(self, fp):
- """
- Does what it says
-
- :return: path to unpacked executable
- """
- logger.debug("unzipping %s" % fp)
- try:
- os.unlink(self.zip_path)
- except (FileNotFoundError, OSError):
- pass
-
- os.makedirs(self.data_path, mode=0o755, exist_ok=True)
-
- with zipfile.ZipFile(fp, mode="r") as zf:
- zf.extract(self.exe_name, os.path.dirname(self.executable_path))
- os.remove(fp)
- os.chmod(self.executable_path, 0o755)
- return self.executable_path
-
- @staticmethod
- def force_kill_instances(exe_name):
- """
- kills running instances.
- :param: executable name to kill, may be a path as well
-
- :return: True on success else False
- """
- exe_name = os.path.basename(exe_name)
- if IS_POSIX:
- r = os.system("kill -f -9 $(pidof %s)" % exe_name)
- else:
- r = os.system("taskkill /f /im %s" % exe_name)
- return not r
-
- @staticmethod
- def gen_random_cdc():
- cdc = random.choices(string.ascii_lowercase, k=26)
- cdc[-6:-4] = map(str.upper, cdc[-6:-4])
- cdc[2] = cdc[0]
- cdc[3] = "_"
- return "".join(cdc).encode()
-
- def is_binary_patched(self, executable_path=None):
- """simple check if executable is patched.
-
- :return: False if not patched, else True
- """
- executable_path = executable_path or self.executable_path
- with io.open(executable_path, "rb") as fh:
- for line in iter(lambda: fh.readline(), b""):
- if b"cdc_" in line:
- return False
- else:
- return True
-
- def patch_exe(self):
- """
- Patches the ChromeDriver binary
-
- :return: False on failure, binary name on success
- """
- logger.info("patching driver executable %s" % self.executable_path)
-
- linect = 0
- replacement = self.gen_random_cdc()
- with io.open(self.executable_path, "r+b") as fh:
- for line in iter(lambda: fh.readline(), b""):
- if b"cdc_" in line:
- fh.seek(-len(line), 1)
- newline = re.sub(b"cdc_.{22}", replacement, line)
- fh.write(newline)
- linect += 1
- return linect
-
- def __repr__(self):
- return "{0:s}({1:s})".format(
- self.__class__.__name__,
- self.executable_path,
- )
-
-
-#
-#
-# def unlock_port(port):
-# import os
-# if not IS_POSIX:
-# try:
-#
-# c = subprocess.Popen('netstat -ano | findstr :%d' % port, shell=True, stdout=subprocess.PIPE,
-# stderr=subprocess.PIPE)
-# stdout, stderr = c.communicate()
-# lines = stdout.splitlines()
-# _pid = lines[0].split(b' ')[-1].decode()
-# c = subprocess.Popen(['taskkill', '/f', '/pid', _pid], shell=True, stdout=subprocess.PIPE,
-# stderr=subprocess.PIPE)
-# stdout, stderr = c.communicate()
-# except Exception as e:
-# logger.debug(e)
-#
-# else:
-# try:
-# os.system('kill -15 $(lsof -i:%d)' % port)
-# except Exception:
-# pass
-#
-
-
-class ChromeOptions(_ChromeOptions):
-
- session = None
-
- def add_extension_file_crx(self, extension=None):
- if extension:
- extension_to_add = os.path.abspath(os.path.expanduser(extension))
- logger.debug("extension_to_add: %s" % extension_to_add)
-
- return super().add_extension(r"%s" % extension)