Merge pull request #357 from ultrafunkamsterdam/3.0.4

3.0.4

README.md

@@ -11,6 +11,28 @@ Automatically downloads the driver binary and patches it.
 * Python 3.6++**
 
 
+### 3.0.4 changes ####
+ - change process creation behavior to be fully detached
+ - changed .get(url) method to always use the contextmanager 
+ - changed .get(url) method to use cdp under the hood. 
+
+    ... the `with` statement is not necessary anymore ..
+ 
+ - todo: work towards asyncification and selenium 4
+ 
+ #### words of wisdom: ####
+ Whenever you encounter the daunted
+ 
+ ```from session not created: This version of ChromeDriver only supports Chrome version 96 # or what ever version``` 
+ 
+ the solution is simple:  
+ ```python
+   import undetected_chromedriver.v2 as uc
+   driver = uc.Chrome(version_main=95)
+ ```
+
+ 
+ 
 **July 2021: Currently busy implementing selenium 4 for undetected-chromedriver**
 
 **newsflash: https://github.com/ultrafunkamsterdam/undetected-chromedriver/pull/255**
@@ -34,7 +56,6 @@ This is also the snippet i recommend using in case you experience an issue.
 ```python
 import undetected_chromedriver.v2 as uc
 driver = uc.Chrome()
-with driver:
 driver.get('https://nowsecure.nl')  # known url using cloudflare's "under attack mode"
 ```
 
@@ -56,9 +77,8 @@ options.add_argument('--user-data-dir=c:\\temp\\profile2')
 
 # just some options passing in to skip annoying popups
 options.add_argument('--no-first-run --no-service-autorun --password-store=basic')
-driver = uc.Chrome(options=options)
+driver = uc.Chrome(options=options, version_main=94)  # version_main allows to specify your chrome version instead of following chrome global version
 
-with driver:
 driver.get('https://nowsecure.nl')  # known url using cloudflare's "under attack mode"
 
 ```
@@ -76,7 +96,7 @@ However i implemented my own for now. Since i needed it myself for investigation
 import undetected_chromedriver.v2 as uc
 from pprint import pformat
 
-driver = uc.Chrome(enable_cdp_event=True)
+driver = uc.Chrome(enable_cdp_events=True)
 
 def mylousyprintfunction(eventdata):
     print(pformat(eventdata))
@@ -102,7 +122,6 @@ driver.add_cdp_listener('Network.dataReceived', mylousyprintfunction)
 
 # now all these events will be printed in my console
 
-with driver:
 driver.get('https://nowsecure.nl')
 
 

undetected_chromedriver/__init__.py

@@ -34,7 +34,7 @@ from . import options
 
 ChromeOptionsV2 = v2.ChromeOptions
 logger = logging.getLogger(__name__)
-__version__ = "3.0.3"
+__version__ = "3.0.4"
 
 
 TARGET_VERSION = 0

undetected_chromedriver/dprocess.py

@@ -0,0 +1,72 @@
+import multiprocessing
+import os
+import platform
+import sys
+from subprocess import PIPE
+from subprocess import Popen
+import atexit
+import traceback
+import logging
+import signal
+
+CREATE_NEW_PROCESS_GROUP = 0x00000200
+DETACHED_PROCESS = 0x00000008
+
+REGISTERED = []
+
+
+def start_detached(executable, *args):
+    """
+    Starts a fully independent subprocess (with no parent)
+    :param executable: executable
+    :param args: arguments to the executable, eg: ['--param1_key=param1_val', '-vvv' ...]
+    :return: pid of the grandchild process
+    """
+
+    # create pipe
+    reader, writer = multiprocessing.Pipe(False)
+
+    # do not keep reference
+    multiprocessing.Process(target=_start_detached, args=(executable, *args), kwargs={'writer': writer},
+                            daemon=True).start()
+    # receive pid from pipe
+    pid = reader.recv()
+    REGISTERED.append(pid)
+    # close pipes
+    writer.close()
+    reader.close()
+
+    return pid
+
+
+def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
+
+    # configure launch
+    kwargs = {}
+    if platform.system() == 'Windows':
+        kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
+    elif sys.version_info < (3, 2):
+        # assume posix
+        kwargs.update(preexec_fn=os.setsid)
+    else:  # Python 3.2+ and Unix
+        kwargs.update(start_new_session=True)
+
+    # run
+    p = Popen([executable, *args], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs)
+
+    # send pid to pipe
+    writer.send(p.pid)
+    exit()
+
+
+def _cleanup():
+    for pid in REGISTERED:
+        try:
+            logging.getLogger(__name__).debug('cleaning up pid %d ' % pid)
+            os.kill(pid, signal.SIGTERM)
+        except:  # noqa
+            pass
+
+
+atexit.register(_cleanup)
+

undetected_chromedriver/v2.py

@@ -3,24 +3,28 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 import os
 import re
 import shutil
-import subprocess
 import sys
 import tempfile
 import time
+import inspect
 
+import requests
 import selenium.webdriver.chrome.service
 import selenium.webdriver.chrome.webdriver
 import selenium.webdriver.common.service
 import selenium.webdriver.remote.webdriver
+import websockets
 
 from .cdp import CDP
 from .options import ChromeOptions
-from .patcher import IS_POSIX, Patcher
+from .patcher import IS_POSIX
+from .patcher import Patcher
 from .reactor import Reactor
 
 __all__ = (
@@ -35,6 +39,8 @@ __all__ = (
 logger = logging.getLogger("uc")
 logger.setLevel(logging.getLogger().getEffectiveLevel())
 
+from .dprocess import start_detached
+
 
 class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
     """
@@ -167,7 +173,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
         try:
             if hasattr(options, "_session") and options._session is not None:
-
                 #  prevent reuse of options,
                 #  as it just appends arguments, not replace them
                 #  you'll get conflicts starting chrome
@@ -291,14 +296,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         if not desired_capabilities:
             desired_capabilities = options.to_capabilities()
 
+        self.browser_pid = start_detached(options.binary_location, *options.arguments)
 
-        self.browser = subprocess.Popen(
+        # self.browser = subprocess.Popen(
-            [options.binary_location, *options.arguments],
+        #     [options.binary_location, *options.arguments],
-            stdin=subprocess.PIPE,
+        #     stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
+        #     stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+        #     stderr=subprocess.PIPE,
-            close_fds=IS_POSIX,
+        #     close_fds=IS_POSIX,
-        )
+        # )
 
         super(Chrome, self).__init__(
             executable_path=patcher.executable_path,
@@ -523,6 +529,22 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
     def __dir__(self):
         return object.__dir__(self)
 
+    def get(self, url):
+
+        tabs = requests.get('http://{0}:{1}/json'.format(*self.options.debugger_address.split(':'))).json()
+        for tab in tabs:
+            if tab['type'] == 'page':
+                break
+
+        async def _get():
+            wsurl = tab['webSocketDebuggerUrl']
+            async with websockets.connect(wsurl) as ws:
+                await ws.send(json.dumps({"method": "Page.navigate", "params": {"url": url}, "id": 1}))
+                return await ws.recv()
+
+        with self:
+            return asyncio.get_event_loop().run_until_complete(_get())
+
     def add_cdp_listener(self, event_name, callback):
         if (
                 self.reactor
@@ -577,7 +599,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             capabilities = self.options.to_capabilities()
         super(Chrome, self).start_session(capabilities, browser_profile)
 
-
     def quit(self):
         logger.debug("closing webdriver")
         self.service.process.kill()
@@ -588,8 +609,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             pass
         try:
             logger.debug("killing browser")
-            self.browser.terminate()
+            os.kill(self.browser_pid)
-            self.browser.wait(1)
+            # self.browser.terminate()
+            # self.browser.wait(1)
 
         except TimeoutError as e:
             logger.debug(e, exc_info=True)
@@ -625,6 +647,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         self.quit()
 
     def __enter__(self):
+        try:
+            curframe = inspect.currentframe()
+            callframe = inspect.getouterframes(curframe, 2)
+            caller = callframe[1][3]
+            logging.getLogger(__name__).debug('__enter__ caller: %s' % caller)
+            if caller == 'get':
+                return
+        except (AttributeError, ValueError, KeyError, OSError) as e:
+            logging.getLogger(__name__).debug(e)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -637,7 +668,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         return hash(self.options.debugger_address)
 
 
-
 def find_chrome_executable():
     """
     Finds the chrome, chrome beta, chrome canary, chromium executable