Resort to the requests backend when using proxies

salt/utils/http.py

@@ -35,6 +35,7 @@
 import salt.utils.network
 import salt.utils.platform
 import salt.utils.stringutils
+import salt.utils.url
 import salt.utils.xmlutil as xml
 import salt.utils.yaml
 import salt.version
@@ -61,14 +62,6 @@
             HAS_MATCHHOSTNAME = False
     # pylint: enable=no-name-in-module
 
-
-try:
-    import salt.ext.tornado.curl_httpclient
-
-    HAS_CURL_HTTPCLIENT = True
-except ImportError:
-    HAS_CURL_HTTPCLIENT = False
-
 try:
     import requests
 
@@ -223,6 +216,39 @@ def query(
     if not backend:
         backend = opts.get("backend", "tornado")
 
+    proxy_host = opts.get("proxy_host", None)
+    if proxy_host:
+        proxy_host = salt.utils.stringutils.to_str(proxy_host)
+    proxy_port = opts.get("proxy_port", None)
+    proxy_username = opts.get("proxy_username", None)
+    if proxy_username:
+        proxy_username = salt.utils.stringutils.to_str(proxy_username)
+    proxy_password = opts.get("proxy_password", None)
+    if proxy_password:
+        proxy_password = salt.utils.stringutils.to_str(proxy_password)
+    no_proxy = opts.get("no_proxy", [])
+
+    if urllib.parse.urlparse(url).hostname in no_proxy:
+        proxy_host = None
+        proxy_port = None
+        proxy_username = None
+        proxy_password = None
+
+    proxy_args = None
+    if backend != "requests" and proxy_host and proxy_port:
+        log.debug("Switching to request backend due to the use of proxies.")
+        backend = "requests"
+        scheme = urllib.parse.urlparse(proxy_host).scheme
+        proxy_url = f"{proxy_host}:{proxy_port}"
+        if proxy_username and proxy_password:
+            proxy_url = salt.utils.url.add_http_basic_auth(
+                proxy_url, proxy_username, proxy_password
+            )
+        if scheme and proxy_url:
+            proxy_args = (scheme, proxy_url)
+        else:
+            log.debug("Failed to set proxy details")
+
     match = re.match(
         r"https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)",
         url,
@@ -336,6 +362,8 @@ def query(
         log.trace("Request Headers: %s", sess.headers)
         sess_cookies = sess.cookies
         sess.verify = verify_ssl
+        if proxy_args:
+            sess.proxies = {proxy_args[0]: proxy_args[1]}
     elif backend == "urllib2":
         sess_cookies = None
     else:
@@ -554,52 +582,10 @@ def query(
             salt.config.DEFAULT_MINION_OPTS["http_request_timeout"],
         )
 
-        client_argspec = None
-
-        proxy_host = opts.get("proxy_host", None)
-        if proxy_host:
-            # tornado requires a str for proxy_host, cannot be a unicode str in py2
-            proxy_host = salt.utils.stringutils.to_str(proxy_host)
-        proxy_port = opts.get("proxy_port", None)
-        proxy_username = opts.get("proxy_username", None)
-        if proxy_username:
-            # tornado requires a str, cannot be unicode str in py2
-            proxy_username = salt.utils.stringutils.to_str(proxy_username)
-        proxy_password = opts.get("proxy_password", None)
-        if proxy_password:
-            # tornado requires a str, cannot be unicode str in py2
-            proxy_password = salt.utils.stringutils.to_str(proxy_password)
-        no_proxy = opts.get("no_proxy", [])
-
-        # Since tornado doesnt support no_proxy, we'll always hand it empty proxies or valid ones
-        # except we remove the valid ones if a url has a no_proxy hostname in it
-        if urllib.parse.urlparse(url_full).hostname in no_proxy:
-            proxy_host = None
-            proxy_port = None
-            proxy_username = None
-            proxy_password = None
-
-        # We want to use curl_http if we have a proxy defined
-        if proxy_host and proxy_port:
-            if HAS_CURL_HTTPCLIENT is False:
-                ret["error"] = (
-                    "proxy_host and proxy_port has been set. This requires pycurl and"
-                    " tornado, but the libraries does not seem to be installed"
-                )
-                log.error(ret["error"])
-                return ret
-
-            salt.ext.tornado.httpclient.AsyncHTTPClient.configure(
-                "tornado.curl_httpclient.CurlAsyncHTTPClient"
-            )
-            client_argspec = salt.utils.args.get_function_argspec(
-                salt.ext.tornado.curl_httpclient.CurlAsyncHTTPClient.initialize
-            )
-        else:
-            salt.ext.tornado.httpclient.AsyncHTTPClient.configure(None)
-            client_argspec = salt.utils.args.get_function_argspec(
-                salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
-            )
+        salt.ext.tornado.httpclient.AsyncHTTPClient.configure(None)
+        client_argspec = salt.utils.args.get_function_argspec(
+            salt.ext.tornado.simple_httpclient.SimpleAsyncHTTPClient.initialize
+        )
 
         supports_max_body_size = "max_body_size" in client_argspec.args
 
@@ -616,10 +602,6 @@ def query(
                 "header_callback": header_callback,
                 "connect_timeout": connect_timeout,
                 "request_timeout": timeout,
-                "proxy_host": proxy_host,
-                "proxy_port": proxy_port,
-                "proxy_username": proxy_username,
-                "proxy_password": proxy_password,
                 "raise_error": raise_error,
                 "decompress_response": False,
             }