Merge pull request #737 from benoitc/happy_eyeballs

improve connection to remote

NOTICE

@@ -21,3 +21,6 @@ Copyright (c) 2009, Erlang Training and Consulting Ltd.
 Copyright (C) 1998 - 2014, Daniel Stenberg, <[email protected]>, et al.
 
 *) hackney_trace (C) 2015 under the Erlang Public LicensE
+
+*) hackney_cidr is based on inet_cidr 1.2.1. vendored for customer purpose.
+Copyright (c) 2024, Enki Multimedia , MIT License

include/hackney.hrl

@@ -59,3 +59,4 @@
 
 -define(HTTP_PROXY_ENV_VARS, ["http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"]).
 -define(HTTPS_PROXY_ENV_VARS, ["https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"]).
+-define(HTTP_NO_PROXY_ENV_VARS, ["no_proxy", "NO_PROXY"]).

src/hackney.erl

@@ -311,17 +311,18 @@ request(Method, #hackney_url{}=URL0, Headers0, Body, Options0) ->
   URL = hackney_url:normalize(URL0, PathEncodeFun),
 
   ?report_trace("request", [{method, Method},
-    {url, URL},
-    {headers, Headers0},
-    {body, Body},
-    {options, Options0}]),
+                            {url, URL},
+                            {headers, Headers0},
+                            {body, Body},
+                            {options, Options0}]),
 
   #hackney_url{transport=Transport,
-    host = Host,
-    port = Port,
-    user = User,
-    password = Password,
-    scheme = Scheme} = URL,
+               host = Host,
+               port = Port,
+               user = User,
+               password = Password,
+               scheme = Scheme} = URL,
+
 
   Options = case User of
               <<>> ->
@@ -676,14 +677,22 @@ maybe_proxy(Transport, Scheme, Host, Port, Options)
   end.
 
 maybe_proxy_from_env(Transport, _Scheme, Host, Port, Options, true) ->
-  ?report_debug("request without proxy", []),
+  ?report_debug("no proxy env is forced, request without proxy", []),
   hackney_connect:connect(Transport, Host, Port, Options, true);
 maybe_proxy_from_env(Transport, Scheme, Host, Port, Options, _) ->
   case get_proxy_env(Scheme) of
     {ok, Url} ->
-      proxy_from_url(Url, Transport, Host, Port, Options);
+      NoProxyEnv = get_no_proxy_env(),
+      case match_no_proxy_env(NoProxyEnv, Host) of
+        false ->
+          ?report_debug("request with proxy", [{proxy, Url}, {host, Host}]),
+          proxy_from_url(Url, Transport, Host, Port, Options);
+        true ->
+         ?report_debug("request without proxy", []),
+         hackney_connect:connect(Transport, Host, Port, Options, true)
+      end;
     false ->
-      ?report_debug("request without proxy", []),
+      ?report_debug("no proxy env setup, request without proxy", []),
       hackney_connect:connect(Transport, Host, Port, Options, true)
   end.
 
@@ -705,17 +714,121 @@ proxy_from_url(Url, Transport, Host, Port, Options) ->
       end
   end.
 
+get_no_proxy_env() ->
+  case application:get_env(hackney, no_proxy) of
+    undefined ->
+      case get_no_proxy_env(?HTTP_NO_PROXY_ENV_VARS) of
+        false ->
+          application:set_env(hackney, no_proxy, false),
+          false;
+        NoProxyEnv ->
+          parse_no_proxy_env(NoProxyEnv, [])
+      end;
+    {ok, NoProxyEnv} ->
+      NoProxyEnv
+  end.
+
+get_no_proxy_env([Key | Rest]) ->
+  case os:getenv(Key) of
+    false -> get_no_proxy_env(Rest);
+    NoProxyStr ->
+      lists:usort(string:tokens(NoProxyStr, ","))
+  end;
+get_no_proxy_env([]) ->
+  false.
+
+parse_no_proxy_env(["*" | _], _Acc) ->
+  application:set_env(hackney, no_proxy, '*'),
+  '*';
+parse_no_proxy_env([S | Rest], Acc) ->
+  try
+    CIDR = hackney_cidr:parse(S),
+    parse_no_proxy_env(Rest, [{cidr, CIDR} | Acc])
+  catch
+    _:_ ->
+      Labels = string:tokens(S, "."),
+      parse_no_proxy_env(Rest, [{host, lists:reverse(Labels)}])
+  end;
+parse_no_proxy_env([], Acc) ->
+  NoProxy = lists:reverse(Acc),
+  application:set_env(hackney, no_proxy, NoProxy),
+  NoProxy.
+
+match_no_proxy_env(false, _Host) -> false;
+match_no_proxy_env('*', _Host) -> true;
+match_no_proxy_env(Patterns, Host) ->
+  do_match_no_proxy_env(Patterns, undefined, undefined, Host).
+
+do_match_no_proxy_env([{cidr, _CIDR} | _]=Patterns, undefined, Labels, Host) ->
+  Addrs = case inet:parse_address(Host) of
+            {ok, Addr} -> [Addr];
+            _ -> getaddrs(Host)
+          end,
+  do_match_no_proxy_env(Patterns, Addrs, Labels, Host);
+do_match_no_proxy_env([{cidr, CIDR} | Rest], Addrs, Labels, Host) ->
+  case test_host_cidr(Addrs, CIDR) of
+    true -> true;
+    false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host)
+  end;
+do_match_no_proxy_env([{host, _Labels} | _] = Patterns, Addrs, undefined, Host) ->
+  HostLabels = string:tokens(Host, "."),
+  do_match_no_proxy_env(Patterns, Addrs, lists:reverse(HostLabels), Host);
+do_match_no_proxy_env([{host, Labels} | Rest], Addrs, HostLabels, Host) ->
+  case test_host_labels(Labels, HostLabels) of
+    true -> true;
+    false -> do_match_no_proxy_env(Rest, Addrs, Labels, Host)
+  end;
+do_match_no_proxy_env([], _, _, _) ->
+  false.
+
+test_host_labels(["*" | R1], [_ | R2]) -> test_host_labels(R1, R2);
+test_host_labels([ A | R1], [A | R2]) -> test_host_labels(R1, R2);
+test_host_labels([], _) -> true;
+test_host_labels(_, _) -> false.
+
+test_host_cidr([Addr, Rest], CIDR) ->
+  case hackney_cidr:contains(CIDR, Addr) of
+    true -> true;
+    false -> test_host_cidr(Rest, CIDR)
+  end;
+test_host_cidr([], _) ->
+  false.
+
+getaddrs(Host) ->
+  IP4Addrs = case inet:getaddrs(Host, inet) of
+               {ok, Addrs} -> Addrs;
+               {error, nxdomain} -> []
+             end,
+  case inet:getaddrs(Host, inet6) of
+    {ok, IP6Addrs} -> [IP6Addrs | IP4Addrs];
+    {error, nxdomain} -> IP4Addrs
+  end.
+
 get_proxy_env(https) ->
-  get_proxy_env(?HTTPS_PROXY_ENV_VARS);
+  case application:get_env(hackney, https_proxy) of
+    undefined ->
+      ProxyEnv = do_get_proxy_env(?HTTPS_PROXY_ENV_VARS),
+      application:set_env(hackney, https_proxy, ProxyEnv),
+      ProxyEnv;
+    {ok, Cached} ->
+      Cached
+  end;
 get_proxy_env(S) when S =:= http; S =:= http_unix ->
-  get_proxy_env(?HTTP_PROXY_ENV_VARS);
+  case application:get_env(hackney, http_proxy) of
+    undefined ->
+      ProxyEnv = do_get_proxy_env(?HTTP_PROXY_ENV_VARS),
+      application:set_env(hackney, http_proxy, ProxyEnv),
+      ProxyEnv;
+    {ok, Cached} ->
+      Cached
+  end.
 
-get_proxy_env([Var | Rest]) ->
+do_get_proxy_env([Var | Rest]) ->
   case os:getenv(Var) of
-    false -> get_proxy_env(Rest);
+    false -> do_get_proxy_env(Rest);
     Url -> {ok, Url}
   end;
-get_proxy_env([]) ->
+do_get_proxy_env([]) ->
   false.
 
 do_connect(ProxyHost, ProxyPort, undefined, Transport, Host, Port, Options) ->

src/hackney_connection.erl

@@ -102,27 +102,13 @@ connect_options(hackney_local_tcp, _Host, ClientOptions) ->
   proplists:get_value(connect_options, ClientOptions, []);
 
 connect_options(Transport, Host, ClientOptions) ->
-  ConnectOpts0 = proplists:get_value(connect_options, ClientOptions, []),
-
-  %% handle ipv6
-  ConnectOpts1 = case lists:member(inet, ConnectOpts0) orelse
-                      lists:member(inet6, ConnectOpts0) of
-                   true ->
-                     ConnectOpts0;
-                   false ->
-                     case hackney_util:is_ipv6(Host) of
-                       true ->
-                         [inet6 | ConnectOpts0];
-                       false ->
-                         ConnectOpts0
-                     end
-                 end,
+  ConnectOpts = proplists:get_value(connect_options, ClientOptions, []),
 
   case Transport of
     hackney_ssl ->
-      ConnectOpts1 ++ ssl_opts(Host, ClientOptions);
+      [{ssl_options, ssl_opts(Host, ClientOptions)} | ConnectOpts];
     _ ->
-      ConnectOpts1
+      ConnectOpts
   end.
 
 

src/hackney_happy.erl

@@ -0,0 +1,140 @@
+-module(hackney_happy).
+
+-export([connect/3, connect/4]).
+
+-include("hackney_internal.hrl").
+-include_lib("kernel/include/inet.hrl").
+
+-define(TIMEOUT, 250).
+-define(CONNECT_TIMEOUT, 5000).
+
+connect(Hostname, Port, Opts) ->
+  connect(Hostname, Port, Opts, ?CONNECT_TIMEOUT).
+
+connect(Hostname, Port, Opts, Timeout) ->
+  do_connect(parse_address(Hostname), Port, Opts, Timeout).
+
+do_connect(Hostname, Port, Opts, Timeout) when is_tuple(Hostname) ->
+  case hackney_cidr:is_ipv6(Hostname) of
+    true ->
+      ?report_debug("connect using IPv6", [{hostname, Hostname}, {port, Port}]),
+      gen_tcp:connect(Hostname, Port, [inet6 | Opts], Timeout);
+    false ->
+      case hackney_cidr:is_ipv4(Hostname) of
+        true ->
+          ?report_debug("connect using IPv4", [{hostname, Hostname}, {port, Port}]),
+          gen_tcp:connect(Hostname, Port, [inet | Opts], Timeout);
+        false ->
+          {error, nxdomain}
+      end
+  end;
+do_connect(Hostname, Port, Opts, Timeout) ->
+  ?report_debug("happy eyeballs, try to connect using IPv6",  [{hostname, Hostname}, {port, Port}]),
+  Self = self(),
+  {Ipv6Addrs, IPv4Addrs} = getaddrs(Hostname),
+  {Pid6, MRef6} = spawn_monitor(fun() -> try_connect(Ipv6Addrs, Port, Opts, Self) end),
+  TRef = erlang:start_timer(?TIMEOUT, self(), try_ipv4),
+  receive
+    {'DOWN', MRef6, _Type, _Pid, {happy_connect, OK}} ->
+      _ = erlang:cancel_timer(TRef, []),
+      OK;
+    {'DOWN', MRef6, _Type, _Pid, _Info} ->
+      _ = erlang:cancel_timer(TRef, []),
+      {Pid4, MRef4} = spawn_monitor(fun() -> try_connect(IPv4Addrs, Port, Opts, Self) end),
+      do_connect_2(Pid4, MRef4, Timeout);
+    {timeout, TRef, try_ipv4} ->
+      PidRef4 = spawn_monitor(fun() -> try_connect(IPv4Addrs, Port, Opts, Self) end),
+      do_connect_1(PidRef4, {Pid6, MRef6}, Timeout)
+  after Timeout -> 
+          _ = erlang:cancel_timer(TRef, []),
+          erlang:demonitor(MRef6, [flush]),
+          {error, connect_timeout}
+  end.
+
+
+do_connect_1({Pid4, MRef4}, {Pid6, MRef6}, Timeout) ->
+  receive
+    {'DOWN', MRef4, _Type, _Pid, {happy_connect, OK}} ->
+      ?report_trace("happy_connect ~p", [OK]),
+      connect_gc(Pid6, MRef6),
+      OK;
+    {'DOWN', MRef4, _Type, _Pid, _Info} ->
+      do_connect_2(Pid6, MRef6, Timeout);
+    {'DOWN', MRef6, _Type, _Pid, {happy_connect, OK}} ->
+      ?report_trace("happy_connect ~p", [OK]),
+      connect_gc(Pid4, MRef4),
+      OK;
+    {'DOWN', MRef6, _Type, Pid, _Info} ->
+      do_connect_2(Pid4, MRef4, Timeout)
+  after Timeout ->
+          connect_gc(Pid4, MRef4),
+          connect_gc(Pid6, MRef6),
+          {error, connect_timeout}
+  end.
+
+do_connect_2(Pid, MRef, Timeout) ->
+  receive
+    {'DOWN', MRef, _Type, _Pid, {happy_connect, OK}} ->
+      ?report_trace("happy_connect ~p", [OK]),
+      OK;
+    {'DOWN', MRef, _Type, _Pid, Info} ->
+      {connect_error, Info}
+  after Timeout ->
+          connect_gc(Pid, MRef),
+          {error, connect_timeout}
+  end.
+
+connect_gc(Pid, MRef) ->
+  catch exit(Pid, normal),
+  erlang:demonitor(MRef, [flush]).
+
+
+-spec parse_address(inet:ip_address() | binary() | string()) -> inet:ip_address() | string().
+parse_address(IPTuple) when is_tuple(IPTuple) -> IPTuple;
+parse_address(IPBin) when is_binary(IPBin) ->
+  parse_address(binary_to_list(IPBin));
+%% IPv6 string with brackets
+parse_address("[" ++ IPString) ->
+  parse_address(lists:sublist(IPString, length(IPString) - 1));
+parse_address(IPString) ->
+  case inet:parse_address(IPString) of
+    {ok, IP} -> IP;
+    {error, _} -> IPString
+  end.
+
+-spec getaddrs(string()) -> {[{inet:ip_address(), 'inet6' | 'inet'}], [{inet:ip_address(), 'inet6' | 'inet'}]}.
+getaddrs("localhost") ->
+  {[{{0,0,0,0,0,0,0,1}, 'inet6'}], [{{127,0,0,1}, 'inet'}]};
+getaddrs(Name) ->
+  IP6Addrs = [{Addr, 'inet6'} || Addr <- getbyname(Name, 'aaaa')],
+  IP4Addrs = [{Addr, 'inet'} || Addr <- getbyname(Name, 'a')],
+  {IP6Addrs, IP4Addrs}.
+
+getbyname(Hostname, Type) ->
+  case (catch inet_res:getbyname(Hostname, Type)) of
+    {'ok', #hostent{h_addr_list=AddrList}} -> lists:usort(AddrList);
+    {error, _Reason} -> [];
+    Else ->
+      %% ERLANG 22 has an issue when g matching somee DNS server messages
+      ?report_debug("DNS error", [{hostname, Hostname}
+                                 ,{type, Type}
+                                 ,{error, Else}]),
+      []
+  end.
+
+try_connect(Ipv6Addrs, Port, Opts, Self) ->
+  try_connect(Ipv6Addrs, Port, Opts, Self, {error, nxdomain}).
+
+try_connect([], _Port, _Opts, _ServerPid, LastError) ->
+  ?report_trace("happy eyeball: failed to connect", [{error, LastError}]),
+  exit(LastError);
+try_connect([{IP, Type} | Rest], Port, Opts, ServerPid, _LastError) ->
+  ?report_trace("try to connect", [{ip, IP}, {type, Type}]),
+  case gen_tcp:connect(IP, Port, [Type | Opts], ?TIMEOUT) of
+    {ok, Socket} = OK ->
+      ?report_trace("success to connect", [{ip, IP}, {type, Type}]),
+      ok = gen_tcp:controlling_process(Socket, ServerPid),
+      exit({happy_connect, OK});
+    Error ->
+      try_connect(Rest, Port, Opts, ServerPid, Error)
+  end.

src/hackney_http_connect.erl

@@ -63,7 +63,7 @@ connect(ProxyHost, ProxyPort, Opts, Timeout)
   ConnectOpts = hackney_util:filter_options(Opts, AcceptedOpts, BaseOpts),
 
   %% connect to the proxy, and upgrade the socket if needed.
-  case gen_tcp:connect(ProxyHost, ProxyPort, ConnectOpts) of
+  case hackney_happy:connect(ProxyHost, ProxyPort, ConnectOpts) of
     {ok, Socket} ->
       case do_handshake(Socket, Host, Port, Opts) of
         ok ->