hackney 2

README.md

@@ -41,16 +41,16 @@ information on what still needs to be done.
 
 #### Useful modules are:
 
-- [`hackney`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney.md): main module. It contains all HTTP client functions.
-- [`hackney_http`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_http.md): HTTP parser in pure Erlang. This parser is able
+- [`hackney`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney.md): main module. It contains all HTTP client functions.
+- [`hackney_http`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_http.md): HTTP parser in pure Erlang. This parser is able
 to parse HTTP responses and requests in a streaming fashion. If not set
 it will be autodetected if it's a request or a response that's needed.
 
-- [`hackney_headers`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_headers.md) Module to manipulate HTTP headers.
-- [`hackney_cookie`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_cookie.md): Module to manipulate cookies.
-- [`hackney_multipart`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_multipart.md): Module to encode/decode multipart.
-- [`hackney_url`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_url.md): Module to parse and create URIs.
-- [`hackney_date`](http://github.com/benoitc/hackney/blob/refactor/doc/hackney_date.md): Module to parse HTTP dates.
+- [`hackney_headers`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_headers.md) Module to manipulate HTTP headers.
+- [`hackney_cookie`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_cookie.md): Module to manipulate cookies.
+- [`hackney_multipart`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_multipart.md): Module to encode/decode multipart.
+- [`hackney_url`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_url.md): Module to parse and create URIs.
+- [`hackney_date`](http://github.com/benoitc/hackney/blob/new_pool/doc/hackney_date.md): Module to parse HTTP dates.
 
 Read the [NEWS](https://raw.github.com/benoitc/hackney/master/NEWS.md) file
 to get the last changelog.
@@ -174,7 +174,7 @@ couple of requests.
 
 ```erlang
 
-Transport = hackney_tcp_transport,
+Transport = hackney_tcp,
 Host = << "https://friendpaste.com" >>,
 Port = 443,
 Options = [],

doc/README.md

@@ -174,7 +174,7 @@ couple of requests.
 
 ```erlang
 
-Transport = hackney_tcp_transport,
+Transport = hackney_tcp,
 Host = << "https://friendpaste.com" >>,
 Port = 443,
 Options = [],

doc/edoc-info

@@ -3,8 +3,7 @@
 {modules,[hackney,hackney_app,hackney_bstr,hackney_connect,hackney_cookie,
           hackney_date,hackney_dummy_metrics,hackney_exometer_metrics,
           hackney_folsom_metrics,hackney_headers,hackney_http,
-          hackney_http_connect,hackney_manager,hackney_mimetypes,
-          hackney_multipart,hackney_pool,hackney_pool_handler,hackney_request,
-          hackney_response,hackney_socks5,hackney_ssl_transport,
-          hackney_stream,hackney_sup,hackney_tcp_transport,hackney_trace,
-          hackney_url,hackney_util]}.
+          hackney_http_connect,hackney_manager,hackney_multipart,hackney_pool,
+          hackney_pool_handler,hackney_request,hackney_response,
+          hackney_socks5,hackney_ssl,hackney_stream,hackney_sup,hackney_tcp,
+          hackney_trace,hackney_url,hackney_util]}.

doc/hackney_ssl_transport.md → doc/hackney_ssl.md

@@ -1,6 +1,6 @@
 
 
-# Module hackney_ssl_transport #
+# Module hackney_ssl #
 * [Function Index](#index)
 * [Function Details](#functions)
 

doc/hackney_tcp_transport.md → doc/hackney_tcp.md

@@ -1,6 +1,6 @@
 
 
-# Module hackney_tcp_transport #
+# Module hackney_tcp #
 * [Function Index](#index)
 * [Function Details](#functions)
 

doc/overview.edoc

@@ -182,7 +182,7 @@ couple of requests.
 #### To create a connection:
 
 <pre lang="erlang">
-Transport = hackney_tcp_transport,
+Transport = hackney_tcp,
 Host = &lt;&lt; "https://friendpaste.com" >>,
 Port = 443,
 Options = [],

include/hackney.hrl

@@ -43,3 +43,19 @@
         method = nil,
         path,
         ctype = nil}).
+
+
+
+-define(DEFAULT_CACHE_SIZE, 1000).
+-define(TAB, hackney_server).
+-define(LOOKUP_CACHE, hackney_lookup).
+
+%% default pool info
+-define(DEFAULT_IDLE_TIMEOUT, 150000). %% default time until a connectino is forced to closed
+-define(DEFAULT_GROUP_LIMIT, 6). %% max number of connections kept for a group
+-define(DEFAULT_PROXY_LIMIT, 20). %% max number of connections cached / proxy
+-define(DEFAULT_MAX_CONNS, 200). %% maximum number of connections kept
+
+%% connectors options
+-define(DEFAULT_NB_CONNECTORS, 20).
+-define(DEFAULT_FALLBACK_TIME, 250).

rebar.config

@@ -18,9 +18,11 @@
 
 {deps, [
         {idna, "1.0.2"},
-        {mimerl, "1.0.0"},
-        {certifi, "0.1.1"},
-        {ssl_verify_hostname, "1.0.5"}
+        {mimerl, "1.0.1"},
+        {certifi, "0.3.0"},
+        {ssl_verify_hostname, "1.0.5"},
+        {lru, "1.3.1"},
+        {metrics, "0.1.0"}
  ]}.
 
 %% Not yet supported in rebar3
@@ -34,9 +36,6 @@
              ]},
 
               {edoc_opts, [{doclet, edown_doclet},
-                           {source_path, ["src/socket",
-                                          "src/http",
-                                          "src/metrics"]},
                            {packages, false},
                            {subpackages, true},
                            {top_level_readme,

rebar.config.script

@@ -1,8 +1,11 @@
-{ok, VSN} = application:get_key(rebar, vsn),
-[VSN1 | _] = string:tokens(VSN, "-"),
-[Maj, Min, Patch] = string:tokens(VSN1, "."),
-IsRebar3 = (list_to_integer(Maj) >= 3),
-
+IsRebar3 = case application:get_key(rebar, vsn) of
+               {ok, VSN} ->
+                   [VSN1 | _] = string:tokens(VSN, "-"),
+                   [Maj, Min, Patch] = string:tokens(VSN1, "."),
+                   (list_to_integer(Maj) >= 3);
+               undefined ->
+                   false
+           end,
 
 Rebar2Deps = [
               {idna, ".*",
@@ -15,11 +18,20 @@ Rebar2Deps = [
 
               {certifi, ".*",
                {git, "https://github.com/certifi/erlang-certifi",
-                {tag, "0.1.0"}}},
+                {tag, "0.2.0"}}},
 
               {ssl_verify_hostname, ".*",
                {git, "https://github.com/deadtrickster/ssl_verify_hostname.erl",
-                {tag, "1.0.5"}}}
+                {tag, "1.0.5"}}},
+
+              {lru, ".*",
+               {git, "https://github.com/barrel-db/erlang-lru.git",
+                {tag, "1.3.1"}}},
+
+              {metrics, ".*",
+               {git, "https://github.com/benoitc/metrics.git",
+                {tag, "0.1.0"}}}
+
              ],
 
 case IsRebar3 of

rebar.lock

@@ -1,4 +1,5 @@
-[{<<"certifi">>,{pkg,<<"certifi">>,<<"0.1.1">>},0},
+[{<<"certifi">>,{pkg,<<"certifi">>,<<"0.2.0">>},0},
  {<<"idna">>,{pkg,<<"idna">>,<<"1.0.2">>},0},
+ {<<"lru">>,{pkg,<<"lru">>,<<"1.2.0">>},0},
  {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.0.0">>},0},
  {<<"ssl_verify_hostname">>,{pkg,<<"ssl_verify_hostname">>,<<"1.0.5">>},0}].

src/hackney.app.src

@@ -4,8 +4,8 @@
 {application, hackney,
  [
  {description, "simple HTTP client"},
- {vsn, "1.3.2"},
- {registered, [hackney_pool]},
+ {vsn, "2.0.0"},
+ {registered, [hackney_sup, hackney_server]},
  {applications, [kernel,
                  stdlib,
                  crypto,

src/hackney.erl

@@ -5,7 +5,9 @@
 %%%
 
 -module(hackney).
--export([start/0, start/1, stop/0]).
+-export([start/0, stop/0]).
+-export([start_pool/2, stop_pool/1, child_spec/2]).
+
 -export([connect/1, connect/2, connect/3, connect/4,
          close/1,
          request_info/1,
@@ -55,15 +57,30 @@ start() ->
     hackney_app:ensure_deps_started(),
     application:start(hackney).
 
-start(PoolHandler) ->
-    application:set_env(hackney, pool_handler, PoolHandler),
-    start().
-
 %% @doc Stop the hackney process. Useful when testing using the shell.
 stop() ->
     application:stop(hackney).
 
 
+start_pool(Name, Opts) ->
+    supervisor:start_child(hackney_sup, child_spec(Name, Opts)).
+
+
+stop_pool(Name) ->
+    case supervisor:terminate_child(hackney_sup, {hackney_pool_sup, Name}) of
+        ok ->
+            %% make sure we delete the child on old erlang version
+            _ = supervisor:delete_child(hackney_sup, {hackney_pool_sup, Name}),
+            ok;
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+child_spec(Name, Opts) ->
+    {{hackney_pool_sup, Name}, {hackney_pool_sup, start_link, [Name, Opts]},
+      permanent, infinity, supervisor, [hackney_pool_sup]}.
+
+
 connect(URL) ->
     connect(URL, []).
 
@@ -303,7 +320,7 @@ request(Method, #hackney_url{}=URL0, Headers, Body, Options0) ->
                               {body, Body},
                               {options, Options0}]),
 
-    #hackney_url{transport=Transport,
+    #hackney_url{scheme=Scheme,
                  host = Host,
                  port = Port,
                  user = User,
@@ -317,7 +334,7 @@ request(Method, #hackney_url{}=URL0, Headers, Body, Options0) ->
                            {basic_auth, {User, Password}})
     end,
 
-    case maybe_proxy(Transport, Host, Port, Options) of
+    case maybe_proxy(Scheme, Host, Port, Options) of
         {ok, Ref, AbsolutePath} ->
             Request = make_request(Method, URL, Headers, Body,
                                    Options, AbsolutePath),
@@ -609,9 +626,9 @@ maybe_proxy(Transport, Host, Port, Options)
                          port = ProxyPort} = hackney_url:normalize(Url1),
             ProxyAuth = proplists:get_value(proxy_auth, Options),
             case {Transport, PTransport} of
-                {hackney_ssl_transport, hackney_ssl_transport} ->
+                {hackney_ssl, hackney_ssl} ->
                     {error, invalid_proxy_transport};
-                {hackney_ssl_transport, _} ->
+                {hackney_ssl, _} ->
                     do_connect(ProxyHost, ProxyPort, ProxyAuth,
                                Transport, Host, Port, Options);
                 _ ->
@@ -625,7 +642,7 @@ maybe_proxy(Transport, Host, Port, Options)
             ?report_debug("HTTP proxy request", [{proxy_host, ProxyHost},
                                                  {proxy_port, ProxyPort}]),
             case Transport of
-                hackney_ssl_transport ->
+                hackney_ssl ->
                     ProxyAuth = proplists:get_value(proxy_auth, Options),
                     do_connect(ProxyHost, ProxyPort, ProxyAuth, Transport, Host,
                         Port, Options);

src/hackney_app.erl

@@ -51,3 +51,4 @@ get_app_env(Key, Default) ->
         {ok, Val} -> Val;
         undefined -> Default
     end.
+

src/hackney_connector.erl

@@ -0,0 +1,110 @@
+-module(hackney_connector).
+
+-export([start_link/2,
+         init/2]).
+
+-include("hackney.hrl").
+-include("hackney_socket.hrl").
+
+-record(state, {pool,
+                fallback_time}).
+
+-define(DEFAULT_LOOKUP_ORDER, [inet6, inet]).
+
+start_link(Pool, FallbackTime) ->
+    Pid = spawn_link(?MODULE, init, [Pool, FallbackTime]),
+    {ok, Pid}.
+
+
+init(Pool, FallbackTime) ->
+    hackney_pool:register_connector(Pool, self()),
+    loop(#state{pool=Pool, fallback_time=FallbackTime}).
+
+
+loop(State) ->
+    receive
+        {connect, Group, {Host, Port, Options, Timeout}} ->
+            Ref = make_ref(),
+            Req = {Group, Host, Port, Options, Timeout},
+            LookupOrder = lru:get(?LOOKUP_CACHE, Host, ?DEFAULT_LOOKUP_ORDER),
+            handle_connect(Ref, Req, LookupOrder, State),
+            loop(State);
+        stop ->
+            exit(normal)
+    end.
+
+
+handle_connect(Ref, Req, LookupOrder, State) ->
+    [F1, F2] = LookupOrder,
+    Pid = spawn_connection(Ref, Req, F1, State#state.pool),
+    TRef = erlang:send_after(State#state.fallback_time, self(), {Ref, fallback}),
+    connect_loop(Ref, TRef, Req, Pid, nil, F2, LookupOrder, State, 2).
+
+connect_loop(_, _, _, _, _, _, _, State, 0) ->
+    loop(State);
+connect_loop(Ref, TRef, {_Group, H, _, _, _} = Req, P1, P2, F2, LookupOrder, State, Wait) ->
+    receive
+        {Ref, connected, P1} ->
+            maybe_kill_job(Ref, TRef, P2),
+            loop(State);
+        {Ref, connected, P2} ->
+            catch exit(P1, normal),
+            flush(Ref, P2),
+            %% lookup order is reversed, store it.
+            lru:add(?LOOKUP_CACHE, H, lists:reverse(LookupOrder)),
+            loop(State);
+        {Ref, fallback} ->
+            Pid = spawn_fallback(P2, Ref, Req, F2, State),
+            connect_loop(Ref, TRef, Req, P1, Pid, F2, LookupOrder, State, Wait);
+        {Ref, 'DOWN', P1, _Error} ->
+            Pid = spawn_fallback(P2, Ref, Req, F2, State),
+            connect_loop(Ref, TRef, Req, P1, Pid, F2, LookupOrder, State, Wait -1);
+        {Ref, 'DOWN', P2, Error} ->
+            error_logger:error_msg(
+                    "hackney connector: connection failure; "
+                    "with reason: ~p~n", [Error]),
+            connect_loop(Ref, TRef, Req, P1, P2, F2, LookupOrder, State, Wait -1)
+    end.
+
+
+maybe_kill_job(Ref, TRef, Pid) ->
+    case is_pid(Pid) of
+        true -> catch exit(Pid, normal);
+        false -> erlang:cancel_timer(TRef)
+    end,
+    flush(Ref, Pid).
+
+
+flush(Ref, Pid) ->
+    receive
+        {Ref, connected, Pid} -> ok;
+        {Ref, fallback, _Familly} -> ok;
+        _Else -> 
+            ok
+    after 0 -> ok
+    end.
+
+spawn_fallback(nil, Ref, Req, Familly, State) ->
+    spawn_connection(Ref, Req, Familly, State#state.pool);
+spawn_fallback(Pid, _, _, _, _) ->
+    Pid.
+
+spawn_connection(Ref, {Group, Host, Port, Opts0, Timeout}, Familly, Pool) ->
+    Opts = [Familly | Opts0],
+    Connector = self(),
+    spawn_link(fun() ->
+                       case hackney_tcp:connect(Host, Port, Opts, Timeout) of
+                           {ok, Sock} ->
+                               HS = #hackney_socket{transport=hackney_tcp,
+                                                    sock=Sock,
+                                                    host=Host,
+                                                    port=Port,
+                                                    group=Group,
+                                                    pool=Pool},
+
+                               Connector ! {Ref, connected, self()},
+                               hackney_pool:release(HS);
+                           Error ->
+                               Connector ! {Ref, 'DOWN', self(), Error}
+                       end
+               end).