Skip to content

Instantly share code, notes, and snippets.

@dch

dch/gist:8c58268f700309ecc90d0c9a6cab8f26

Created September 19, 2018 08:31
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dch/8c58268f700309ecc90d0c9a6cab8f26 to your computer and use it in GitHub Desktop.
Save dch/8c58268f700309ecc90d0c9a6cab8f26 to your computer and use it in GitHub Desktop.
Download ZIP
git diff --patience CouchDB-4.0.1...v4.4.1
Raw
gistfile1.txt
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a48c26e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,13 @@
+_build
+ebin
+*.beam
+*.sw*
+deps
+.DS_Store
+erl_crash.dump
+.eunit
+mime.types
+.rebar
+*.plt
+.rebar
+*~
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..3993bf2
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,14 @@
+language: erlang
+otp_release:
+ - R16B
+ - R16B03-1
+ - 17.0
+ - 17.1
+ - 18.0
+ - 18.1
+ - 18.2.1
+ - 19.3
+ - 20.0
+before_script:
+ - "./bootstrap_travis.sh"
+script: "./rebar3 eunit"
diff --git a/BSD_LICENSE b/BSD_LICENSE
new file mode 100644
index 0000000..318a96e
--- /dev/null
+++ b/BSD_LICENSE
@@ -0,0 +1,10 @@
+Copyright (c) 2005-2014, Chandrashekhar Mullaparthi
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+ * Neither the name of the T-Mobile nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/CHANGELOG b/CHANGELOG
new file mode 100644
index 0000000..7240b17
--- /dev/null
+++ b/CHANGELOG
@@ -0,0 +1,365 @@
+CONTRIBUTIONS & CHANGE HISTORY
+==============================
+
+23-08-2018 - v4.4.1
+ * Fixes to TLS socket handling (PR#163)
+ * Fix ipv6 address family handling (PR#155)
+ * Don't send messages to closed/inactive connections (PR#152)
+
+28-01-2017 - v4.4
+ * Fixes to SOCKS over SSL processing
+ * Added stream_full_chunks option
+ * Merged pull requests 145, 151
+
+07-06-2016 - v4.3
+ * Adopted erlang.mk for compiling. I find it easier to understand
+ how 'make' behaves compared to rebar. This repo can still be built
+ using rebar for those who prefer it
+ * Removed references to lager. Introduced configurable logging function
+ * Fixed an issue where the calling process was getting an extra
+ spurious timeout message when the request was timing out
+
+19-04-2016 - v4.2.4
+ * Fixed travis-ci build as it was failing in running tests.
+ No code changes to ibrowse
+
+19-04-2016 - v4.2.3
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/143
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/142
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/139
+ * Fixed behaviour of option preserve_status_line
+
+25-11-2015 - v4.2.2
+ * Fix to ibrowse.app.src to enable publishing using Hex
+
+25-11-2015 - v4.2.1
+ * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/132
+ * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/137
+
+28-09-2015 - v4.2
+ * Merged long pending improvements to pipelining
+ https://github.com/cmullaparthi/ibrowse/pull/123
+ * Merged pull request https://github.com/cmullaparthi/ibrowse/pull/131
+
+03-08-2015 - v4.1.2
+ * R18 compatibility fix
+ https://github.com/cmullaparthi/ibrowse/issues/129
+ * Add max_attempts option
+ https://github.com/cmullaparthi/ibrowse/pull/125
+ * Fix for https://github.com/cmullaparthi/ibrowse/pull/120
+ * Enhanced SOCKS5 support
+ https://github.com/cmullaparthi/ibrowse/pull/117
+
+10-07-2014 - v4.1.1
+ * Added support for accepting binaries as header names
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/110
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/111
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/112
+
+18-04-2013 - v4.1.0
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/101
+ * Support for https://github.com/cmullaparthi/ibrowse/issues/90
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/86
+ * Merged various contributions. Please see commit history for details
+ * Introduced the return_raw_request option
+
+09-04-2013 - v4.0.2
+ * Tagging master with new version to cover changes
+ contributed over the past few months via various pull requests
+
+07-08-2012 - v4.0.1
+ * Fix issue 67 properly.
+
+03-08-2012 - v4.0.0
+ * Fixed a regression in handling HEAD.
+ https://github.com/cmullaparthi/ibrowse/issues/67
+
+ * Fixed a bug in handling SSL requests through a proxy
+
+06-04-2012 - v3.0.4
+ * Fix for the following issue
+ https://github.com/cmullaparthi/ibrowse/issues/67
+
+13-03-2012 - v3.0.3
+ * Fixes the following issues
+ https://github.com/cmullaparthi/ibrowse/issues/64
+ https://github.com/cmullaparthi/ibrowse/issues/63
+ https://github.com/cmullaparthi/ibrowse/issues/62
+
+31-01-2012 - v3.0.2
+ * Fixed bug when stopping ibrowse. Not service affecting.
+
+23-01-2012 - v3.0.1
+ * Fixed bug highlighted by Dialyzer
+
+23-01-2012 - v3.0.0
+ * Change to the way pipelining works.
+ * Fixed various issues reported
+
+13-04-2011 - v2.2.0
+ * Filipe David Manana added IPv6 support. This is a mjor new
+ feature, Thank you Filipe!
+ * Joseph Wayne Norton contributed tweaks to .gitignore
+
+09-02-2011 - v2.1.4
+ * Fixed a bug reported by Ryan Zezeski with the
+ save_response_to_file option.
+ https://github.com/cmullaparthi/ibrowse/issues#issue/33
+
+16-01-2011 - v2.1.3
+ * Fixed issues with streaming and chunked responses when using
+ the 'caller controls socket' feature. See following links for
+ details. Contributed by Filipe David Manana.
+ https://github.com/cmullaparthi/ibrowse/pull/24
+ https://github.com/cmullaparthi/ibrowse/pull/25
+ https://github.com/cmullaparthi/ibrowse/pull/27
+ https://github.com/cmullaparthi/ibrowse/pull/28
+ https://github.com/cmullaparthi/ibrowse/pull/29
+
+ * Fix for issue 32 reported by fholzhauser
+ https://github.com/cmullaparthi/ibrowse/issues#issue/32
+
+ * Fixed some dialyzer warnings. Thanks to Kostis for reporting
+ them.
+
+20-12-2010 - v2.1.2
+ * Pipelining wasn't working when used in conjunction with the
+ {stream_to, {self(), once}} option. Bug report by
+ Filipe David Manana.
+
+10-12-2010 - v2.1.1
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/20
+ by Filipe David Manana
+
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/21
+ by Filipe David Manana
+
+ * Fix for https://github.com/cmullaparthi/ibrowse/issues/issue/23
+ by Filipe David Manana
+
+ * Fix for bugs when using SSL by Jo?o Lopes
+
+25-10-2010 - v2.1.0
+ * Fixed build on OpenSolaris. Bug report and patch from
+ tholschuh.
+ http://github.com/cmullaparthi/ibrowse/issues/issue/10
+
+ * Fixed behaviour of inactivity_timeout option. Reported by
+ Jo?o Lopes.
+ http://github.com/cmullaparthi/ibrowse/issues/issue/11
+
+ * Prevent atom table pollution when bogus URLs are input to
+ ibrowse. Bug report by Jo?o Lopes.
+ http://github.com/cmullaparthi/ibrowse/issues/issue/13
+
+ * Automatically do Chunked-Transfer encoding of request body
+ when the body is generated by a fun. Patch provided by
+ Filipe David Manana.
+ http://github.com/cmullaparthi/ibrowse/issues/issue/14
+
+ * Depending on input options, ibrowse sometimes included multiple
+ Content-Length headers. Bug reported by Paul J. Davis
+ http://github.com/cmullaparthi/ibrowse/issues/issue/15
+
+ * Deal with webservers which do not provide a Reason-Phrase on the
+ response Status-Line. Patch provided by Jeroen Koops.
+ http://github.com/cmullaparthi/ibrowse/issues/issue/16
+
+ * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/17
+ This was reported by Filipe David Manana.
+
+ * Fixed http://github.com/cmullaparthi/ibrowse/issues/issue/19
+ This was reported by Dan Kelley and Filipe David Manana.
+
+ * Added ibrowse:stream_close/1 to close the connection
+ associated with a certain response stream. Patch provided by
+ Jo?o Lopes.
+
+ * Prevent port number being included in the Host header when port
+ 443 is intended. Bug reported by Andrew Tunnell-Jones
+
+24-09-2010 - v2.0.1
+ * Removed a spurious io:format statement
+
+22-09-2010 - v2.0.0.
+
+ * Added option preserve_chunked_encoding. This allows the
+ caller to get the raw HTTP response when the
+ Transfer-Encoding is Chunked. This feature was requested
+ by Benoit Chesneau who wanted to write a HTTP proxy using
+ ibrowse.
+
+ * Fixed bug with the {stream_to, {Pid, once}} option. Bug
+ report and lot of help from Filipe David Manana. Thank
+ you Filipe.
+
+ * The {error, conn_failed} and {error, send_failed} return
+ values are now of the form {error, {conn_failed, Err}}
+ and {error, {send_failed, Err}}. This is so that the
+ specific socket error can be returned to the caller. I
+ think it looks a bit ugly, but that is the best
+ compromise I could come up with.
+
+ * Added application configuration parameters
+ default_max_sessions and default_max_pipeline_size. These
+ were previously hard coded to 10.
+
+ * Versioning of ibrowse now follows the Semantic Versioning
+ principles. See http://semver.org. Thanks to Anthony
+ Molinaro for nudging me in this direction.
+
+ * The connect_timeout option now only applies to the
+ connection setup phase. In previous versions, the time
+ taken to setup the connection was deducted from the
+ specified timeout value for the request.
+
+17-07-2010 - * Merged change made by Filipe David Manana to use the base64
+ module for encoding/decoding.
+
+11-06-2010 - * Removed use of deprecated concat_binary. Patch supplied by
+ Steve Vinoski
+
+10-06-2010 - * Fixed bug in https requests not going via the proxy
+
+12-05-2010 - * Added support for the CONNECT method to tunnel HTTPS through
+ a proxy. When a https URL is requested through a proxy,
+ ibrowse will automatically use the CONNECT method to first
+ setup a tunnel through the proxy. Once this succeeds, the
+ actual request is dispatched. Successfully tested with the
+ new SSL implementation in R13B-03
+ * Added SSL support for direct connections.
+ See ibrowse:spawn_worker_process/1 and
+ ibrowse:spawn_link_worker_process/1
+ * Added option to return raw status line and raw unparsed headers
+
+23-04-2010 - * Fixes to URL parsing by Karol Skocik
+
+08-11-2009 - * Added option headers_as_is
+
+04-10-2009 - * Patch from Kostis Sagonas to cleanup some code and suppress
+ dialyzer warnings
+
+24-09-2009 - * When a filename was supplied with the 'save_response_to_file'
+ option, the option was being ignored. Bug report from
+ Adam Kocoloski
+
+05-09-2009 - * Introduced option to allow caller to set socket options.
+
+29-07-2009 - * The ETS table created for load balancing of requests was not
+ being deleted which led to the node not being able to create
+ any more ETS tables if queries were made to many number of
+ webservers. ibrowse now deletes the ETS table it creates once the
+ last connection to a webserver is dropped.
+ Reported by Seth Falcon.
+ * Spurious data being returned at end of body in certain cases of
+ chunked encoded responses from the server.
+ Reported by Chris Newcombe.
+
+03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller
+ to control when it wants to receive more data. If this option
+ is used, the call ibrowse:stream_next(Req_id) should be used
+ to get more data.
+ * Patch submitted by Steve Vinoski to remove compiler warnings
+ about the use of obsolete guards
+
+29-06-2009 - * Fixed following issues reported by Oscar Hellstr?m
+ * Use {active, once} instead of {active, true}
+ * Fix 'dodgy' timeout handling
+ * Use binaries internally instead of lists to reduce memory
+ consumption on 64 bit platforms. The default response format
+ is still 'list' to maintain backwards compatibility. Use the
+ option {response_format, binary} to get responses as binaries.
+ * Fixed chunking bug (reported by Adam Kocoloski)
+ * Added new option {inactivity_timeout, Milliseconds} to timeout
+ requests if no data is received on the link for the specified
+ interval. Useful when responses are large and links are flaky.
+ * Added ibrowse:all_trace_off/0 to turn off all tracing
+ * Change to the way responses to asynchronous requests are
+ returned. The following messages have been removed.
+ * {ibrowse_async_response, Req_id, {chunk_start, Chunk_size}}
+ * {ibrowse_async_response, Req_id, chunk_end}
+ * Fixed Makefiles as part of Debian packaging
+ (thanks to Thomas Lindgren)
+ * Moved repository from Sourceforge to Github
+
+11-06-2009 - * Added option to control size of streamed chunks. Also added
+ option for the client to receive responses in binary format.
+
+21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file.
+ Reported by Erik Reitsma on the erlyaws mailing list
+ * Fixed bug when cleaning up closing connections
+
+27-03-2008 - * Major rewrite of the load balancing feature. Additional module,
+ ibrowse_lb.erl, introduced to achieve this.
+ * Can now get a handle to a connection process which is not part of
+ the load balancing pool. Useful when an application is making
+ requests to a webserver which are time consuming (such as
+ uploading a large file). Such requests can be put on a separate
+ connection, and all other smaller/quicker requests can use the
+ load balancing pool. See ibrowse:spawn_worker_process/2 and
+ ibrowse:spawn_link_worker_process/2
+ * Ram Krishnan sent a patch to enable a client to send a lot of
+ data in a request by providing a fun which is invoked by the
+ connection handling process. This fun can fetch the data from
+ any where. This is useful when trying to upload a large file
+ to a webserver.
+ * Use the TCP_NODELAY option on every socket by default
+ * Rudimentary support for load testing of ibrowse. Undocumented,
+ but see ibrowse_test:load_test/3. Use the source, Luke!
+ * New function ibrowse:show_dest_status/2 to view state of
+ connections/pipelines to a web server
+
+20-02-2008 - Ram Krishnan sent another patch for another hidden bug in the
+ save_response_to_file feature.
+
+07-02-2008 - Ram Krishnan (kriyative _at_ gmail dot com) sent a simple patch to
+ enable specifying the filename in the save_response_to_file option.
+ When testing the patch, I realised that my original implementation
+ of this feature was quite flaky and a lot of corner cases were
+ not covered. Fixed all of them. Thanks Ram!
+
+17-10-2007 - Matthew Reilly (matthew dot reilly _at_ sipphone dot com)
+ sent a bug report and a fix. If the chunk trailer spans two TCP
+ packets, then ibrowse fails to recognise that the chunked transfer
+ has ended.
+
+29-08-2007 - Bug report by Peter Kristensen(ptx _at_ daimi dot au dot dk).
+ ibrowse crashes when the webserver returns just the Status line
+ and nothing else.
+
+28-06-2007 - Added host_header option to enable connection to secure sites
+ via stunnel
+
+20-04-2007 - Geoff Cant sent a patch to remove URL encoding for digits in
+ ibrowse_lib:url_encode/1.
+ ibrowse had a dependency on the inets application because the
+ ibrowse_http_client.erl invoked httpd_util:encode_base64/1. This
+ dependency is now removed and the encode_base64/1 has been
+ implemented in ibrowse_lib.erl
+
+06-03-2007 - Eric Merritt sent a patch to support WebDAV requests.
+
+12-01-2007 - Derek Upham sent in a bug fix. The reset_state function was not
+ behaving correctly when the transfer encoding was not chunked.
+
+13-11-2006 - Youn?s Hafri reported a bug where ibrowse was not returning the
+ temporary filename when the server was closing the connection
+ after sending the data (as in HTTP/1.0).
+ Released ibrowse under the BSD license
+
+12-10-2006 - Chris Newcombe reported bug in dealing with requests where no
+ body is expected in the response. The first request would succeed
+ and the next request would hang.
+
+24-May-2006 - Sean Hinde reported a bug. Async responses with pipelining was
+ returning the wrong result.
+
+08-Dec-2005 - Richard Cameron (camster@citeulike.org). Patch to ibrowse to
+ prevent port number being included in the Host header when port
+ 80 is intended.
+
+22-Nov-2005 - Added ability to generate requests using the Chunked
+ Transfer-Encoding.
+
+08-May-2005 - Youn?s Hafri made a CRUX LINUX port of ibrowse.
+ http://yhafri.club.fr/crux/index.html
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
new file mode 100644
index 0000000..352d55e
--- /dev/null
+++ b/CONTRIBUTORS
@@ -0,0 +1,72 @@
+CONTRIBUTORS
+============
+The following people have helped maked ibrowse better by reporting bugs,
+supplying patches and also asking for new features. Please write to me if you
+have contributed and I've missed you out.
+
+In alphabetical order:
+
+Adam Kocoloski
+Andrew Tunnell-Jones
+Anthony Molinaro
+Benjamin P Lee (https://github.com/benjaminplee)
+Benoit Chesneau (https://github.com/benoitc)
+Brian Richards (http://github.com/richbria)
+Chris Newcombe
+Dan Kelley
+Dan Schwabe (https://github.com/dfschwabe)
+Dave Cottlehuber
+Derek Upham
+Eric Merritt
+Erik Reitsma
+Fabian MuscarielloG
+Filipe David Manana
+Geoff Cant
+Jeroen Koops
+Jo?o Lopes
+Joseph Wayne Norton
+Karol Skocik
+Konstantin Nikiforov
+Kostis Sagonas
+Marcelo Gornstein (https://github.com/marcelog)
+Matthew Reilly
+Michael Terry
+Oscar Hellstr?m
+Paul J. Davis
+Peter Kristensen
+Ram Krishnan
+Richard Cameron
+Robert Newson (https://github.com/rnewson)
+Ryan Flynn
+Ryan Zezeski
+Sean Hinde
+Serge Polkovnikov (https://github.com/serge2)
+Sergey Samokhi
+Seth Falcon
+Steve Vinoski
+Thomas Lindgren
+Vincent Ambo
+Youn?s Hafri
+Yury Gargay (https://github.com/surik)
+fholzhauser (https://github.com/fholzhauser/)
+getong (https://github.com/getong)
+lissana (https://github.com/lissana)
+hyperthunk (https://github.com/hyperthunk/)
+Mistagrooves (https://github.com/Mistagrooves/)
+tholschuh (https://github.com/tholschuh/)
+https://github.com/apauley
+https://github.com/AeroNotix
+https://github.com/dis
+https://github.com/f355
+https://github.com/flycodepl
+https://github.com/helllamer
+https://github.com/marutha
+https://github.com/nrdufour
+https://github.com/pib
+https://github.com/puzza007
+https://github.com/rflynn
+https://github.com/Vagabond
+https://github.com/divolgin
+https://github.com/vans163
+https://github.com/shakugan
+https://github.com/p2k
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..a405991
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,9 @@
+ibrowse - a HTTP client written in erlang
+Copyright (C) 2005-2014 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
+
+This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..0ff4c2d
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,18 @@
+PROJECT=ibrowse
+PLT_APPS=erts kernel stdlib ssl crypto public_key
+TEST_ERLC_OPTS=-pa ../ibrowse/ebin
+
+include erlang.mk
+
+test: app eunit unit_tests old_tests
+ @echo "====================================================="
+
+unit_tests:
+ @echo "====================================================="
+ @echo "Running tests..."
+ @cd test && make test && cd ..
+
+old_tests:
+ @echo "====================================================="
+ @echo "Running old tests..."
+ @cd test && make old_tests && cd ..
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..d2d42e7
--- /dev/null
+++ b/README.md
@@ -0,0 +1,297 @@
+# ibrowse [![Build Status](https://secure.travis-ci.org/cmullaparthi/ibrowse.png)](http://travis-ci.org/cmullaparthi/ibrowse)
+
+ibrowse is a HTTP client written in erlang.
+
+**License:** ibrowse is available under two different licenses.
+ LGPL or the BSD license.
+
+**Comments to:** chandrashekhar.mullaparthi@gmail.com
+
+**Current Version:** 4.4
+
+**Latest Version:** git://github.com/cmullaparthi/ibrowse.git
+
+
+
+## Features
+
+* [RFC2616](http://www.ietf.org/rfc/rfc2616.txt) compliant (AFAIK)
+* supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE,
+ MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, MOVE and COPY
+* Understands HTTP/0.9, HTTP/1.0 and HTTP/1.1
+* Understands chunked encoding
+* Can generate requests using [Chunked Transfer-Encoding](http://en.wikipedia.org/wiki/Chunked_transfer_encoding)
+* Pools of connections to each webserver
+* Pipelining support
+* Download to file
+* Asynchronous requests. Responses are streamed to a process
+* Basic authentication
+* Supports proxy authentication
+* Supports SOCKS5
+ * Authentication methods 0 (No authentication) and 2(Username/password) supported
+* Can talk to secure webservers using SSL
+* *Any other features in the code not listed here :)*
+
+
+
+## Usage Examples
+
+Remember to start ibrowse first:
+
+```erlang
+5> ibrowse:start().
+{ok,<0.94.0>}
+```
+
+
+
+### Synchronous Requests
+
+A simple `GET` request:
+
+```erlang
+6> ibrowse:send_req("http://intranet/messenger/", [], get).
+{ok,"200",
+ [{"Server","Microsoft-IIS/5.0"},
+ {"Content-Location","http://intranet/messenger/index.html"},
+ {"Date","Fri, 17 Dec 2004 15:16:19 GMT"},
+ {"Content-Type","text/html"},
+ {"Accept-Ranges","bytes"},
+ {"Last-Modified","Fri, 17 Dec 2004 08:38:21 GMT"},
+ {"Etag","\"aa7c9dc313e4c41:d77\""},
+ {"Content-Length","953"}],
+ "<html>...</html>"}
+```
+
+
+A `GET` using a proxy:
+
+```erlang
+7> ibrowse:send_req("http://www.google.com/", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080}], 1000).
+{ok,"302",
+ [{"Date","Fri, 17 Dec 2004 15:22:56 GMT"},
+ {"Content-Length","217"},
+ {"Content-Type","text/html"},
+ {"Set-Cookie",
+ "PREF=ID=f58155c797f9..."},
+ {"Server","GWS/2.1"},
+ {"Location",
+ "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D110329..."},
+ {"Via","1.1 netapp01 (NetCache NetApp/5.5R2)"}],
+ "<HTML>...</HTML>\r\n"}
+```
+
+
+A `GET` response saved to file. A temporary file is created and the
+filename returned. The response will only be saved to file if the
+status code is in the `200` range. The directory to download to can
+be set using the application env var `download_dir` - the default
+is the current working directory:
+
+```erlang
+8> ibrowse:send_req("http://www.erlang.se/", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {save_response_to_file, true}], 1000).
+{error,req_timedout}
+
+9> ibrowse:send_req("http://www.erlang.se/", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {save_response_to_file, true}], 5000).
+{ok,"200",
+ [{"Transfer-Encoding","chunked"},
+ {"Date","Fri, 17 Dec 2004 15:24:36 GMT"},
+ {"Content-Type","text/html"},
+ {"Server","Apache/1.3.9 (Unix)"},
+ {"Via","1.1 netapp01 (NetCache NetApp/5.5R2)"}],
+ {file,"/Users/chandru/code/ibrowse/src/ibrowse_tmp_file_1103297041125854"}}
+```
+
+
+Setting the size of the connection pool and pipeline. This sets the
+number of maximum connections to the specified server to `10` and the pipeline
+size to `1`. Connections are assumed to be already setup.
+
+```erlang
+11> ibrowse:set_dest("www.hotmail.com", 80, [{max_sessions, 10},
+ {max_pipeline_size, 1}]).
+ok
+```
+
+
+Example using the `HEAD` method:
+
+```erlang
+56> ibrowse:send_req("http://www.erlang.org", [], head).
+{ok,"200",
+ [{"Date","Mon, 28 Feb 2005 04:40:53 GMT"},
+ {"Server","Apache/1.3.9 (Unix)"},
+ {"Last-Modified","Thu, 10 Feb 2005 09:31:23 GMT"},
+ {"Etag","\"8d71d-1efa-420b29eb\""},
+ {"Accept-ranges","bytes"},
+ {"Content-Length","7930"},
+ {"Content-Type","text/html"}],
+ []}
+```
+
+
+Example using the `OPTIONS` method:
+
+```erlang
+62> ibrowse:send_req("http://www.sun.com", [], options).
+{ok,"200",
+ [{"Server","Sun Java System Web Server 6.1"},
+ {"Date","Mon, 28 Feb 2005 04:44:39 GMT"},
+ {"Content-Length","0"},
+ {"P3p",
+ "policyref=\"http://www.sun.com/p3p/Sun_P3P_Policy.xml\", CP=\"CAO DSP COR CUR ADMa DEVa TAIa PSAa PSDa CONi TELi OUR SAMi PUBi IND PHY ONL PUR COM NAV INT DEM CNT STA POL PRE GOV\""},
+ {"Set-Cookie",
+ "SUN_ID=X.X.X.X:169191109565879; EXPIRES=Wednesday, 31-Dec-2025 23:59:59 GMT; DOMAIN=.sun.com; PATH=/"},
+ {"Allow",
+ "HEAD, GET, PUT, POST, DELETE, TRACE, OPTIONS, MOVE, INDEX, MKDIR, RMDIR"}],
+ []}
+```
+
+
+
+### Asynchronous Requests
+
+Example of an asynchronous `GET` request:
+
+```erlang
+18> ibrowse:send_req("http://www.google.com", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {stream_to, self()}]).
+{ibrowse_req_id,{1115,327256,389608}}
+
+19> flush().
+Shell got {ibrowse_async_headers,{1115,327256,389608},
+ "302",
+ [{"Date","Thu, 05 May 2005 21:06:41 GMT"},
+ {"Content-Length","217"},
+ {"Content-Type","text/html"},
+ {"Set-Cookie",
+ "PREF=ID=b601f16bfa32f071:CR=1:TM=1115327201:LM=1115327201:S=OX5hSB525AMjUUu7; expires=Sun, 17-Jan-2038 19:14:07 GMT; path=/; domain=.google.com"},
+ {"Server","GWS/2.1"},
+ {"Location",
+ "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/"},
+ {"Via",
+ "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]}
+Shell got {ibrowse_async_response,{1115,327256,389608},
+ "<HTML>...</HTML>\r\n"}
+Shell got {ibrowse_async_response_end,{1115,327256,389608}}
+ok
+```
+
+
+Another asynchronous `GET` request:
+
+```erlang
+24> ibrowse:send_req("http://yaws.hyber.org/simple_ex2.yaws", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {stream_to, self()}]).
+{ibrowse_req_id,{1115,327430,512314}}
+
+25> flush().
+Shell got {ibrowse_async_headers,{1115,327430,512314},
+ "200",
+ [{"Date","Thu, 05 May 2005 20:58:08 GMT"},
+ {"Content-Length","64"},
+ {"Content-Type","text/html;charset="},
+ {"Server",
+ "Yaws/1.54 Yet Another Web Server"},
+ {"Via",
+ "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]}
+Shell got {ibrowse_async_response,{1115,327430,512314},
+ "<html>...</html>\n"}
+Shell got {ibrowse_async_response_end,{1115,327430,512314}}
+```
+
+
+Example of request which fails when using the async option. Here
+the `{ibrowse_req_id, ReqId}` is not returned. Instead the error code is
+returned.
+
+```erlang
+68> ibrowse:send_req("http://www.earlyriser.org", [], get, [], [{stream_to, self()}]).
+{error,conn_failed}
+```
+
+
+
+### Other Examples
+
+Example of request using both Proxy-Authorization and authorization
+by the final webserver:
+
+```erlang
+17> ibrowse:send_req("http://www.erlang.se/lic_area/protected/patches/erl_756_otp_beam.README",
+ [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {basic_auth, {"XXXXX", "XXXXXX"}}]).
+{ok,"200",
+ [{"Accept-Ranges","bytes"},
+ {"Date","Thu, 05 May 2005 21:02:09 GMT"},
+ {"Content-Length","2088"},
+ {"Content-Type","text/plain"},
+ {"Server","Apache/1.3.9 (Unix)"},
+ {"Last-Modified","Tue, 03 May 2005 15:08:18 GMT"},
+ {"ETag","\"1384c8-828-427793e2\""},
+ {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}],
+ "Patch Id:\t\terl_756_otp_beam\n..."}
+```
+
+
+Example of a `TRACE` request. Very interesting! yaws.hyber.org didn't
+support this. Nor did www.google.com. But good old BBC supports this:
+
+```erlang
+37> ibrowse:send_req("http://www.bbc.co.uk/", [], trace, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080}]).
+{ok,"200",
+ [{"Transfer-Encoding","chunked"},
+ {"Date","Thu, 05 May 2005 21:40:27 GMT"},
+ {"Content-Type","message/http"},
+ {"Server","Apache/2.0.51 (Unix)"},
+ {"Set-Cookie",
+ "BBC-UID=7452e72a..."},
+ {"Set-Cookie",
+ "BBC-UID=7452e72a..."},
+ {"Via","1.1 hatproxy01 (NetCache NetApp/5.6.2)"}],
+ "TRACE / HTTP/1.1\r\nHost: www.bbc.co.uk\r\nConnection: keep-alive\r\nX-Forwarded-For: 172.24.28.29\r\nVia: 1.1 hatproxy01 (NetCache NetApp/5.6.2)\r\nCookie: BBC-UID=7452e...\r\n\r\n"}
+```
+
+A `GET` using a socks5:
+
+```erlang
+ibrowse:send_req("http://google.com", [], get, [],
+ [{socks5_host, "127.0.0.1"},
+ {socks5_port, 5335}]).
+
+ibrowse:send_req("http://google.com", [], get, [],
+ [{socks5_host, "127.0.0.1"},
+ {socks5_port, 5335},
+ {socks5_user, "user4321"},
+ {socks5_password, "pass7654"}]).
+```
diff --git a/bootstrap_travis.sh b/bootstrap_travis.sh
new file mode 100755
index 0000000..a222df3
--- /dev/null
+++ b/bootstrap_travis.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+curl -O -L https://s3.amazonaws.com/rebar3/rebar3
+chmod +x rebar3
+./rebar3 update
\ No newline at end of file
diff --git a/doc/edoc-info b/doc/edoc-info
new file mode 100644
index 0000000..8d400b5
--- /dev/null
+++ b/doc/edoc-info
@@ -0,0 +1,4 @@
+%% encoding: UTF-8
+{application,ibrowse}.
+{modules,[ibrowse,ibrowse_app,ibrowse_http_client,ibrowse_lb,ibrowse_lib,
+ ibrowse_socks5,ibrowse_sup]}.
diff --git a/doc/erlang.png b/doc/erlang.png
new file mode 100644
index 0000000..987a618
Binary files /dev/null and b/doc/erlang.png differ
diff --git a/doc/ibrowse.html b/doc/ibrowse.html
new file mode 100644
index 0000000..b6738a6
--- /dev/null
+++ b/doc/ibrowse.html
@@ -0,0 +1,387 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse</h1>
+<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client in erlang.
+<p>Copyright © 2005-2014 Chandrashekhar Mullaparthi</p>
+
+<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
+<p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p>
+
+<h2><a name="description">Description</a></h2><p>The ibrowse application implements an HTTP 1.1 client in erlang. This
+module implements the API of the HTTP client. There is one named
+process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
+one process to handle one TCP connection to a webserver
+(implemented in the module ibrowse_http_client). Multiple connections to a
+webserver are setup based on the settings for each webserver. The
+ibrowse process also determines which connection to pipeline a
+certain request on. The functions to call are send_req/3,
+send_req/4, send_req/5, send_req/6.</p>
+
+ <p>Here are a few sample invocations.</p>
+
+ <code>
+ ibrowse:send_req("http://intranet/messenger/", [], get).
+ <br><br>
+
+ ibrowse:send_req("http://www.google.com/", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080}], 1000).
+ <br><br>
+
+ ibrowse:send_req("http://www.erlang.org/download/otp_src_R10B-3.tar.gz", [], get, [],
+ [{proxy_user, "XXXXX"},
+ {proxy_password, "XXXXX"},
+ {proxy_host, "proxy"},
+ {proxy_port, 8080},
+ {save_response_to_file, true}], 1000).
+ <br><br>
+
+ ibrowse:send_req("http://www.erlang.org", [], head).
+
+ <br><br>
+ ibrowse:send_req("http://www.sun.com", [], options).
+
+ <br><br>
+ ibrowse:send_req("http://www.bbc.co.uk", [], trace).
+
+ <br><br>
+ ibrowse:send_req("http://www.google.com", [], get, [],
+ [{stream_to, self()}]).
+ </code>
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#add_config-1">add_config/1</a></td><td>Add additional configuration elements at runtime.</td></tr>
+<tr><td valign="top"><a href="#all_trace_off-0">all_trace_off/0</a></td><td>Turn Off ALL tracing.</td></tr>
+<tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#get_config_value-1">get_config_value/1</a></td><td>Internal export.</td></tr>
+<tr><td valign="top"><a href="#get_config_value-2">get_config_value/2</a></td><td>Internal export.</td></tr>
+<tr><td valign="top"><a href="#get_metrics-0">get_metrics/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#get_metrics-2">get_metrics/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#rescan_config-0">rescan_config/0</a></td><td>Clear current configuration for ibrowse and load from the file
+ ibrowse.conf in the IBROWSE_EBIN/../priv directory.</td></tr>
+<tr><td valign="top"><a href="#rescan_config-1">rescan_config/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#send_req-3">send_req/3</a></td><td>This is the basic function to send a HTTP request.</td></tr>
+<tr><td valign="top"><a href="#send_req-4">send_req/4</a></td><td>Same as send_req/3.</td></tr>
+<tr><td valign="top"><a href="#send_req-5">send_req/5</a></td><td>Same as send_req/4.</td></tr>
+<tr><td valign="top"><a href="#send_req-6">send_req/6</a></td><td>Same as send_req/5.</td></tr>
+<tr><td valign="top"><a href="#send_req_direct-4">send_req_direct/4</a></td><td>Same as send_req/3 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#send_req_direct-5">send_req_direct/5</a></td><td>Same as send_req/4 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#send_req_direct-6">send_req_direct/6</a></td><td>Same as send_req/5 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#send_req_direct-7">send_req_direct/7</a></td><td>Same as send_req/6 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Deprecated.</td></tr>
+<tr><td valign="top"><a href="#set_max_attempts-3">set_max_attempts/3</a></td><td>Set the maximum attempts for each connection to a specific Host:Port.</td></tr>
+<tr><td valign="top"><a href="#set_max_pipeline_size-3">set_max_pipeline_size/3</a></td><td>Set the maximum pipeline size for each connection to a specific Host:Port.</td></tr>
+<tr><td valign="top"><a href="#set_max_sessions-3">set_max_sessions/3</a></td><td>Set the maximum number of connections allowed to a specific Host:Port.</td></tr>
+<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td>Shows some internal information about load balancing.</td></tr>
+<tr><td valign="top"><a href="#show_dest_status-1">show_dest_status/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#show_dest_status-2">show_dest_status/2</a></td><td>Shows some internal information about load balancing to a
+ specified Host:Port.</td></tr>
+<tr><td valign="top"><a href="#spawn_link_worker_process-1">spawn_link_worker_process/1</a></td><td>Same as spawn_worker_process/1 except the the calling process
+ is linked to the worker process which is spawned.</td></tr>
+<tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td>Same as spawn_link_worker_process/1 except with Erlang process options.</td></tr>
+<tr><td valign="top"><a href="#spawn_worker_process-1">spawn_worker_process/1</a></td><td>Creates a HTTP client process to the specified Host:Port which
+ is not part of the load balancing pool.</td></tr>
+<tr><td valign="top"><a href="#spawn_worker_process-2">spawn_worker_process/2</a></td><td>Same as spawn_worker_process/1 except with Erlang process options.</td></tr>
+<tr><td valign="top"><a href="#start-0">start/0</a></td><td>Starts the ibrowse process without linking.</td></tr>
+<tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td>Starts the ibrowse process linked to the calling process.</td></tr>
+<tr><td valign="top"><a href="#stop-0">stop/0</a></td><td>Stop the ibrowse process.</td></tr>
+<tr><td valign="top"><a href="#stop_worker_process-1">stop_worker_process/1</a></td><td>Terminate a worker process spawned using
+ spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr>
+<tr><td valign="top"><a href="#stream_close-1">stream_close/1</a></td><td>Tell ibrowse to close the connection associated with the
+ specified stream.</td></tr>
+<tr><td valign="top"><a href="#stream_next-1">stream_next/1</a></td><td>Tell ibrowse to stream the next chunk of data to the
+ caller.</td></tr>
+<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#trace_off-0">trace_off/0</a></td><td>Turn tracing off for the ibrowse process.</td></tr>
+<tr><td valign="top"><a href="#trace_off-2">trace_off/2</a></td><td>Turn tracing OFF for all connections to the specified HTTP
+ server.</td></tr>
+<tr><td valign="top"><a href="#trace_on-0">trace_on/0</a></td><td>Turn tracing on for the ibrowse process.</td></tr>
+<tr><td valign="top"><a href="#trace_on-2">trace_on/2</a></td><td>Turn tracing on for all connections to the specified HTTP
+ server.</td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="add_config-1">add_config/1</a></h3>
+<div class="spec">
+<p><tt>add_config(Terms) -&gt; any()</tt></p>
+</div><p>Add additional configuration elements at runtime.</p>
+
+<h3 class="function"><a name="all_trace_off-0">all_trace_off/0</a></h3>
+<div class="spec">
+<p><tt>all_trace_off() -&gt; ok</tt><br></p>
+</div><p>Turn Off ALL tracing</p>
+
+<h3 class="function"><a name="code_change-3">code_change/3</a></h3>
+<div class="spec">
+<p><tt>code_change(OldVsn, State, Extra) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="get_config_value-1">get_config_value/1</a></h3>
+<div class="spec">
+<p><tt>get_config_value(Key) -&gt; any()</tt></p>
+</div><p>Internal export</p>
+
+<h3 class="function"><a name="get_config_value-2">get_config_value/2</a></h3>
+<div class="spec">
+<p><tt>get_config_value(Key, DefVal) -&gt; any()</tt></p>
+</div><p>Internal export</p>
+
+<h3 class="function"><a name="get_metrics-0">get_metrics/0</a></h3>
+<div class="spec">
+<p><tt>get_metrics() -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="get_metrics-2">get_metrics/2</a></h3>
+<div class="spec">
+<p><tt>get_metrics(Host, Port) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_call-3">handle_call/3</a></h3>
+<div class="spec">
+<p><tt>handle_call(Request, From, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3>
+<div class="spec">
+<p><tt>handle_cast(Msg, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_info-2">handle_info/2</a></h3>
+<div class="spec">
+<p><tt>handle_info(Info, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="init-1">init/1</a></h3>
+<div class="spec">
+<p><tt>init(X1) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="rescan_config-0">rescan_config/0</a></h3>
+<div class="spec">
+<p><tt>rescan_config() -&gt; any()</tt></p>
+</div><p>Clear current configuration for ibrowse and load from the file
+ ibrowse.conf in the IBROWSE_EBIN/../priv directory. Current
+ configuration is cleared only if the ibrowse.conf file is readable
+ using file:consult/1</p>
+
+<h3 class="function"><a name="rescan_config-1">rescan_config/1</a></h3>
+<div class="spec">
+<p><tt>rescan_config(Terms) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="send_req-3">send_req/3</a></h3>
+<div class="spec">
+<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>) -&gt; <a href="#type-response">response()</a></tt>
+<ul class="definitions"><li><tt><a name="type-headerList">headerList()</a> = [{<a href="#type-header">header()</a>, <a href="#type-value">value()</a>}]</tt></li><li><tt><a name="type-header">header()</a> = atom() | string() | binary()</tt></li><li><tt><a name="type-value">value()</a> = term()</tt></li><li><tt><a name="type-method">method()</a> = get | post | head | options | put | delete | trace | mkcol | propfind | proppatch | lock | unlock | move | copy</tt></li><li><tt>Status = string()</tt></li><li><tt>ResponseHeaders = [<a href="#type-respHeader">respHeader()</a>]</tt></li><li><tt><a name="type-respHeader">respHeader()</a> = {<a href="#type-headerName">headerName()</a>, <a href="#type-headerValue">headerValue()</a>}</tt></li><li><tt><a name="type-headerName">headerName()</a> = string()</tt></li><li><tt><a name="type-headerValue">headerValue()</a> = string()</tt></li><li><tt><a name="type-response">response()</a> = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, <a href="#type-req_id">req_id()</a>} | {error, Reason}</tt></li><li><tt><a name="type-req_id">req_id()</a> = term()</tt></li><li><tt>ResponseBody = string() | {file, Filename}</tt></li><li><tt>Reason = term()</tt></li></ul></p>
+</div><p>This is the basic function to send a HTTP request.
+ The Status return value indicates the HTTP status code returned by the webserver</p>
+
+<h3 class="function"><a name="send_req-4">send_req/4</a></h3>
+<div class="spec">
+<p><tt>send_req(Url, Headers, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>) -&gt; <a href="#type-response">response()</a></tt>
+<ul class="definitions"><li><tt><a name="type-body">body()</a> = [] | string() | binary() | <a href="#type-fun_arity_0">fun_arity_0()</a> | {<a href="#type-fun_arity_1">fun_arity_1()</a>, <a href="#type-initial_state">initial_state()</a>}</tt></li><li><tt><a name="type-initial_state">initial_state()</a> = term()</tt></li></ul></p>
+</div><p>Same as send_req/3.
+ If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1. <br>
+ If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. <pre>Fun() = {ok, Data} | eof</pre><br>
+ If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. <pre>Fun(State) = {ok, Data} | {ok, Data, NewState} | eof</pre></p>
+
+<h3 class="function"><a name="send_req-5">send_req/5</a></h3>
+<div class="spec">
+<p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -&gt; <a href="#type-response">response()</a></tt>
+<ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li><li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_full_chunks, boolean()} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, boolean()} | {is_ssl, boolean()} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, boolean()} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {socket_options, Sock_opts} | {transfer_encoding, {chunked, ChunkSize}} | {headers_as_is, boolean()} | {give_raw_headers, boolean()} | {preserve_chunked_encoding, boolean()} | {workaround, head_response_with_body} | {worker_process_options, list()} | {return_raw_request, true} | {max_attempts, integer()}</tt></li><li><tt><a name="type-stream_to">stream_to()</a> = <a href="#type-process">process()</a> | {<a href="#type-process">process()</a>, once}</tt></li><li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li><li><tt><a name="type-username">username()</a> = string()</tt></li><li><tt><a name="type-password">password()</a> = string()</tt></li><li><tt>SSLOpt = term()</tt></li><li><tt>Sock_opts = [Sock_opt]</tt></li><li><tt>Sock_opt = term()</tt></li><li><tt>ChunkSize = integer()</tt></li><li><tt><a name="type-srtf">srtf()</a> = boolean() | <a href="#type-filename">filename()</a> | {append, <a href="#type-filename">filename()</a>}</tt></li><li><tt><a name="type-filename">filename()</a> = string()</tt></li><li><tt><a name="type-response_format">response_format()</a> = list | binary</tt></li></ul></p>
+</div><p>Same as send_req/4.</p>
+
+<h3 class="function"><a name="send_req-6">send_req/6</a></h3>
+<div class="spec">
+<p><tt>send_req(Url, Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>, Timeout) -&gt; <a href="#type-response">response()</a></tt>
+<ul class="definitions"><li><tt>Timeout = integer() | infinity</tt></li></ul></p>
+</div><p>Same as send_req/5.
+ All timeout values are in milliseconds.</p>
+
+<h3 class="function"><a name="send_req_direct-4">send_req_direct/4</a></h3>
+<div class="spec">
+<p><tt>send_req_direct(Conn_pid, Url, Headers, Method) -&gt; any()</tt></p>
+</div><p>Same as send_req/3 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
+
+<h3 class="function"><a name="send_req_direct-5">send_req_direct/5</a></h3>
+<div class="spec">
+<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body) -&gt; any()</tt></p>
+</div><p>Same as send_req/4 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
+
+<h3 class="function"><a name="send_req_direct-6">send_req_direct/6</a></h3>
+<div class="spec">
+<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -&gt; any()</tt></p>
+</div><p>Same as send_req/5 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
+
+<h3 class="function"><a name="send_req_direct-7">send_req_direct/7</a></h3>
+<div class="spec">
+<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -&gt; any()</tt></p>
+</div><p>Same as send_req/6 except that the first argument is the PID
+ returned by spawn_worker_process/2 or spawn_link_worker_process/2</p>
+
+<h3 class="function"><a name="set_dest-3">set_dest/3</a></h3>
+<div class="spec">
+<p><tt>set_dest(Host, Port, T) -&gt; any()</tt></p>
+</div><p>Deprecated. Use set_max_sessions/3 and set_max_pipeline_size/3
+ for achieving the same effect.</p>
+
+<h3 class="function"><a name="set_max_attempts-3">set_max_attempts/3</a></h3>
+<div class="spec">
+<p><tt>set_max_attempts(Host::string(), Port::integer(), Max::integer()) -&gt; ok</tt><br></p>
+</div><p>Set the maximum attempts for each connection to a specific Host:Port.</p>
+
+<h3 class="function"><a name="set_max_pipeline_size-3">set_max_pipeline_size/3</a></h3>
+<div class="spec">
+<p><tt>set_max_pipeline_size(Host::string(), Port::integer(), Max::integer()) -&gt; ok</tt><br></p>
+</div><p>Set the maximum pipeline size for each connection to a specific Host:Port.</p>
+
+<h3 class="function"><a name="set_max_sessions-3">set_max_sessions/3</a></h3>
+<div class="spec">
+<p><tt>set_max_sessions(Host::string(), Port::integer(), Max::integer()) -&gt; ok</tt><br></p>
+</div><p>Set the maximum number of connections allowed to a specific Host:Port.</p>
+
+<h3 class="function"><a name="show_dest_status-0">show_dest_status/0</a></h3>
+<div class="spec">
+<p><tt>show_dest_status() -&gt; any()</tt></p>
+</div><p>Shows some internal information about load balancing. Info
+ about workers spawned using spawn_worker_process/2 or
+ spawn_link_worker_process/2 is not included.</p>
+
+<h3 class="function"><a name="show_dest_status-1">show_dest_status/1</a></h3>
+<div class="spec">
+<p><tt>show_dest_status(Url) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3>
+<div class="spec">
+<p><tt>show_dest_status(Host, Port) -&gt; any()</tt></p>
+</div><p>Shows some internal information about load balancing to a
+ specified Host:Port. Info about workers spawned using
+ spawn_worker_process/2 or spawn_link_worker_process/2 is not
+ included.</p>
+
+<h3 class="function"><a name="spawn_link_worker_process-1">spawn_link_worker_process/1</a></h3>
+<div class="spec">
+<p><tt>spawn_link_worker_process(Url::string() | {Host::string(), Port::integer()}) -&gt; {ok, pid()}</tt><br></p>
+</div><p>Same as spawn_worker_process/1 except the the calling process
+ is linked to the worker process which is spawned.</p>
+
+<h3 class="function"><a name="spawn_link_worker_process-2">spawn_link_worker_process/2</a></h3>
+<div class="spec">
+<p><tt>spawn_link_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt><br></p>
+</div><p>Same as spawn_link_worker_process/1 except with Erlang process options.</p>
+
+<h3 class="function"><a name="spawn_worker_process-1">spawn_worker_process/1</a></h3>
+<div class="spec">
+<p><tt>spawn_worker_process(Url::string() | {Host::string(), Port::integer()}) -&gt; {ok, pid()}</tt><br></p>
+</div><p>Creates a HTTP client process to the specified Host:Port which
+ is not part of the load balancing pool. This is useful in cases
+ where some requests to a webserver might take a long time whereas
+ some might take a very short time. To avoid getting these quick
+ requests stuck in the pipeline behind time consuming requests, use
+ this function to get a handle to a connection process. <br>
+ <b>Note:</b> Calling this function only creates a worker process. No connection
+ is setup. The connection attempt is made only when the first
+ request is sent via any of the send_req_direct/4,5,6,7 functions.<br>
+ <b>Note:</b> It is the responsibility of the calling process to control
+ pipeline size on such connections.</p>
+
+<h3 class="function"><a name="spawn_worker_process-2">spawn_worker_process/2</a></h3>
+<div class="spec">
+<p><tt>spawn_worker_process(Host::string(), Port::integer()) -&gt; {ok, pid()}</tt><br></p>
+</div><p>Same as spawn_worker_process/1 except with Erlang process options.</p>
+
+<h3 class="function"><a name="start-0">start/0</a></h3>
+<div class="spec">
+<p><tt>start() -&gt; any()</tt></p>
+</div><p>Starts the ibrowse process without linking. Useful when testing using the shell</p>
+
+<h3 class="function"><a name="start_link-0">start_link/0</a></h3>
+<div class="spec">
+<p><tt>start_link() -&gt; {ok, pid()}</tt><br></p>
+</div><p>Starts the ibrowse process linked to the calling process. Usually invoked by the supervisor ibrowse_sup</p>
+
+<h3 class="function"><a name="stop-0">stop/0</a></h3>
+<div class="spec">
+<p><tt>stop() -&gt; any()</tt></p>
+</div><p>Stop the ibrowse process. Useful when testing using the shell.</p>
+
+<h3 class="function"><a name="stop_worker_process-1">stop_worker_process/1</a></h3>
+<div class="spec">
+<p><tt>stop_worker_process(Conn_pid::pid()) -&gt; ok</tt><br></p>
+</div><p>Terminate a worker process spawned using
+ spawn_worker_process/2 or spawn_link_worker_process/2. Requests in
+ progress will get the error response <pre>{error, closing_on_request}</pre></p>
+
+<h3 class="function"><a name="stream_close-1">stream_close/1</a></h3>
+<div class="spec">
+<p><tt>stream_close(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt><br></p>
+</div><p>Tell ibrowse to close the connection associated with the
+ specified stream. Should be used in conjunction with the
+ <code>stream_to</code> option. Note that all requests in progress on
+ the connection which is serving this Req_id will be aborted, and an
+ error returned.</p>
+
+<h3 class="function"><a name="stream_next-1">stream_next/1</a></h3>
+<div class="spec">
+<p><tt>stream_next(Req_id::<a href="#type-req_id">req_id()</a>) -&gt; ok | {error, unknown_req_id}</tt><br></p>
+</div><p>Tell ibrowse to stream the next chunk of data to the
+ caller. Should be used in conjunction with the
+ <code>stream_to</code> option</p>
+
+<h3 class="function"><a name="terminate-2">terminate/2</a></h3>
+<div class="spec">
+<p><tt>terminate(Reason, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="trace_off-0">trace_off/0</a></h3>
+<div class="spec">
+<p><tt>trace_off() -&gt; any()</tt></p>
+</div><p>Turn tracing off for the ibrowse process</p>
+
+<h3 class="function"><a name="trace_off-2">trace_off/2</a></h3>
+<div class="spec">
+<p><tt>trace_off(Host, Port) -&gt; ok</tt><br></p>
+</div><p>Turn tracing OFF for all connections to the specified HTTP
+ server.</p>
+
+<h3 class="function"><a name="trace_on-0">trace_on/0</a></h3>
+<div class="spec">
+<p><tt>trace_on() -&gt; any()</tt></p>
+</div><p>Turn tracing on for the ibrowse process</p>
+
+<h3 class="function"><a name="trace_on-2">trace_on/2</a></h3>
+<div class="spec">
+<p><tt>trace_on(Host, Port) -&gt; ok</tt>
+<ul class="definitions"><li><tt>Host = string()</tt></li><li><tt>Port = integer()</tt></li></ul></p>
+</div><p>Turn tracing on for all connections to the specified HTTP
+ server. Host is whatever is specified as the domain name in the URL</p>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_app.html b/doc/ibrowse_app.html
new file mode 100644
index 0000000..368a4b3
--- /dev/null
+++ b/doc/ibrowse_app.html
@@ -0,0 +1,38 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_app</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_app</h1>
+<ul class="index"><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
+<p><b>Behaviours:</b> <a href="application.html"><tt>application</tt></a>.</p>
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#start-2">start/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#stop-1">stop/1</a></td><td></td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="start-2">start/2</a></h3>
+<div class="spec">
+<p><tt>start(Type, StartArgs) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="stop-1">stop/1</a></h3>
+<div class="spec">
+<p><tt>stop(State) -&gt; any()</tt></p>
+</div>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_http_client.html b/doc/ibrowse_http_client.html
new file mode 100644
index 0000000..66f0f53
--- /dev/null
+++ b/doc/ibrowse_http_client.html
@@ -0,0 +1,98 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_http_client</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_http_client</h1>
+<ul class="index"><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
+<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#send_req-7">send_req/7</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start-1">start/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start-2">start/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start_link-1">start_link/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start_link-2">start_link/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#stop-1">stop/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="code_change-3">code_change/3</a></h3>
+<div class="spec">
+<p><tt>code_change(OldVsn, State, Extra) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_call-3">handle_call/3</a></h3>
+<div class="spec">
+<p><tt>handle_call(Request, From, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3>
+<div class="spec">
+<p><tt>handle_cast(Msg, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_info-2">handle_info/2</a></h3>
+<div class="spec">
+<p><tt>handle_info(Info, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="init-1">init/1</a></h3>
+<div class="spec">
+<p><tt>init(Url) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="send_req-7">send_req/7</a></h3>
+<div class="spec">
+<p><tt>send_req(Conn_Pid, Url, Headers, Method, Body, Options, Timeout) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start-1">start/1</a></h3>
+<div class="spec">
+<p><tt>start(Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start-2">start/2</a></h3>
+<div class="spec">
+<p><tt>start(Args, Options) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start_link-1">start_link/1</a></h3>
+<div class="spec">
+<p><tt>start_link(Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start_link-2">start_link/2</a></h3>
+<div class="spec">
+<p><tt>start_link(Args, Options) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="stop-1">stop/1</a></h3>
+<div class="spec">
+<p><tt>stop(Conn_pid) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="terminate-2">terminate/2</a></h3>
+<div class="spec">
+<p><tt>terminate(Reason, State) -&gt; any()</tt></p>
+</div>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_lb.html b/doc/ibrowse_lb.html
new file mode 100644
index 0000000..b115982
--- /dev/null
+++ b/doc/ibrowse_lb.html
@@ -0,0 +1,80 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_lb</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_lb</h1>
+<ul class="index"><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
+<p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p>
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_cast-2">handle_cast/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#handle_info-2">handle_info/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#spawn_connection-6">spawn_connection/6</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start_link-1">start_link/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#stop-1">stop/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="code_change-3">code_change/3</a></h3>
+<div class="spec">
+<p><tt>code_change(OldVsn, State, Extra) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_call-3">handle_call/3</a></h3>
+<div class="spec">
+<p><tt>handle_call(Request, From, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3>
+<div class="spec">
+<p><tt>handle_cast(Msg, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="handle_info-2">handle_info/2</a></h3>
+<div class="spec">
+<p><tt>handle_info(Info, State) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="init-1">init/1</a></h3>
+<div class="spec">
+<p><tt>init(X1) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="spawn_connection-6">spawn_connection/6</a></h3>
+<div class="spec">
+<p><tt>spawn_connection(Lb_pid, Url, Max_sessions, Max_pipeline_size, SSL_options, Process_options) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start_link-1">start_link/1</a></h3>
+<div class="spec">
+<p><tt>start_link(Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="stop-1">stop/1</a></h3>
+<div class="spec">
+<p><tt>stop(Lb_pid) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="terminate-2">terminate/2</a></h3>
+<div class="spec">
+<p><tt>terminate(Reason, State) -&gt; any()</tt></p>
+</div>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_lib.html b/doc/ibrowse_lib.html
new file mode 100644
index 0000000..4411692
--- /dev/null
+++ b/doc/ibrowse_lib.html
@@ -0,0 +1,114 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_lib</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_lib</h1>
+<ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>Module with a few useful functions.
+
+
+<h2><a name="description">Description</a></h2>Module with a few useful functions
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#decode_base64-1">decode_base64/1</a></td><td>Implements the base64 decoding algorithm.</td></tr>
+<tr><td valign="top"><a href="#decode_rfc822_date-1">decode_rfc822_date/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#do_trace-2">do_trace/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#do_trace-3">do_trace/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#do_trace-3">do_trace/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#encode_base64-1">encode_base64/1</a></td><td>Implements the base64 encoding algorithm.</td></tr>
+<tr><td valign="top"><a href="#get_trace_status-2">get_trace_status/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#get_value-2">get_value/2</a></td><td></td></tr>
+<tr><td valign="top"><a href="#get_value-3">get_value/3</a></td><td></td></tr>
+<tr><td valign="top"><a href="#parse_url-1">parse_url/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#printable_date-0">printable_date/0</a></td><td></td></tr>
+<tr><td valign="top"><a href="#printable_date-1">printable_date/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#status_code-1">status_code/1</a></td><td>Given a status code, returns an atom describing the status code.</td></tr>
+<tr><td valign="top"><a href="#url_encode-1">url_encode/1</a></td><td>URL-encodes a string based on RFC 1738.</td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="decode_base64-1">decode_base64/1</a></h3>
+<div class="spec">
+<p><tt>decode_base64(List::In) -&gt; Out | <a href="#type-exit">exit</a>({error, invalid_input})</tt>
+<ul class="definitions"><li><tt>In = string() | binary()</tt></li><li><tt>Out = string() | binary()</tt></li></ul></p>
+</div><p>Implements the base64 decoding algorithm. The output data type matches in the input data type.</p>
+
+<h3 class="function"><a name="decode_rfc822_date-1">decode_rfc822_date/1</a></h3>
+<div class="spec">
+<p><tt>decode_rfc822_date(String) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="do_trace-2">do_trace/2</a></h3>
+<div class="spec">
+<p><tt>do_trace(Fmt, Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="do_trace-3">do_trace/3</a></h3>
+<div class="spec">
+<p><tt>do_trace(X1, Fmt, Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="do_trace-3">do_trace/3</a></h3>
+<div class="spec">
+<p><tt>do_trace(X1, Fmt, Args) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="encode_base64-1">encode_base64/1</a></h3>
+<div class="spec">
+<p><tt>encode_base64(List::In) -&gt; Out</tt>
+<ul class="definitions"><li><tt>In = string() | binary()</tt></li><li><tt>Out = string() | binary()</tt></li></ul></p>
+</div><p>Implements the base64 encoding algorithm. The output data type matches in the input data type.</p>
+
+<h3 class="function"><a name="get_trace_status-2">get_trace_status/2</a></h3>
+<div class="spec">
+<p><tt>get_trace_status(Host, Port) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="get_value-2">get_value/2</a></h3>
+<div class="spec">
+<p><tt>get_value(Tag, TVL) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="get_value-3">get_value/3</a></h3>
+<div class="spec">
+<p><tt>get_value(Tag, TVL, DefVal) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="parse_url-1">parse_url/1</a></h3>
+<div class="spec">
+<p><tt>parse_url(Url) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="printable_date-0">printable_date/0</a></h3>
+<div class="spec">
+<p><tt>printable_date() -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="printable_date-1">printable_date/1</a></h3>
+<div class="spec">
+<p><tt>printable_date(Now) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="status_code-1">status_code/1</a></h3>
+<div class="spec">
+<p><tt>status_code(StatusCode::<a href="#type-status_code">status_code()</a>) -&gt; StatusDescription</tt>
+<ul class="definitions"><li><tt><a name="type-status_code">status_code()</a> = string() | integer()</tt></li><li><tt>StatusDescription = atom()</tt></li></ul></p>
+</div><p>Given a status code, returns an atom describing the status code.</p>
+
+<h3 class="function"><a name="url_encode-1">url_encode/1</a></h3>
+<div class="spec">
+<p><tt>url_encode(Str) -&gt; UrlEncodedStr</tt>
+<ul class="definitions"><li><tt>Str = string()</tt></li><li><tt>UrlEncodedStr = string()</tt></li></ul></p>
+</div><p>URL-encodes a string based on RFC 1738. Returns a flat list.</p>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_socks5.html b/doc/ibrowse_socks5.html
new file mode 100644
index 0000000..bddd420
--- /dev/null
+++ b/doc/ibrowse_socks5.html
@@ -0,0 +1,31 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_socks5</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_socks5</h1>
+<ul class="index"><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#connect-5">connect/5</a></td><td></td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="connect-5">connect/5</a></h3>
+<div class="spec">
+<p><tt>connect(Host, Port, Options, SockOptions, Timeout) -&gt; any()</tt></p>
+</div>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/ibrowse_sup.html b/doc/ibrowse_sup.html
new file mode 100644
index 0000000..0e5226d
--- /dev/null
+++ b/doc/ibrowse_sup.html
@@ -0,0 +1,38 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>Module ibrowse_sup</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<hr>
+
+<h1>Module ibrowse_sup</h1>
+<ul class="index"><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>
+
+<p><b>Behaviours:</b> <a href="supervisor.html"><tt>supervisor</tt></a>.</p>
+
+<h2><a name="index">Function Index</a></h2>
+<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#init-1">init/1</a></td><td></td></tr>
+<tr><td valign="top"><a href="#start_link-0">start_link/0</a></td><td></td></tr>
+</table>
+
+<h2><a name="functions">Function Details</a></h2>
+
+<h3 class="function"><a name="init-1">init/1</a></h3>
+<div class="spec">
+<p><tt>init(X1) -&gt; any()</tt></p>
+</div>
+
+<h3 class="function"><a name="start_link-0">start_link/0</a></h3>
+<div class="spec">
+<p><tt>start_link() -&gt; any()</tt></p>
+</div>
+<hr>
+
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/index.html b/doc/index.html
new file mode 100644
index 0000000..657b98c
--- /dev/null
+++ b/doc/index.html
@@ -0,0 +1,17 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<title>The ibrowse application</title>
+</head>
+<frameset cols="20%,80%">
+<frame src="modules-frame.html" name="modulesFrame" title="">
+
+<frame src="overview-summary.html" name="overviewFrame" title="">
+<noframes>
+<h2>This page uses frames</h2>
+<p>Your browser does not accept frames.
+<br>You should go to the <a href="overview-summary.html">non-frame version</a> instead.
+</p>
+</noframes>
+</frameset>
+</html>
\ No newline at end of file
diff --git a/doc/modules-frame.html b/doc/modules-frame.html
new file mode 100644
index 0000000..3d6f50c
--- /dev/null
+++ b/doc/modules-frame.html
@@ -0,0 +1,18 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<title>The ibrowse application</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<h2 class="indextitle">Modules</h2>
+<table width="100%" border="0" summary="list of modules">
+<tr><td><a href="ibrowse.html" target="overviewFrame" class="module">ibrowse</a></td></tr>
+<tr><td><a href="ibrowse_app.html" target="overviewFrame" class="module">ibrowse_app</a></td></tr>
+<tr><td><a href="ibrowse_http_client.html" target="overviewFrame" class="module">ibrowse_http_client</a></td></tr>
+<tr><td><a href="ibrowse_lb.html" target="overviewFrame" class="module">ibrowse_lb</a></td></tr>
+<tr><td><a href="ibrowse_lib.html" target="overviewFrame" class="module">ibrowse_lib</a></td></tr>
+<tr><td><a href="ibrowse_socks5.html" target="overviewFrame" class="module">ibrowse_socks5</a></td></tr>
+<tr><td><a href="ibrowse_sup.html" target="overviewFrame" class="module">ibrowse_sup</a></td></tr></table>
+</body>
+</html>
\ No newline at end of file
diff --git a/doc/overview-summary.html b/doc/overview-summary.html
new file mode 100644
index 0000000..b563096
--- /dev/null
+++ b/doc/overview-summary.html
@@ -0,0 +1,16 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<title>The ibrowse application</title>
+<link rel="stylesheet" type="text/css" href="stylesheet.css" title="EDoc">
+</head>
+<body bgcolor="white">
+<div class="navbar"><a name="#navbar_top"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<h1>The ibrowse application</h1>
+
+<hr>
+<div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div>
+<p><i>Generated by EDoc, Nov 6 2015, 11:40:24.</i></p>
+</body>
+</html>
diff --git a/doc/short-desc b/doc/short-desc
new file mode 100644
index 0000000..56fa909
--- /dev/null
+++ b/doc/short-desc
@@ -0,0 +1 @@
+A powerful HTTP/1.1 client written in erlang
diff --git a/doc/stylesheet.css b/doc/stylesheet.css
new file mode 100644
index 0000000..ab170c0
--- /dev/null
+++ b/doc/stylesheet.css
@@ -0,0 +1,55 @@
+/* standard EDoc style sheet */
+body {
+ font-family: Verdana, Arial, Helvetica, sans-serif;
+ margin-left: .25in;
+ margin-right: .2in;
+ margin-top: 0.2in;
+ margin-bottom: 0.2in;
+ color: #000000;
+ background-color: #ffffff;
+}
+h1,h2 {
+ margin-left: -0.2in;
+}
+div.navbar {
+ background-color: #add8e6;
+ padding: 0.2em;
+}
+h2.indextitle {
+ padding: 0.4em;
+ background-color: #add8e6;
+}
+h3.function,h3.typedecl {
+ background-color: #add8e6;
+ padding-left: 1em;
+}
+div.spec {
+ margin-left: 2em;
+ background-color: #eeeeee;
+}
+a.module {
+ text-decoration:none
+}
+a.module:hover {
+ background-color: #eeeeee;
+}
+ul.definitions {
+ list-style-type: none;
+}
+ul.index {
+ list-style-type: none;
+ background-color: #eeeeee;
+}
+
+/*
+ * Minor style tweaks
+ */
+ul {
+ list-style-type: square;
+}
+table {
+ border-collapse: collapse;
+}
+td {
+ padding: 3
+}
diff --git a/erlang.mk b/erlang.mk
new file mode 100644
index 0000000..972c5d1
--- /dev/null
+++ b/erlang.mk
@@ -0,0 +1,1279 @@
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+.PHONY: all deps rel app docs tests clean distclean help erlang-mk
+
+ERLANG_MK_VERSION = 1
+
+# Core configuration.
+
+PROJECT ?= $(notdir $(CURDIR))
+PROJECT := $(strip $(PROJECT))
+
+# Verbosity.
+
+V ?= 0
+
+gen_verbose_0 = @echo " GEN " $@;
+gen_verbose = $(gen_verbose_$(V))
+
+# "erl" command.
+
+ERL = erl +A0 -noinput -boot start_clean
+
+# Core targets.
+
+ifneq ($(words $(MAKECMDGOALS)),1)
+.NOTPARALLEL:
+endif
+
+all:: deps
+ @$(MAKE) --no-print-directory app
+ #@$(MAKE) --no-print-directory rel
+
+# Noop to avoid a Make warning when there's nothing to do.
+#rel::
+# @echo -n
+
+clean:: clean-crashdump
+
+clean-crashdump:
+ifneq ($(wildcard erl_crash.dump),)
+ $(gen_verbose) rm -f erl_crash.dump
+endif
+
+distclean:: clean
+
+help::
+ @printf "%s\n" \
+ "erlang.mk (version $(ERLANG_MK_VERSION)) is distributed under the terms of the ISC License." \
+ "Copyright (c) 2013-2014 Loïc Hoguin <essen@ninenines.eu>" \
+ "" \
+ "Usage: [V=1] make [-jNUM] [target]" \
+ "" \
+ "Core targets:" \
+ " all Run deps, app and rel targets in that order" \
+ " deps Fetch dependencies (if needed) and compile them" \
+ " app Compile the project" \
+ " rel Build a release for this project, if applicable" \
+ " docs Build the documentation for this project" \
+ " tests Run the tests for this project" \
+ " clean Delete temporary and output files from most targets" \
+ " distclean Delete all temporary and output files" \
+ " help Display this help and exit" \
+ "" \
+ "The target clean only removes files that are commonly removed." \
+ "Dependencies and releases are left untouched." \
+ "" \
+ "Setting V=1 when calling make enables verbose mode." \
+ "Parallel execution is supported through the -j Make flag."
+
+# Core functions.
+
+ifeq ($(shell which wget 2>/dev/null | wc -l), 1)
+define core_http_get
+ wget --no-check-certificate -O $(1) $(2)|| rm $(1)
+endef
+else
+define core_http_get
+ $(ERL) -eval 'ssl:start(), inets:start(), case httpc:request(get, {"$(2)", []}, [{autoredirect, true}], []) of {ok, {{_, 200, _}, _, Body}} -> case file:write_file("$(1)", Body) of ok -> ok; {error, R1} -> halt(R1) end; {error, R2} -> halt(R2) end, halt(0).'
+endef
+endif
+
+# Automated update.
+
+ERLANG_MK_BUILD_CONFIG ?= build.config
+ERLANG_MK_BUILD_DIR ?= .erlang.mk.build
+
+erlang-mk:
+ git clone https://github.com/ninenines/erlang.mk $(ERLANG_MK_BUILD_DIR)
+ if [ -f $(ERLANG_MK_BUILD_CONFIG) ]; then cp $(ERLANG_MK_BUILD_CONFIG) $(ERLANG_MK_BUILD_DIR); fi
+ cd $(ERLANG_MK_BUILD_DIR) && make
+ cp $(ERLANG_MK_BUILD_DIR)/erlang.mk ./erlang.mk
+ rm -rf $(ERLANG_MK_BUILD_DIR)
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: distclean-deps distclean-pkg pkg-list pkg-search
+
+# Configuration.
+
+AUTOPATCH ?= edown gen_leader gproc
+export AUTOPATCH
+
+DEPS_DIR ?= $(CURDIR)/deps
+export DEPS_DIR
+
+REBAR_DEPS_DIR = $(DEPS_DIR)
+export REBAR_DEPS_DIR
+
+ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(DEPS))
+
+ifeq ($(filter $(DEPS_DIR),$(subst :, ,$(ERL_LIBS))),)
+ifeq ($(ERL_LIBS),)
+ ERL_LIBS = $(DEPS_DIR)
+else
+ ERL_LIBS := $(ERL_LIBS):$(DEPS_DIR)
+endif
+endif
+export ERL_LIBS
+
+PKG_FILE2 ?= $(CURDIR)/.erlang.mk.packages.v2
+export PKG_FILE2
+
+PKG_FILE_URL ?= https://raw.githubusercontent.com/ninenines/erlang.mk/master/packages.v2.tsv
+
+# Core targets.
+
+deps:: $(ALL_DEPS_DIRS)
+ @for dep in $(ALL_DEPS_DIRS) ; do \
+ if [ -f $$dep/rebar ] ; then \
+ cd $$dep ; ./rebar compile skip_deps=true deps_dir=.. ; \
+ elif [ -f $$dep/rebar.config ] ; then \
+ cd $$dep ; rebar compile skip_deps=true deps_dir=.. ; \
+ elif [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ] ; then \
+ $(MAKE) -C $$dep ; \
+ else \
+ echo "include $(CURDIR)/erlang.mk" | ERLC_OPTS=+debug_info $(MAKE) -f - -C $$dep ; \
+ fi ; \
+ done
+
+distclean:: distclean-deps distclean-pkg
+
+# Deps related targets.
+
+define dep_autopatch
+ $(ERL) -eval " \
+DepDir = \"$(DEPS_DIR)/$(1)/\", \
+fun() -> \
+ {ok, Conf} = file:consult(DepDir ++ \"rebar.config\"), \
+ File = case lists:keyfind(deps, 1, Conf) of false -> []; {_, Deps} -> \
+ [begin {Method, Repo, Commit} = case Repos of \
+ {git, R} -> {git, R, master}; \
+ {M, R, {branch, C}} -> {M, R, C}; \
+ {M, R, {tag, C}} -> {M, R, C}; \
+ {M, R, C} -> {M, R, C} \
+ end, \
+ io_lib:format(\"DEPS += ~s\ndep_~s = ~s ~s ~s~n\", [Name, Name, Method, Repo, Commit]) \
+ end || {Name, _, Repos} <- Deps] \
+ end, \
+ ok = file:write_file(\"$(DEPS_DIR)/$(1)/Makefile\", [\"ERLC_OPTS = +debug_info\n\n\", File, \"\ninclude erlang.mk\"]) \
+end(), \
+AppSrcOut = \"$(DEPS_DIR)/$(1)/src/$(1).app.src\", \
+AppSrcIn = case filelib:is_regular(AppSrcOut) of false -> \"$(DEPS_DIR)/$(1)/ebin/$(1).app\"; true -> AppSrcOut end, \
+fun() -> \
+ {ok, [{application, $(1), L}]} = file:consult(AppSrcIn), \
+ L2 = case lists:keyfind(modules, 1, L) of {_, _} -> L; false -> [{modules, []}|L] end, \
+ L3 = case lists:keyfind(vsn, 1, L2) of {vsn, git} -> lists:keyreplace(vsn, 1, L2, {vsn, \"git\"}); _ -> L2 end, \
+ ok = file:write_file(AppSrcOut, io_lib:format(\"~p.~n\", [{application, $(1), L3}])) \
+end(), \
+case AppSrcOut of AppSrcIn -> ok; _ -> ok = file:delete(AppSrcIn) end, \
+halt()."
+endef
+
+ifeq ($(V),0)
+define dep_autopatch_verbose
+ @echo " PATCH " $(1);
+endef
+endif
+
+define dep_fetch
+ if [ "$$$$VS" = "git" ]; then \
+ git clone -n -- $$$$REPO $(DEPS_DIR)/$(1); \
+ cd $(DEPS_DIR)/$(1) && git checkout -q $$$$COMMIT; \
+ elif [ "$$$$VS" = "hg" ]; then \
+ hg clone -U $$$$REPO $(DEPS_DIR)/$(1); \
+ cd $(DEPS_DIR)/$(1) && hg update -q $$$$COMMIT; \
+ elif [ "$$$$VS" = "svn" ]; then \
+ svn checkout $$$$REPO $(DEPS_DIR)/$(1); \
+ else \
+ echo "Unknown or invalid dependency: $(1). Please consult the erlang.mk README for instructions." >&2; \
+ exit 78; \
+ fi
+endef
+
+define dep_target
+$(DEPS_DIR)/$(1):
+ @mkdir -p $(DEPS_DIR)
+ifeq (,$(dep_$(1)))
+ @if [ ! -f $(PKG_FILE2) ]; then $(call core_http_get,$(PKG_FILE2),$(PKG_FILE_URL)); fi
+ @DEPPKG=$$$$(awk 'BEGIN { FS = "\t" }; $$$$1 == "$(1)" { print $$$$2 " " $$$$3 " " $$$$4 }' $(PKG_FILE2);); \
+ VS=$$$$(echo $$$$DEPPKG | cut -d " " -f1); \
+ REPO=$$$$(echo $$$$DEPPKG | cut -d " " -f2); \
+ COMMIT=$$$$(echo $$$$DEPPKG | cut -d " " -f3); \
+ $(call dep_fetch,$(1))
+else
+ @VS=$(word 1,$(dep_$(1))); \
+ REPO=$(word 2,$(dep_$(1))); \
+ COMMIT=$(word 3,$(dep_$(1))); \
+ $(call dep_fetch,$(1))
+endif
+ifneq ($(filter $(1),$(AUTOPATCH)),)
+ $(call dep_autopatch_verbose,$(1)) if [ -f $(DEPS_DIR)/$(1)/rebar.config ]; then \
+ $(call dep_autopatch,$(1)); \
+ cd $(DEPS_DIR)/$(1)/ && ln -s ../../erlang.mk; \
+ elif [ ! -f $(DEPS_DIR)/$(1)/Makefile ]; then \
+ echo "ERLC_OPTS = +debug_info\ninclude erlang.mk" > $(DEPS_DIR)/$(1)/Makefile; \
+ cd $(DEPS_DIR)/$(1)/ && ln -s ../../erlang.mk; \
+ fi
+endif
+endef
+
+$(foreach dep,$(DEPS),$(eval $(call dep_target,$(dep))))
+
+distclean-deps:
+ $(gen_verbose) rm -rf $(DEPS_DIR)
+
+# Packages related targets.
+
+$(PKG_FILE2):
+ @$(call core_http_get,$(PKG_FILE2),$(PKG_FILE_URL))
+
+pkg-list: $(PKG_FILE2)
+ @cat $(PKG_FILE2) | awk 'BEGIN { FS = "\t" }; { print \
+ "Name:\t\t" $$1 "\n" \
+ "Repository:\t" $$3 "\n" \
+ "Website:\t" $$5 "\n" \
+ "Description:\t" $$6 "\n" }'
+
+ifdef q
+pkg-search: $(PKG_FILE2)
+ @cat $(PKG_FILE2) | grep -i ${q} | awk 'BEGIN { FS = "\t" }; { print \
+ "Name:\t\t" $$1 "\n" \
+ "Repository:\t" $$3 "\n" \
+ "Website:\t" $$5 "\n" \
+ "Description:\t" $$6 "\n" }'
+else
+pkg-search:
+ $(error Usage: make pkg-search q=STRING)
+endif
+
+ifeq ($(PKG_FILE2),$(CURDIR)/.erlang.mk.packages.v2)
+distclean-pkg:
+ $(gen_verbose) rm -f $(PKG_FILE2)
+endif
+
+help::
+ @printf "%s\n" "" \
+ "Package-related targets:" \
+ " pkg-list List all known packages" \
+ " pkg-search q=STRING Search for STRING in the package index"
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: clean-app
+
+# Configuration.
+
+ERLC_OPTS ?= -Werror +debug_info +warn_export_vars +warn_shadow_vars \
+ +warn_obsolete_guard # +bin_opt_info +warn_export_all +warn_missing_spec
+COMPILE_FIRST ?=
+COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .erl,$(COMPILE_FIRST)))
+ERLC_EXCLUDE ?=
+ERLC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .erl,$(ERLC_EXCLUDE)))
+
+ERLC_MIB_OPTS ?=
+COMPILE_MIB_FIRST ?=
+COMPILE_MIB_FIRST_PATHS = $(addprefix mibs/,$(addsuffix .mib,$(COMPILE_MIB_FIRST)))
+
+# Verbosity.
+
+appsrc_verbose_0 = @echo " APP " $(PROJECT).app.src;
+appsrc_verbose = $(appsrc_verbose_$(V))
+
+erlc_verbose_0 = @echo " ERLC " $(filter-out $(patsubst %,%.erl,$(ERLC_EXCLUDE)),\
+ $(filter %.erl %.core,$(?F)));
+erlc_verbose = $(erlc_verbose_$(V))
+
+xyrl_verbose_0 = @echo " XYRL " $(filter %.xrl %.yrl,$(?F));
+xyrl_verbose = $(xyrl_verbose_$(V))
+
+mib_verbose_0 = @echo " MIB " $(filter %.bin %.mib,$(?F));
+mib_verbose = $(mib_verbose_$(V))
+
+# Targets.
+
+ifeq ($(wildcard ebin/test),)
+app:: app-build
+else
+app:: clean app-build
+endif
+
+app-build: erlc-include ebin/$(PROJECT).app
+ $(eval MODULES := $(shell find ebin -type f -name \*.beam \
+ | sed "s/ebin\//'/;s/\.beam/',/" | sed '$$s/.$$//'))
+ @if [ -z "$$(grep -E '^[^%]*{modules,' src/$(PROJECT).app.src)" ]; then \
+ echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk README for instructions." >&2; \
+ exit 1; \
+ fi
+ $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null || true))
+ $(appsrc_verbose) cat src/$(PROJECT).app.src \
+ | sed "s/{modules,[[:space:]]*\[\]}/{modules, \[$(MODULES)\]}/" \
+ | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(GITDESCRIBE)\"}/" \
+ > ebin/$(PROJECT).app
+
+erlc-include:
+ -@if [ -d ebin/ ]; then \
+ find include/ src/ -type f -name \*.hrl -newer ebin -exec touch $(shell find src/ -type f -name "*.erl") \; 2>/dev/null || printf ''; \
+ fi
+
+define compile_erl
+ $(erlc_verbose) erlc -v $(ERLC_OPTS) -o ebin/ \
+ -pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),\
+ $(COMPILE_FIRST_PATHS) $(1))
+endef
+
+define compile_xyrl
+ $(xyrl_verbose) erlc -v -o ebin/ $(1)
+ $(xyrl_verbose) erlc $(ERLC_OPTS) -o ebin/ ebin/*.erl
+ @rm ebin/*.erl
+endef
+
+define compile_mib
+ $(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ \
+ -I priv/mibs/ $(COMPILE_MIB_FIRST_PATHS) $(1)
+ $(mib_verbose) erlc -o include/ -- priv/mibs/*.bin
+endef
+
+ifneq ($(wildcard src/),)
+ebin/$(PROJECT).app::
+ @mkdir -p ebin/
+
+ifneq ($(wildcard mibs/),)
+ebin/$(PROJECT).app:: $(shell find mibs -type f -name \*.mib)
+ @mkdir -p priv/mibs/ include
+ $(if $(strip $?),$(call compile_mib,$?))
+endif
+
+ebin/$(PROJECT).app:: $(shell find src -type f -name \*.erl) \
+ $(shell find src -type f -name \*.core)
+ $(if $(strip $?),$(call compile_erl,$?))
+
+ebin/$(PROJECT).app:: $(shell find src -type f -name \*.xrl) \
+ $(shell find src -type f -name \*.yrl)
+ $(if $(strip $?),$(call compile_xyrl,$?))
+endif
+
+clean:: clean-app
+
+clean-app:
+ $(gen_verbose) rm -rf ebin/ priv/mibs/ \
+ $(addprefix include/,$(addsuffix .hrl,$(notdir $(basename $(wildcard mibs/*.mib)))))
+
+# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: test-deps test-dir test-build clean-test-dir
+
+# Configuration.
+
+TEST_DIR ?= test
+
+ALL_TEST_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(TEST_DEPS))
+
+TEST_ERLC_OPTS ?= +debug_info +warn_export_vars +warn_shadow_vars +warn_obsolete_guard
+TEST_ERLC_OPTS += -DTEST=1
+
+# Targets.
+
+$(foreach dep,$(TEST_DEPS),$(eval $(call dep_target,$(dep))))
+
+test-deps: $(ALL_TEST_DEPS_DIRS)
+ @for dep in $(ALL_TEST_DEPS_DIRS) ; do $(MAKE) -C $$dep; done
+
+ifneq ($(strip $(TEST_DIR)),)
+test-dir:
+ $(gen_verbose) erlc -v $(TEST_ERLC_OPTS) -I include/ -o $(TEST_DIR) \
+ $(wildcard $(TEST_DIR)/*.erl $(TEST_DIR)/*/*.erl) -pa ebin/
+endif
+
+ifeq ($(wildcard ebin/test),)
+test-build: ERLC_OPTS=$(TEST_ERLC_OPTS)
+test-build: #todo clean deps test-deps
+ @$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)"
+ $(gen_verbose) touch ebin/test
+else
+test-build: ERLC_OPTS=$(TEST_ERLC_OPTS)
+test-build: #todo deps test-deps
+ @$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)"
+endif
+
+clean:: clean-test-dir
+
+clean-test-dir:
+ifneq ($(wildcard $(TEST_DIR)/*.beam),)
+ $(gen_verbose) rm -f $(TEST_DIR)/*.beam
+endif
+
+# Copyright (c) 2014-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: bootstrap bootstrap-lib bootstrap-rel new list-templates
+
+# Core targets.
+
+help::
+ @printf "%s\n" "" \
+ "Bootstrap targets:" \
+ " bootstrap Generate a skeleton of an OTP application" \
+ " bootstrap-lib Generate a skeleton of an OTP library" \
+ " bootstrap-rel Generate the files needed to build a release" \
+ " new t=TPL n=NAME Generate a module NAME based on the template TPL" \
+ " list-templates List available templates"
+
+# Bootstrap templates.
+
+bs_appsrc = "{application, $(PROJECT), [" \
+ " {description, \"\"}," \
+ " {vsn, \"0.1.0\"}," \
+ " {id, \"git\"}," \
+ " {modules, []}," \
+ " {registered, []}," \
+ " {applications, [" \
+ " kernel," \
+ " stdlib" \
+ " ]}," \
+ " {mod, {$(PROJECT)_app, []}}," \
+ " {env, []}" \
+ "]}."
+bs_appsrc_lib = "{application, $(PROJECT), [" \
+ " {description, \"\"}," \
+ " {vsn, \"0.1.0\"}," \
+ " {id, \"git\"}," \
+ " {modules, []}," \
+ " {registered, []}," \
+ " {applications, [" \
+ " kernel," \
+ " stdlib" \
+ " ]}" \
+ "]}."
+bs_Makefile = "PROJECT = $(PROJECT)" \
+ "include erlang.mk"
+bs_app = "-module($(PROJECT)_app)." \
+ "-behaviour(application)." \
+ "" \
+ "-export([start/2])." \
+ "-export([stop/1])." \
+ "" \
+ "start(_Type, _Args) ->" \
+ " $(PROJECT)_sup:start_link()." \
+ "" \
+ "stop(_State) ->" \
+ " ok."
+bs_relx_config = "{release, {$(PROJECT)_release, \"1\"}, [$(PROJECT)]}." \
+ "{extended_start_script, true}." \
+ "{sys_config, \"rel/sys.config\"}." \
+ "{vm_args, \"rel/vm.args\"}."
+bs_sys_config = "[" \
+ "]."
+bs_vm_args = "-name $(PROJECT)@127.0.0.1" \
+ "-setcookie $(PROJECT)" \
+ "-heart"
+# Normal templates.
+tpl_supervisor = "-module($(n))." \
+ "-behaviour(supervisor)." \
+ "" \
+ "-export([start_link/0])." \
+ "-export([init/1])." \
+ "" \
+ "start_link() ->" \
+ " supervisor:start_link({local, ?MODULE}, ?MODULE, [])." \
+ "" \
+ "init([]) ->" \
+ " Procs = []," \
+ " {ok, {{one_for_one, 1, 5}, Procs}}."
+tpl_gen_server = "-module($(n))." \
+ "-behaviour(gen_server)." \
+ "" \
+ "%% API." \
+ "-export([start_link/0])." \
+ "" \
+ "%% gen_server." \
+ "-export([init/1])." \
+ "-export([handle_call/3])." \
+ "-export([handle_cast/2])." \
+ "-export([handle_info/2])." \
+ "-export([terminate/2])." \
+ "-export([code_change/3])." \
+ "" \
+ "-record(state, {" \
+ "})." \
+ "" \
+ "%% API." \
+ "" \
+ "-spec start_link() -> {ok, pid()}." \
+ "start_link() ->" \
+ " gen_server:start_link(?MODULE, [], [])." \
+ "" \
+ "%% gen_server." \
+ "" \
+ "init([]) ->" \
+ " {ok, \#state{}}." \
+ "" \
+ "handle_call(_Request, _From, State) ->" \
+ " {reply, ignored, State}." \
+ "" \
+ "handle_cast(_Msg, State) ->" \
+ " {noreply, State}." \
+ "" \
+ "handle_info(_Info, State) ->" \
+ " {noreply, State}." \
+ "" \
+ "terminate(_Reason, _State) ->" \
+ " ok." \
+ "" \
+ "code_change(_OldVsn, State, _Extra) ->" \
+ " {ok, State}."
+tpl_gen_fsm = "-module($(n))." \
+ "-behaviour(gen_fsm)." \
+ "" \
+ "%% API." \
+ "-export([start_link/0])." \
+ "" \
+ "%% gen_fsm." \
+ "-export([init/1])." \
+ "-export([state_name/2])." \
+ "-export([handle_event/3])." \
+ "-export([state_name/3])." \
+ "-export([handle_sync_event/4])." \
+ "-export([handle_info/3])." \
+ "-export([terminate/3])." \
+ "-export([code_change/4])." \
+ "" \
+ "-record(state, {" \
+ "})." \
+ "" \
+ "%% API." \
+ "" \
+ "-spec start_link() -> {ok, pid()}." \
+ "start_link() ->" \
+ " gen_fsm:start_link(?MODULE, [], [])." \
+ "" \
+ "%% gen_fsm." \
+ "" \
+ "init([]) ->" \
+ " {ok, state_name, \#state{}}." \
+ "" \
+ "state_name(_Event, StateData) ->" \
+ " {next_state, state_name, StateData}." \
+ "" \
+ "handle_event(_Event, StateName, StateData) ->" \
+ " {next_state, StateName, StateData}." \
+ "" \
+ "state_name(_Event, _From, StateData) ->" \
+ " {reply, ignored, state_name, StateData}." \
+ "" \
+ "handle_sync_event(_Event, _From, StateName, StateData) ->" \
+ " {reply, ignored, StateName, StateData}." \
+ "" \
+ "handle_info(_Info, StateName, StateData) ->" \
+ " {next_state, StateName, StateData}." \
+ "" \
+ "terminate(_Reason, _StateName, _StateData) ->" \
+ " ok." \
+ "" \
+ "code_change(_OldVsn, StateName, StateData, _Extra) ->" \
+ " {ok, StateName, StateData}."
+tpl_cowboy_http = "-module($(n))." \
+ "-behaviour(cowboy_http_handler)." \
+ "" \
+ "-export([init/3])." \
+ "-export([handle/2])." \
+ "-export([terminate/3])." \
+ "" \
+ "-record(state, {" \
+ "})." \
+ "" \
+ "init(_, Req, _Opts) ->" \
+ " {ok, Req, \#state{}}." \
+ "" \
+ "handle(Req, State=\#state{}) ->" \
+ " {ok, Req2} = cowboy_req:reply(200, Req)," \
+ " {ok, Req2, State}." \
+ "" \
+ "terminate(_Reason, _Req, _State) ->" \
+ " ok."
+tpl_cowboy_loop = "-module($(n))." \
+ "-behaviour(cowboy_loop_handler)." \
+ "" \
+ "-export([init/3])." \
+ "-export([info/3])." \
+ "-export([terminate/3])." \
+ "" \
+ "-record(state, {" \
+ "})." \
+ "" \
+ "init(_, Req, _Opts) ->" \
+ " {loop, Req, \#state{}, 5000, hibernate}." \
+ "" \
+ "info(_Info, Req, State) ->" \
+ " {loop, Req, State, hibernate}." \
+ "" \
+ "terminate(_Reason, _Req, _State) ->" \
+ " ok."
+tpl_cowboy_rest = "-module($(n))." \
+ "" \
+ "-export([init/3])." \
+ "-export([content_types_provided/2])." \
+ "-export([get_html/2])." \
+ "" \
+ "init(_, _Req, _Opts) ->" \
+ " {upgrade, protocol, cowboy_rest}." \
+ "" \
+ "content_types_provided(Req, State) ->" \
+ " {[{{<<\"text\">>, <<\"html\">>, '*'}, get_html}], Req, State}." \
+ "" \
+ "get_html(Req, State) ->" \
+ " {<<\"<html><body>This is REST!</body></html>\">>, Req, State}."
+tpl_cowboy_ws = "-module($(n))." \
+ "-behaviour(cowboy_websocket_handler)." \
+ "" \
+ "-export([init/3])." \
+ "-export([websocket_init/3])." \
+ "-export([websocket_handle/3])." \
+ "-export([websocket_info/3])." \
+ "-export([websocket_terminate/3])." \
+ "" \
+ "-record(state, {" \
+ "})." \
+ "" \
+ "init(_, _, _) ->" \
+ " {upgrade, protocol, cowboy_websocket}." \
+ "" \
+ "websocket_init(_, Req, _Opts) ->" \
+ " Req2 = cowboy_req:compact(Req)," \
+ " {ok, Req2, \#state{}}." \
+ "" \
+ "websocket_handle({text, Data}, Req, State) ->" \
+ " {reply, {text, Data}, Req, State};" \
+ "websocket_handle({binary, Data}, Req, State) ->" \
+ " {reply, {binary, Data}, Req, State};" \
+ "websocket_handle(_Frame, Req, State) ->" \
+ " {ok, Req, State}." \
+ "" \
+ "websocket_info(_Info, Req, State) ->" \
+ " {ok, Req, State}." \
+ "" \
+ "websocket_terminate(_Reason, _Req, _State) ->" \
+ " ok."
+tpl_ranch_protocol = "-module($(n))." \
+ "-behaviour(ranch_protocol)." \
+ "" \
+ "-export([start_link/4])." \
+ "-export([init/4])." \
+ "" \
+ "-type opts() :: []." \
+ "-export_type([opts/0])." \
+ "" \
+ "-record(state, {" \
+ " socket :: inet:socket()," \
+ " transport :: module()" \
+ "})." \
+ "" \
+ "start_link(Ref, Socket, Transport, Opts) ->" \
+ " Pid = spawn_link(?MODULE, init, [Ref, Socket, Transport, Opts])," \
+ " {ok, Pid}." \
+ "" \
+ "-spec init(ranch:ref(), inet:socket(), module(), opts()) -> ok." \
+ "init(Ref, Socket, Transport, _Opts) ->" \
+ " ok = ranch:accept_ack(Ref)," \
+ " loop(\#state{socket=Socket, transport=Transport})." \
+ "" \
+ "loop(State) ->" \
+ " loop(State)."
+
+# Plugin-specific targets.
+
+bootstrap:
+ifneq ($(wildcard src/),)
+ $(error Error: src/ directory already exists)
+endif
+ @printf "%s\n" $(bs_Makefile) > Makefile
+ @mkdir src/
+ @printf "%s\n" $(bs_appsrc) > src/$(PROJECT).app.src
+ @printf "%s\n" $(bs_app) > src/$(PROJECT)_app.erl
+ $(eval n := $(PROJECT)_sup)
+ @printf "%s\n" $(tpl_supervisor) > src/$(PROJECT)_sup.erl
+
+bootstrap-lib:
+ifneq ($(wildcard src/),)
+ $(error Error: src/ directory already exists)
+endif
+ @printf "%s\n" $(bs_Makefile) > Makefile
+ @mkdir src/
+ @printf "%s\n" $(bs_appsrc_lib) > src/$(PROJECT).app.src
+
+bootstrap-rel:
+ifneq ($(wildcard relx.config),)
+ $(error Error: relx.config already exists)
+endif
+ifneq ($(wildcard rel/),)
+ $(error Error: rel/ directory already exists)
+endif
+ @printf "%s\n" $(bs_relx_config) > relx.config
+ @mkdir rel/
+ @printf "%s\n" $(bs_sys_config) > rel/sys.config
+ @printf "%s\n" $(bs_vm_args) > rel/vm.args
+
+new:
+ifeq ($(wildcard src/),)
+ $(error Error: src/ directory does not exist)
+endif
+ifndef t
+ $(error Usage: make new t=TEMPLATE n=NAME)
+endif
+ifndef tpl_$(t)
+ $(error Unknown template)
+endif
+ifndef n
+ $(error Usage: make new t=TEMPLATE n=NAME)
+endif
+ @printf "%s\n" $(tpl_$(t)) > src/$(n).erl
+
+list-templates:
+ @echo Available templates: $(sort $(patsubst tpl_%,%,$(filter tpl_%,$(.VARIABLES))))
+
+# Copyright (c) 2014-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: clean-c_src distclean-c_src-env
+# todo
+
+# Configuration.
+
+C_SRC_DIR = $(CURDIR)/c_src
+C_SRC_ENV ?= $(C_SRC_DIR)/env.mk
+C_SRC_OUTPUT ?= $(CURDIR)/priv/$(PROJECT).so
+
+# System type and C compiler/flags.
+
+UNAME_SYS := $(shell uname -s)
+ifeq ($(UNAME_SYS), Darwin)
+ CC ?= cc
+ CFLAGS ?= -O3 -std=c99 -arch x86_64 -finline-functions -Wall -Wmissing-prototypes
+ CXXFLAGS ?= -O3 -arch x86_64 -finline-functions -Wall
+ LDFLAGS ?= -arch x86_64 -flat_namespace -undefined suppress
+else ifeq ($(UNAME_SYS), FreeBSD)
+ CC ?= cc
+ CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes
+ CXXFLAGS ?= -O3 -finline-functions -Wall
+else ifeq ($(UNAME_SYS), Linux)
+ CC ?= gcc
+ CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes
+ CXXFLAGS ?= -O3 -finline-functions -Wall
+endif
+
+CFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I $(ERL_INTERFACE_INCLUDE_DIR)
+CXXFLAGS += -fPIC -I $(ERTS_INCLUDE_DIR) -I $(ERL_INTERFACE_INCLUDE_DIR)
+
+LDLIBS += -L $(ERL_INTERFACE_LIB_DIR) -lerl_interface -lei
+LDFLAGS += -shared
+
+# Verbosity.
+
+c_verbose_0 = @echo " C " $(?F);
+c_verbose = $(c_verbose_$(V))
+
+cpp_verbose_0 = @echo " CPP " $(?F);
+cpp_verbose = $(cpp_verbose_$(V))
+
+link_verbose_0 = @echo " LD " $(@F);
+link_verbose = $(link_verbose_$(V))
+
+# Targets.
+
+ifeq ($(wildcard $(C_SRC_DIR)),)
+else ifneq ($(wildcard $(C_SRC_DIR)/Makefile),)
+app::
+ $(MAKE) -C $(C_SRC_DIR)
+
+clean::
+ $(MAKE) -C $(C_SRC_DIR) clean
+
+else
+SOURCES := $(shell find $(C_SRC_DIR) -type f \( -name "*.c" -o -name "*.C" -o -name "*.cc" -o -name "*.cpp" \))
+OBJECTS = $(addsuffix .o, $(basename $(SOURCES)))
+
+COMPILE_C = $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c
+COMPILE_CPP = $(cpp_verbose) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c
+
+app:: $(C_SRC_ENV) $(C_SRC_OUTPUT)
+
+$(C_SRC_OUTPUT): $(OBJECTS)
+ @mkdir -p priv/
+ $(link_verbose) $(CC) $(OBJECTS) $(LDFLAGS) $(LDLIBS) -o $(C_SRC_OUTPUT)
+
+%.o: %.c
+ $(COMPILE_C) $(OUTPUT_OPTION) $<
+
+%.o: %.cc
+ $(COMPILE_CPP) $(OUTPUT_OPTION) $<
+
+%.o: %.C
+ $(COMPILE_CPP) $(OUTPUT_OPTION) $<
+
+%.o: %.cpp
+ $(COMPILE_CPP) $(OUTPUT_OPTION) $<
+
+$(C_SRC_ENV):
+ @$(ERL) -eval "file:write_file(\"$(C_SRC_ENV)\", \
+ io_lib:format( \
+ \"ERTS_INCLUDE_DIR ?= ~s/erts-~s/include/~n\" \
+ \"ERL_INTERFACE_INCLUDE_DIR ?= ~s~n\" \
+ \"ERL_INTERFACE_LIB_DIR ?= ~s~n\", \
+ [code:root_dir(), erlang:system_info(version), \
+ code:lib_dir(erl_interface, include), \
+ code:lib_dir(erl_interface, lib)])), \
+ halt()."
+
+clean:: clean-c_src
+
+clean-c_src:
+ $(gen_verbose) rm -f $(C_SRC_OUTPUT) $(OBJECTS)
+
+distclean:: distclean-c_src-env
+
+distclean-c_src-env:
+ $(gen_verbose) rm -f $(C_SRC_ENV)
+
+-include $(C_SRC_ENV)
+endif
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: ct distclean-ct
+
+# Configuration.
+
+CT_OPTS ?=
+ifneq ($(wildcard $(TEST_DIR)),)
+ CT_SUITES ?= $(sort $(subst _SUITE.erl,,$(shell find $(TEST_DIR) -type f -name \*_SUITE.erl -exec basename {} \;)))
+else
+ CT_SUITES ?=
+endif
+
+# Core targets.
+
+tests:: ct
+
+distclean:: distclean-ct
+
+help::
+ @printf "%s\n" "" \
+ "Common_test targets:" \
+ " ct Run all the common_test suites for this project" \
+ "" \
+ "All your common_test suites have their associated targets." \
+ "A suite named http_SUITE can be ran using the ct-http target."
+
+# Plugin-specific targets.
+
+CT_RUN = ct_run \
+ -no_auto_compile \
+ -noinput \
+ -pa ebin $(DEPS_DIR)/*/ebin \
+ -dir $(TEST_DIR) \
+ -logdir logs
+
+ifeq ($(CT_SUITES),)
+ct:
+else
+ct: test-build
+ @mkdir -p logs/
+ $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS)
+endif
+
+define ct_suite_target
+ct-$(1): test-build
+ @mkdir -p logs/
+ $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(1)) $(CT_OPTS)
+endef
+
+$(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test))))
+
+distclean-ct:
+ $(gen_verbose) rm -rf logs/
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: plt distclean-plt dialyze
+
+# Configuration.
+
+DIALYZER_PLT ?= $(CURDIR)/.$(PROJECT).plt
+export DIALYZER_PLT
+
+PLT_APPS ?=
+DIALYZER_DIRS ?= --src -r src
+DIALYZER_OPTS ?= -Werror_handling -Wrace_conditions \
+ -Wunmatched_returns # -Wunderspecs
+
+# Core targets.
+
+distclean:: distclean-plt
+
+help::
+ @printf "%s\n" "" \
+ "Dialyzer targets:" \
+ " plt Build a PLT file for this project" \
+ " dialyze Analyze the project using Dialyzer"
+
+# Plugin-specific targets.
+
+$(DIALYZER_PLT): deps app
+ @dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(ALL_DEPS_DIRS)
+
+plt: $(DIALYZER_PLT)
+
+distclean-plt:
+ $(gen_verbose) rm -f $(DIALYZER_PLT)
+
+ifneq ($(wildcard $(DIALYZER_PLT)),)
+dialyze:
+else
+dialyze: $(DIALYZER_PLT)
+endif
+ @dialyzer --no_native $(DIALYZER_DIRS) $(DIALYZER_OPTS)
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# Copyright (c) 2015, Viktor Söderqvist <viktor@zuiderkwast.se>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: distclean-edoc build-doc-deps
+
+# Configuration.
+
+EDOC_OPTS ?=
+
+# Core targets.
+
+docs:: distclean-edoc build-doc-deps
+ $(gen_verbose) $(ERL) -eval 'edoc:application($(PROJECT), ".", [$(EDOC_OPTS)]), halt().'
+
+distclean:: distclean-edoc
+
+# Plugin-specific targets.
+
+DOC_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(DOC_DEPS))
+
+$(foreach dep,$(DOC_DEPS),$(eval $(call dep_target,$(dep))))
+
+build-doc-deps: $(DOC_DEPS_DIRS)
+ @for dep in $(DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep; done
+
+distclean-edoc:
+ $(gen_verbose) rm -f doc/*.css doc/*.html doc/*.png doc/edoc-info
+
+# Copyright (c) 2014, Juan Facorro <juan@inaka.net>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: elvis distclean-elvis
+
+# Configuration.
+
+ELVIS_CONFIG ?= $(CURDIR)/elvis.config
+
+ELVIS ?= $(CURDIR)/elvis
+export ELVIS
+
+ELVIS_URL ?= https://github.com/inaka/elvis/releases/download/0.2.3/elvis
+ELVIS_CONFIG_URL ?= https://github.com/inaka/elvis/releases/download/0.2.3/elvis.config
+ELVIS_OPTS ?=
+
+# Core targets.
+
+help::
+ @printf "%s\n" "" \
+ "Elvis targets:" \
+ " elvis Run Elvis using the local elvis.config or download the default otherwise"
+
+distclean:: distclean-elvis
+
+# Plugin-specific targets.
+
+$(ELVIS):
+ @$(call core_http_get,$(ELVIS),$(ELVIS_URL))
+ @chmod +x $(ELVIS)
+
+$(ELVIS_CONFIG):
+ @$(call core_http_get,$(ELVIS_CONFIG),$(ELVIS_CONFIG_URL))
+
+elvis: $(ELVIS) $(ELVIS_CONFIG)
+ @$(ELVIS) rock -c $(ELVIS_CONFIG) $(ELVIS_OPTS)
+
+distclean-elvis:
+ $(gen_verbose) rm -rf $(ELVIS)
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+# Configuration.
+
+DTL_FULL_PATH ?= 0
+
+# Verbosity.
+
+dtl_verbose_0 = @echo " DTL " $(filter %.dtl,$(?F));
+dtl_verbose = $(dtl_verbose_$(V))
+
+# Core targets.
+
+define compile_erlydtl
+ $(dtl_verbose) $(ERL) -pa ebin/ $(DEPS_DIR)/erlydtl/ebin/ -eval ' \
+ Compile = fun(F) -> \
+ S = fun (1) -> re:replace(filename:rootname(string:sub_string(F, 11), ".dtl"), "/", "_", [{return, list}, global]); \
+ (0) -> filename:basename(F, ".dtl") \
+ end, \
+ Module = list_to_atom(string:to_lower(S($(DTL_FULL_PATH))) ++ "_dtl"), \
+ {ok, _} = erlydtl:compile(F, Module, [{out_dir, "ebin/"}, return_errors, {doc_root, "templates"}]) \
+ end, \
+ _ = [Compile(F) || F <- string:tokens("$(1)", " ")], \
+ halt().'
+endef
+
+ifneq ($(wildcard src/),)
+ebin/$(PROJECT).app:: $(shell find templates -type f -name \*.dtl 2>/dev/null)
+ $(if $(strip $?),$(call compile_erlydtl,$?))
+endif
+
+# Copyright (c) 2014 Dave Cottlehuber <dch@skunkwerks.at>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: distclean-escript escript
+
+# Configuration.
+
+ESCRIPT_NAME ?= $(PROJECT)
+ESCRIPT_COMMENT ?= This is an -*- erlang -*- file
+
+ESCRIPT_BEAMS ?= "ebin/*", "deps/*/ebin/*"
+ESCRIPT_SYS_CONFIG ?= "rel/sys.config"
+ESCRIPT_EMU_ARGS ?= -pa . \
+ -sasl errlog_type error \
+ -escript main $(ESCRIPT_NAME)
+ESCRIPT_SHEBANG ?= /usr/bin/env escript
+ESCRIPT_STATIC ?= "deps/*/priv/**", "priv/**"
+
+# Core targets.
+
+distclean:: distclean-escript
+
+help::
+ @printf "%s\n" "" \
+ "Escript targets:" \
+ " escript Build an executable escript archive" \
+
+# Plugin-specific targets.
+
+# Based on https://github.com/synrc/mad/blob/master/src/mad_bundle.erl
+# Copyright (c) 2013 Maxim Sokhatsky, Synrc Research Center
+# Modified MIT License, https://github.com/synrc/mad/blob/master/LICENSE :
+# Software may only be used for the great good and the true happiness of all
+# sentient beings.
+
+define ESCRIPT_RAW
+'Read = fun(F) -> {ok, B} = file:read_file(filename:absname(F)), B end,'\
+'Files = fun(L) -> A = lists:concat([filelib:wildcard(X)||X<- L ]),'\
+' [F || F <- A, not filelib:is_dir(F) ] end,'\
+'Squash = fun(L) -> [{filename:basename(F), Read(F) } || F <- L ] end,'\
+'Zip = fun(A, L) -> {ok,{_,Z}} = zip:create(A, L, [{compress,all},memory]), Z end,'\
+'Ez = fun(Escript) ->'\
+' Static = Files([$(ESCRIPT_STATIC)]),'\
+' Beams = Squash(Files([$(ESCRIPT_BEAMS), $(ESCRIPT_SYS_CONFIG)])),'\
+' Archive = Beams ++ [{ "static.gz", Zip("static.gz", Static)}],'\
+' escript:create(Escript, [ $(ESCRIPT_OPTIONS)'\
+' {archive, Archive, [memory]},'\
+' {shebang, "$(ESCRIPT_SHEBANG)"},'\
+' {comment, "$(ESCRIPT_COMMENT)"},'\
+' {emu_args, " $(ESCRIPT_EMU_ARGS)"}'\
+' ]),'\
+' file:change_mode(Escript, 8#755)'\
+'end,'\
+'Ez("$(ESCRIPT_NAME)"),'\
+'halt().'
+endef
+
+ESCRIPT_COMMAND = $(subst ' ',,$(ESCRIPT_RAW))
+
+escript:: distclean-escript deps app
+ $(gen_verbose) $(ERL) -eval $(ESCRIPT_COMMAND)
+
+distclean-escript:
+ $(gen_verbose) rm -f $(ESCRIPT_NAME)
+
+# Copyright (c) 2014, Enrique Fernandez <enrique.fernandez@erlang-solutions.com>
+# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is contributed to erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: eunit
+
+# Configuration
+
+ifeq ($(strip $(TEST_DIR)),)
+TAGGED_EUNIT_TESTS = {dir,"ebin"}
+else
+ifeq ($(wildcard $(TEST_DIR)),)
+TAGGED_EUNIT_TESTS = {dir,"ebin"}
+else
+# All modules in TEST_DIR
+TEST_DIR_MODS = $(notdir $(basename $(shell find $(TEST_DIR) -type f -name *.beam)))
+# All modules in 'ebin'
+EUNIT_EBIN_MODS = $(notdir $(basename $(shell find ebin -type f -name *.beam)))
+# Only those modules in TEST_DIR with no matching module in 'ebin'.
+# This is done to avoid some tests being executed twice.
+EUNIT_MODS = $(filter-out $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(TEST_DIR_MODS))
+TAGGED_EUNIT_TESTS = {dir,"ebin"} $(foreach mod,$(EUNIT_MODS),$(shell echo $(mod) | sed -e 's/\(.*\)/{module,\1}/g'))
+endif
+endif
+
+EUNIT_OPTS ?=
+
+# Utility functions
+
+define str-join
+ $(shell echo '$(strip $(1))' | sed -e "s/ /,/g")
+endef
+
+# Core targets.
+
+tests:: eunit
+
+help::
+ @printf "%s\n" "" \
+ "EUnit targets:" \
+ " eunit Run all the EUnit tests for this project"
+
+# Plugin-specific targets.
+
+EUNIT_RUN = $(ERL) \
+ -pa $(TEST_DIR) $(DEPS_DIR)/*/ebin \
+ -pz ebin \
+ -eval 'case eunit:test([$(call str-join,$(TAGGED_EUNIT_TESTS))], [$(EUNIT_OPTS)]) of ok -> halt(0); error -> halt(1) end.'
+
+eunit: test-build
+ $(gen_verbose) $(EUNIT_RUN)
+
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: relx-rel distclean-relx-rel distclean-relx
+
+# Configuration.
+
+RELX_CONFIG ?= $(CURDIR)/relx.config
+
+RELX ?= $(CURDIR)/relx
+export RELX
+
+RELX_URL ?= https://github.com/erlware/relx/releases/download/v1.2.0/relx
+RELX_OPTS ?=
+RELX_OUTPUT_DIR ?= _rel
+
+ifeq ($(firstword $(RELX_OPTS)),-o)
+ RELX_OUTPUT_DIR = $(word 2,$(RELX_OPTS))
+else
+ RELX_OPTS += -o $(RELX_OUTPUT_DIR)
+endif
+
+# Core targets.
+
+#ifneq ($(wildcard $(RELX_CONFIG)),)
+#rel:: distclean-relx-rel relx-rel
+#endif
+
+distclean:: distclean-relx-rel distclean-relx
+
+# Plugin-specific targets.
+
+define relx_fetch
+ $(call core_http_get,$(RELX),$(RELX_URL))
+ chmod +x $(RELX)
+endef
+
+$(RELX):
+ @$(call relx_fetch)
+
+relx-rel: $(RELX)
+ @$(RELX) -c $(RELX_CONFIG) $(RELX_OPTS)
+
+distclean-relx-rel:
+ $(gen_verbose) rm -rf $(RELX_OUTPUT_DIR)
+
+distclean-relx:
+ $(gen_verbose) rm -rf $(RELX)
+
+# Copyright (c) 2014, M Robert Martin <rob@version2beta.com>
+# This file is contributed to erlang.mk and subject to the terms of the ISC License.
+
+.PHONY: shell
+
+# Configuration.
+
+SHELL_PATH ?= -pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin
+SHELL_OPTS ?=
+
+ALL_SHELL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(SHELL_DEPS))
+
+# Core targets
+
+help::
+ @printf "%s\n" "" \
+ "Shell targets:" \
+ " shell Run an erlang shell with SHELL_OPTS or reasonable default"
+
+# Plugin-specific targets.
+
+$(foreach dep,$(SHELL_DEPS),$(eval $(call dep_target,$(dep))))
+
+build-shell-deps: $(ALL_SHELL_DEPS_DIRS)
+ @for dep in $(ALL_SHELL_DEPS_DIRS) ; do $(MAKE) -C $$dep ; done
+
+shell: build-shell-deps
+ $(gen_verbose) erl $(SHELL_PATH) $(SHELL_OPTS)
+
+# Copyright (c) 2015, Loïc Hoguin <essen@ninenines.eu>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+ifneq ($(wildcard $(DEPS_DIR)/triq),)
+.PHONY: triq
+
+# Targets.
+
+tests:: triq
+
+define triq_run
+$(ERL) -pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin \
+ -eval "try $(1) of true -> halt(0); _ -> halt(1) catch error:undef -> io:format(\"Undefined property or module~n\"), halt() end."
+endef
+
+ifdef t
+ifeq (,$(findstring :,$(t)))
+triq: test-build
+ @$(call triq_run,triq:check($(t)))
+else
+triq: test-build
+ @echo Testing $(t)/0
+ @$(call triq_run,triq:check($(t)()))
+endif
+else
+triq: test-build
+ $(eval MODULES := $(shell find ebin -type f -name \*.beam \
+ | sed "s/ebin\//'/;s/\.beam/',/" | sed '$$s/.$$//'))
+ $(gen_verbose) $(call triq_run,[true] =:= lists:usort([triq:check(M) || M <- [$(MODULES)]]))
+endif
+endif
diff --git a/include/ibrowse.hrl b/include/ibrowse.hrl
index 18dde82..5da1f0d 100644
--- a/include/ibrowse.hrl
+++ b/include/ibrowse.hrl
@@ -12,10 +12,15 @@
host_type % 'hostname', 'ipv4_address' or 'ipv6_address'
}).
--record(lb_pid, {host_port, pid}).
+-record(lb_pid, {host_port, pid, ets_tid}).
-record(client_conn, {key, cur_pipeline_size = 0, reqs_served = 0}).
-record(ibrowse_conf, {key, value}).
+-define(CONNECTIONS_LOCAL_TABLE, ibrowse_lb).
+-define(LOAD_BALANCER_NAMED_TABLE, ibrowse_lb).
+-define(CONF_TABLE, ibrowse_conf).
+-define(STREAM_TABLE, ibrowse_stream).
+
-endif.
diff --git a/priv/ibrowse.conf b/priv/ibrowse.conf
new file mode 100644
index 0000000..c142601
--- /dev/null
+++ b/priv/ibrowse.conf
@@ -0,0 +1,17 @@
+%% Configuration file for specifying settings for HTTP servers which this
+%% client will connect to.
+%% The format of each entry is (one per line)
+%% {dest, Hostname, Portnumber, MaxSessions, MaxPipelineSize, Options}.
+%%
+%% where Hostname = string()
+%% Portnumber = integer()
+%% MaxSessions = integer()
+%% MaxPipelineSize = integer()
+%% Options = [{Tag, Val} | ...]
+%% Tag = term()
+%% Value = term()
+%% e.g.
+%% {dest, "covig02", 8000, 10, 10, [{is_ssl, true}, {ssl_options, [option()]}]}.
+%% If SSL is to be used, both the options, is_ssl and ssl_options MUST be specified
+%% where option() is all options supported in the ssl module
+
diff --git a/rebar.config b/rebar.config
new file mode 100644
index 0000000..eaa892c
--- /dev/null
+++ b/rebar.config
@@ -0,0 +1,11 @@
+{erl_opts, [
+ debug_info,
+ warnings_as_errors,
+ warn_unused_vars,
+ nowarn_shadow_vars,
+ warn_unused_import,
+ {platform_define, "18|19|^2", new_rand},
+ {platform_define, "^2", ets_ref}
+ ]}.
+{xref_checks, [undefined_function_calls, deprecated_function_calls]}.
+{eunit_opts, [verbose]}.
diff --git a/rebar.lock b/rebar.lock
new file mode 100644
index 0000000..57afcca
--- /dev/null
+++ b/rebar.lock
@@ -0,0 +1 @@
+[].
diff --git a/src/Emakefile.src b/src/Emakefile.src
new file mode 100644
index 0000000..ff46b78
--- /dev/null
+++ b/src/Emakefile.src
@@ -0,0 +1,7 @@
+'../src/ibrowse'.
+'../src/ibrowse_http_client'.
+'../src/ibrowse_app'.
+'../src/ibrowse_sup'.
+'../src/ibrowse_lib'.
+'../src/ibrowse_lb'.
+'../src/ibrowse_test'.
diff --git a/src/ibrowse.app.src b/src/ibrowse.app.src
index 1d88084..7702e80 100644
--- a/src/ibrowse.app.src
+++ b/src/ibrowse.app.src
@@ -1,7 +1,13 @@
{application, ibrowse,
[{description, "Erlang HTTP client application"},
- {vsn, "4.0.1"},
+ {vsn, "4.4.1"},
{registered, [ibrowse_sup, ibrowse]},
{applications, [kernel,stdlib]},
{env, []},
- {mod, {ibrowse_app, []}}]}.
+ {mod, {ibrowse_app, []}},
+ {maintainers, ["Chandrashekhar Mullaparthi"]},
+ {licenses, ["GPLv2", "BSD"]},
+ {modules, []},
+ {links, [{"Github", "https://github.com/cmullaparthi/ibrowse"}]}
+ ]
+}.
diff --git a/src/ibrowse.erl b/src/ibrowse.erl
index 80a4282..2066799 100644
--- a/src/ibrowse.erl
+++ b/src/ibrowse.erl
@@ -6,7 +6,7 @@
%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
%%%-------------------------------------------------------------------
%% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com>
-%% @copyright 2005-2012 Chandrashekhar Mullaparthi
+%% @copyright 2005-2014 Chandrashekhar Mullaparthi
%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This
%% module implements the API of the HTTP client. There is one named
%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is
@@ -90,6 +90,7 @@
stream_close/1,
set_max_sessions/3,
set_max_pipeline_size/3,
+ set_max_attempts/3,
set_dest/3,
trace_on/0,
trace_off/0,
@@ -110,7 +111,8 @@
-import(ibrowse_lib, [
parse_url/1,
get_value/3,
- do_trace/2
+ do_trace/2,
+ log_msg/2
]).
-record(state, {trace = false}).
@@ -120,6 +122,7 @@
-define(DEF_MAX_SESSIONS,10).
-define(DEF_MAX_PIPELINE_SIZE,10).
+-define(DEF_MAX_ATTEMPTS,3).
%%====================================================================
%% External functions
@@ -150,7 +153,7 @@ stop() ->
%% The Status return value indicates the HTTP status code returned by the webserver
%% @spec send_req(Url::string(), Headers::headerList(), Method::method()) -> response()
%% headerList() = [{header(), value()}]
-%% header() = atom() | string()
+%% header() = atom() | string() | binary()
%% value() = term()
%% method() = get | post | head | options | put | delete | trace | mkcol | propfind | proppatch | lock | unlock | move | copy
%% Status = string()
@@ -175,9 +178,11 @@ send_req(Url, Headers, Method) ->
send_req(Url, Headers, Method, Body) ->
send_req(Url, Headers, Method, Body, []).
-%% @doc Same as send_req/4.
-%% For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html">ssl</a> manpage. If the
-%% HTTP Version to use is not specified, the default is 1.1.
+%% @doc Same as send_req/4.
+
+%% For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html">ssl</a> manpage.
+%% For a description of Process Options, look in the <a href="http://www.erlang.org/doc/man/gen_server.html">gen_server</a> manpage.
+%% If the HTTP Version to use is not specified, the default is 1.1.
%% <br/>
%% <ul>
%% <li>The <code>host_header</code> option is useful in the case where ibrowse is
@@ -250,16 +255,24 @@ send_req(Url, Headers, Method, Body) ->
%% headers. Not quite sure why someone would want this, but one of my
%% users asked for it, so here it is. </li>
%%
+%% <li> The <code>preserve_status_line</code> option is to get the raw status line as a custom header
+%% in the response. The status line is returned as a tuple {ibrowse_status_line, Status_line_binary}
+%% If both the <code>give_raw_headers</code> and <code>preserve_status_line</code> are specified
+%% in a request, only the <code>give_raw_headers</code> is honoured. </li>
+%%
%% <li> The <code>preserve_chunked_encoding</code> option enables the caller
%% to receive the raw data stream when the Transfer-Encoding of the server
%% response is Chunked.
%% </li>
+%% <li> The <code>return_raw_request</code> option enables the caller to get the exact request which was sent by ibrowse to the server, along with the response. When this option is used, the response for synchronous requests is a 5-tuple instead of the usual 4-tuple. For asynchronous requests, the calling process gets a message <code>{ibrowse_async_raw_req, Raw_req}</code>.
+%% </li>
%% </ul>
%%
%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response()
%% optionList() = [option()]
%% option() = {max_sessions, integer()} |
%% {response_format,response_format()}|
+%% {stream_full_chunks, boolean()} |
%% {stream_chunk_size, integer()} |
%% {max_pipeline_size, integer()} |
%% {trace, boolean()} |
@@ -286,8 +299,26 @@ send_req(Url, Headers, Method, Body) ->
%% {headers_as_is, boolean()} |
%% {give_raw_headers, boolean()} |
%% {preserve_chunked_encoding,boolean()} |
-%% {workaround, head_response_with_body}
+%% {workaround, head_response_with_body} |
+%% {worker_process_options, list()} |
+%% {return_raw_request, true} |
+%% {max_attempts, integer()} |
+%% {socks5_host, host()} |
+%% {socks5_port, integer()} |
+%% {socks5_user, binary()} |
+%% {socks5_password, binary()}
%%
+%% ip4_address() = {0..255, 0..255, 0..255, 0..255}
+%% ip6_address() =
+%% {0..65535,
+%% 0..65535,
+%% 0..65535,
+%% 0..65535,
+%% 0..65535,
+%% 0..65535,
+%% 0..65535,
+%% 0..65535}
+%% host() = string() | ip4_address() | ip6_address()
%% stream_to() = process() | {process(), once}
%% process() = pid() | atom()
%% username() = string()
@@ -319,6 +350,7 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
end,
Max_sessions = get_max_sessions(Host, Port, Options),
Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
+ Max_attempts = get_max_attempts(Host, Port, Options),
Options_1 = merge_options(Host, Port, Options),
{SSLOptions, IsSSL} =
case (Protocol == https) orelse
@@ -330,7 +362,7 @@ send_req(Url, Headers, Method, Body, Options, Timeout) ->
Max_sessions,
Max_pipeline_size,
{SSLOptions, IsSSL},
- Headers, Method, Body, Options_1, Timeout, 0);
+ Headers, Method, Body, Options_1, Timeout, Timeout, os:timestamp(), Max_attempts, 0);
Err ->
{error, {url_parsing_failed, Err}}
end.
@@ -339,27 +371,41 @@ try_routing_request(Lb_pid, Parsed_url,
Max_sessions,
Max_pipeline_size,
{SSLOptions, IsSSL},
- Headers, Method, Body, Options_1, Timeout, Try_count) when Try_count < 3 ->
+ Headers, Method, Body, Options_1, Timeout,
+ Ori_timeout, Req_start_time, Max_attempts, Try_count) when Try_count < Max_attempts ->
+ ProcessOptions = get_value(worker_process_options, Options_1, []),
case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
Max_sessions,
Max_pipeline_size,
- {SSLOptions, IsSSL}) of
- {ok, Conn_Pid} ->
+ {SSLOptions, IsSSL},
+ ProcessOptions) of
+ {ok, {_Pid_cur_spec_size, _, Conn_Pid}} ->
case do_send_req(Conn_Pid, Parsed_url, Headers,
Method, Body, Options_1, Timeout) of
{error, sel_conn_closed} ->
- try_routing_request(Lb_pid, Parsed_url,
- Max_sessions,
- Max_pipeline_size,
- {SSLOptions, IsSSL},
- Headers, Method, Body, Options_1, Timeout, Try_count + 1);
+ Time_now = os:timestamp(),
+ Time_taken_so_far = trunc(round(timer:now_diff(Time_now, Req_start_time)/1000)),
+ Time_remaining = Ori_timeout - Time_taken_so_far,
+ Time_remaining_percent = trunc(round((Time_remaining/Ori_timeout)*100)),
+ %% io:format("~p -- Time_remaining: ~p (~p%)~n", [self(), Time_remaining, Time_remaining_percent]),
+ case (Time_remaining > 0) andalso (Time_remaining_percent >= 5) of
+ true ->
+ try_routing_request(Lb_pid, Parsed_url,
+ Max_sessions,
+ Max_pipeline_size,
+ {SSLOptions, IsSSL},
+ Headers, Method, Body, Options_1,
+ Time_remaining, Ori_timeout, Req_start_time, Max_attempts, Try_count + 1);
+ false ->
+ {error, retry_later}
+ end;
Res ->
Res
end;
Err ->
Err
end;
-try_routing_request(_, _, _, _, _, _, _, _, _, _, _) ->
+try_routing_request(_, _, _, _, _, _, _, _, _, _, _, _, _, _) ->
{error, retry_later}.
merge_options(Host, Port, Options) ->
@@ -388,12 +434,20 @@ get_max_pipeline_size(Host, Port, Options) ->
get_config_value({max_pipeline_size, Host, Port},
default_max_pipeline_size())).
+get_max_attempts(Host, Port, Options) ->
+ get_value(max_attempts, Options,
+ get_config_value({max_attempts, Host, Port},
+ default_max_attempts())).
+
default_max_sessions() ->
safe_get_env(ibrowse, default_max_sessions, ?DEF_MAX_SESSIONS).
default_max_pipeline_size() ->
safe_get_env(ibrowse, default_max_pipeline_size, ?DEF_MAX_PIPELINE_SIZE).
+default_max_attempts() ->
+ safe_get_env(ibrowse, default_max_attempts, ?DEF_MAX_ATTEMPTS).
+
safe_get_env(App, Key, Def_val) ->
case application:get_env(App, Key) of
undefined ->
@@ -428,17 +482,39 @@ set_max_sessions(Host, Port, Max) when is_integer(Max), Max > 0 ->
set_max_pipeline_size(Host, Port, Max) when is_integer(Max), Max > 0 ->
gen_server:call(?MODULE, {set_config_value, {max_pipeline_size, Host, Port}, Max}).
+%% @doc Set the maximum attempts for each connection to a specific Host:Port.
+%% @spec set_max_attempts(Host::string(), Port::integer(), Max::integer()) -> ok
+set_max_attempts(Host, Port, Max) when is_integer(Max), Max > 0 ->
+ gen_server:call(?MODULE, {set_config_value, {max_attempts, Host, Port}, Max}).
+
do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) ->
case catch ibrowse_http_client:send_req(Conn_Pid, Parsed_url,
Headers, Method, ensure_bin(Body),
Options, Timeout) of
{'EXIT', {timeout, _}} ->
+ P_info = case catch erlang:process_info(Conn_Pid, [messages, message_queue_len, backtrace]) of
+ [_|_] = Conn_Pid_info_list ->
+ Conn_Pid_info_list;
+ _ ->
+ process_info_not_available
+ end,
+ log_msg("{ibrowse_http_client, send_req, ~1000.p} gen_server call timeout.~nProcess info: ~p~n",
+ [[Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout], P_info]),
{error, req_timedout};
- {'EXIT', {noproc, {gen_server, call, [Conn_Pid, _, _]}}} ->
+ {'EXIT', {normal, _}} = Ex_rsn ->
+ log_msg("{ibrowse_http_client, send_req, ~1000.p} gen_server call got ~1000.p~n",
+ [[Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout], Ex_rsn]),
+ {error, req_timedout};
+ {error, X} when X == connection_closed;
+ X == {send_failed, {error, enotconn}};
+ X == {send_failed,{error,einval}};
+ X == {send_failed,{error,closed}};
+ X == connection_closing;
+ ((X == connection_closed_no_retry) andalso ((Method == get) orelse (Method == head))) ->
{error, sel_conn_closed};
- {'EXIT', {normal, _}} ->
- {error, req_timedout};
- {error, connection_closed} ->
+ {error, connection_closed_no_retry} ->
+ {error, connection_closed};
+ {error, {'EXIT', {noproc, _}}} ->
{error, sel_conn_closed};
{'EXIT', Reason} ->
{error, {'EXIT', Reason}};
@@ -449,6 +525,13 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) ->
binary ->
Ret
end;
+ {ok, St_code, Headers, Body, Req} = Ret when is_binary(Body) ->
+ case get_value(response_format, Options, list) of
+ list ->
+ {ok, St_code, Headers, binary_to_list(Body), Req};
+ binary ->
+ Ret
+ end;
Ret ->
Ret
end.
@@ -470,28 +553,32 @@ ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body.
%% request is sent via any of the send_req_direct/4,5,6,7 functions.<br/>
%% <b>Note:</b> It is the responsibility of the calling process to control
%% pipeline size on such connections.
-%%
-%% @spec spawn_worker_process(Url::string()) -> {ok, pid()}
-spawn_worker_process(Url) ->
- ibrowse_http_client:start(Url).
-%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port
-%% instead of a URL.
+%% @spec spawn_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()}
+spawn_worker_process(Args) ->
+ spawn_worker_process(Args, []).
+
+%% @doc Same as spawn_worker_process/1 except with Erlang process options.
%% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
-spawn_worker_process(Host, Port) ->
- ibrowse_http_client:start({Host, Port}).
+spawn_worker_process(Host, Port) when is_list(Host), is_integer(Port) ->
+ %% Convert old API calls to new API format.
+ spawn_worker_process({Host, Port}, []);
+spawn_worker_process(Args, Options) ->
+ ibrowse_http_client:start(Args, Options).
%% @doc Same as spawn_worker_process/1 except the the calling process
%% is linked to the worker process which is spawned.
-%% @spec spawn_link_worker_process(Url::string()) -> {ok, pid()}
-spawn_link_worker_process(Url) ->
- ibrowse_http_client:start_link(Url).
+%% @spec spawn_link_worker_process(Url::string() | {Host::string(), Port::integer()}) -> {ok, pid()}
+spawn_link_worker_process(Args) ->
+ spawn_link_worker_process(Args, []).
-%% @doc Same as spawn_worker_process/2 except the the calling process
-%% is linked to the worker process which is spawned.
+%% @doc Same as spawn_link_worker_process/1 except with Erlang process options.
%% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()}
-spawn_link_worker_process(Host, Port) ->
- ibrowse_http_client:start_link({Host, Port}).
+spawn_link_worker_process(Host, Port) when is_list(Host), is_integer(Port) ->
+ %% Convert old API calls to new API format.
+ spawn_link_worker_process({Host, Port}, []);
+spawn_link_worker_process(Args, Options) ->
+ ibrowse_http_client:start_link(Args, Options).
%% @doc Terminate a worker process spawned using
%% spawn_worker_process/2 or spawn_link_worker_process/2. Requests in
@@ -592,19 +679,35 @@ all_trace_off() ->
%% @doc Shows some internal information about load balancing. Info
%% about workers spawned using spawn_worker_process/2 or
%% spawn_link_worker_process/2 is not included.
+-ifdef(ets_ref).
show_dest_status() ->
io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n",
["Server:port", "ETS", "Num conns", "LB Pid"]),
io:format("~80.80.=s~n", [""]),
Metrics = get_metrics(),
lists:foreach(
- fun({Host, Port, Lb_pid, Tid, Size}) ->
+ fun({Host, Port, {Lb_pid, _, Tid, Size, _}}) ->
+ io:format("~40.40s | ~-5.5s | ~-5.5s | ~p~n",
+ [Host ++ ":" ++ integer_to_list(Port),
+ ref_to_list(Tid),
+ integer_to_list(Size),
+ Lb_pid])
+ end, Metrics).
+-else.
+show_dest_status() ->
+ io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n",
+ ["Server:port", "ETS", "Num conns", "LB Pid"]),
+ io:format("~80.80.=s~n", [""]),
+ Metrics = get_metrics(),
+ lists:foreach(
+ fun({Host, Port, {Lb_pid, _, Tid, Size, _}}) ->
io:format("~40.40s | ~-5.5s | ~-5.5s | ~p~n",
[Host ++ ":" ++ integer_to_list(Port),
integer_to_list(Tid),
integer_to_list(Size),
Lb_pid])
end, Metrics).
+-endif.
show_dest_status(Url) ->
#url{host = Host, port = Port} = ibrowse_lib:parse_url(Url),
@@ -616,74 +719,68 @@ show_dest_status(Url) ->
%% included.
show_dest_status(Host, Port) ->
case get_metrics(Host, Port) of
- {Lb_pid, MsgQueueSize, Tid, Size,
- {{First_p_sz, First_speculative_sz},
- {Last_p_sz, Last_speculative_sz}}} ->
+ {Lb_pid, MsgQueueSize,
+ Tid, Size,
+ {{First_p_sz, First_p_sz},
+ {Last_p_sz, Last_p_sz}}} ->
io:format("Load Balancer Pid : ~p~n"
"LB process msg q size : ~p~n"
"LB ETS table id : ~p~n"
"Num Connections : ~p~n"
- "Smallest pipeline : ~p:~p~n"
- "Largest pipeline : ~p:~p~n",
+ "Smallest pipeline : ~p~n"
+ "Largest pipeline : ~p~n",
[Lb_pid, MsgQueueSize, Tid, Size,
- First_p_sz, First_speculative_sz,
- Last_p_sz, Last_speculative_sz]);
+ First_p_sz, Last_p_sz]);
_Err ->
io:format("Metrics not available~n", [])
end.
get_metrics() ->
- Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host),
- is_integer(Port) ->
- true;
- (_) ->
- false
- end, ets:tab2list(ibrowse_lb)),
- All_ets = ets:all(),
- lists:map(fun({lb_pid, {Host, Port}, Lb_pid}) ->
- case lists:dropwhile(
- fun(Tid) ->
- ets:info(Tid, owner) /= Lb_pid
- end, All_ets) of
- [] ->
- {Host, Port, Lb_pid, unknown, 0};
- [Tid | _] ->
- Size = case catch (ets:info(Tid, size)) of
- N when is_integer(N) -> N;
- _ -> 0
- end,
- {Host, Port, Lb_pid, Tid, Size}
- end
- end, Dests).
+ Dests = lists:filter(
+ fun(#lb_pid{host_port = {Host, Port}}) when is_list(Host),
+ is_integer(Port) ->
+ true;
+ (_) ->
+ false
+ end, ets:tab2list(ibrowse_lb)),
+ lists:foldl(
+ fun(#lb_pid{host_port = {X_host, X_port}}, X_acc) ->
+ case get_metrics(X_host, X_port) of
+ {_, _, _, _, _} = X_res ->
+ [{X_host, X_port, X_res} | X_acc];
+ _X_res ->
+ X_acc
+ end
+ end, [], Dests).
get_metrics(Host, Port) ->
case ets:lookup(ibrowse_lb, {Host, Port}) of
[] ->
no_active_processes;
- [#lb_pid{pid = Lb_pid}] ->
- MsgQueueSize = (catch process_info(Lb_pid, message_queue_len)),
- %% {Lb_pid, MsgQueueSize,
- case lists:dropwhile(
- fun(Tid) ->
- ets:info(Tid, owner) /= Lb_pid
- end, ets:all()) of
- [] ->
- {Lb_pid, MsgQueueSize, unknown, 0, unknown};
- [Tid | _] ->
+ [#lb_pid{pid = Lb_pid, ets_tid = Tid}] ->
+ MsgQueueSize = case (catch process_info(Lb_pid, message_queue_len)) of
+ {message_queue_len, Msg_q_len} ->
+ Msg_q_len;
+ _ ->
+ -1
+ end,
+ case Tid of
+ undefined ->
+ {Lb_pid, MsgQueueSize, undefined, 0, {{0, 0}, {0, 0}}};
+ _ ->
try
Size = ets:info(Tid, size),
case Size of
0 ->
- ok;
+ {Lb_pid, MsgQueueSize, Tid, 0, {{0, 0}, {0, 0}}};
_ ->
- First = ets:first(Tid),
- Last = ets:last(Tid),
- [{_, First_p_sz, First_speculative_sz}] = ets:lookup(Tid, First),
- [{_, Last_p_sz, Last_speculative_sz}] = ets:lookup(Tid, Last),
- {Lb_pid, MsgQueueSize, Tid, Size,
- {{First_p_sz, First_speculative_sz}, {Last_p_sz, Last_speculative_sz}}}
+ {First_p_sz, _, _} = ets:first(Tid),
+ {Last_p_sz, _, _} = ets:last(Tid),
+ {Lb_pid, MsgQueueSize,
+ Tid, Size,
+ {{First_p_sz, First_p_sz}, {Last_p_sz, Last_p_sz}}}
end
- catch _:_ ->
+ catch _:_Err ->
not_available
end
end
@@ -923,7 +1020,6 @@ code_change(_OldVsn, State, _Extra) ->
%%--------------------------------------------------------------------
do_get_connection(#url{host = Host, port = Port}, []) ->
{ok, Pid} = ibrowse_lb:start_link([Host, Port]),
- ets:insert(ibrowse_lb, #lb_pid{host_port = {Host, Port}, pid = Pid}),
Pid;
do_get_connection(_Url, [#lb_pid{pid = Pid}]) ->
Pid.
diff --git a/src/ibrowse_http_client.erl b/src/ibrowse_http_client.erl
index a1cf6eb..5faf074 100644
--- a/src/ibrowse_http_client.erl
+++ b/src/ibrowse_http_client.erl
@@ -15,7 +15,9 @@
%% External exports
-export([
start_link/1,
+ start_link/2,
start/1,
+ start/2,
stop/1,
send_req/7
]).
@@ -39,8 +41,7 @@
-record(state, {host, port, connect_timeout,
inactivity_timer_ref,
- use_http_proxy = false, http_proxy_auth_digest,
- socks5_host, socks5_port, socks5_user, socks5_password,
+ use_proxy = false, proxy_auth_basic,
ssl_options = [], is_ssl = false, socket,
proxy_tunnel_setup = false,
tunnel_setup_queue = [],
@@ -52,17 +53,19 @@
deleted_crlf = false, transfer_encoding,
chunk_size, chunk_size_buffer = <<>>,
recvd_chunk_size, interim_reply_sent = false,
- lb_ets_tid, cur_pipeline_size = 0, prev_req_id
+ lb_ets_tid, cur_pipeline_size = 0, prev_req_id,
+ proc_state
}).
-record(request, {url, method, options, from,
stream_to, caller_controls_socket = false,
caller_socket_options = [],
req_id,
+ stream_full_chunks = false,
stream_chunk_size,
save_response_to_file = false,
tmp_file_name, tmp_file_fd, preserve_chunked_encoding,
- response_format, timer_ref}).
+ response_format, timer_ref, raw_req}).
-import(ibrowse_lib, [
get_value/2,
@@ -72,6 +75,12 @@
-define(DEFAULT_STREAM_CHUNK_SIZE, 1024*1024).
-define(dec2hex(X), erlang:integer_to_list(X, 16)).
+
+%% Macros to prevent spelling mistakes causing bugs
+-define(dont_retry_pipelined_requests, dont_retry_pipelined_requests).
+-define(can_retry_pipelined_requests, can_retry_pipelined_requests).
+-define(dead_proc_walking, dead_proc_walking).
+
%%====================================================================
%% External functions
%%====================================================================
@@ -80,10 +89,16 @@
%% Description: Starts the server
%%--------------------------------------------------------------------
start(Args) ->
- gen_server:start(?MODULE, Args, []).
+ start(Args, []).
+
+start(Args, Options) ->
+ gen_server:start(?MODULE, Args, Options).
start_link(Args) ->
- gen_server:start_link(?MODULE, Args, []).
+ start_link(Args, []).
+
+start_link(Args, Options) ->
+ gen_server:start_link(?MODULE, Args, Options).
stop(Conn_pid) ->
case catch gen_server:call(Conn_pid, stop) of
@@ -95,9 +110,15 @@ stop(Conn_pid) ->
end.
send_req(Conn_Pid, Url, Headers, Method, Body, Options, Timeout) ->
- gen_server:call(
- Conn_Pid,
- {send_req, {Url, Headers, Method, Body, Options, Timeout}}, Timeout).
+ case catch gen_server:call(Conn_Pid,
+ {send_req, {Url, Headers, Method, Body, Options, Timeout}}, Timeout) of
+ {'EXIT', {timeout, _}} ->
+ {error, req_timedout};
+ {'EXIT', {noproc, _}} ->
+ {error, connection_closed};
+ Res ->
+ Res
+ end.
%%====================================================================
%% Server functions
@@ -112,6 +133,7 @@ send_req(Conn_Pid, Url, Headers, Method, Body, Options, Timeout) ->
%% {stop, Reason}
%%--------------------------------------------------------------------
init({Lb_Tid, #url{host = Host, port = Port}, {SSLOptions, Is_ssl}}) ->
+ process_flag(trap_exit, true),
State = #state{host = Host,
port = Port,
ssl_options = SSLOptions,
@@ -121,6 +143,7 @@ init({Lb_Tid, #url{host = Host, port = Port}, {SSLOptions, Is_ssl}}) ->
put(my_trace_flag, ibrowse_lib:get_trace_status(Host, Port)),
{ok, set_inac_timer(State)};
init(Url) when is_list(Url) ->
+ process_flag(trap_exit, true),
case catch ibrowse_lib:parse_url(Url) of
#url{protocol = Protocol} = Url_rec ->
init({undefined, Url_rec, {[], Protocol == https}});
@@ -128,6 +151,7 @@ init(Url) when is_list(Url) ->
{error, invalid_url}
end;
init({Host, Port}) ->
+ process_flag(trap_exit, true),
State = #state{host = Host,
port = Port},
put(ibrowse_trace_token, [Host, $:, integer_to_list(Port)]),
@@ -149,6 +173,10 @@ init({Host, Port}) ->
handle_call({send_req, _}, _From, #state{is_closing = true} = State) ->
{reply, {error, connection_closing}, State};
+handle_call({send_req, _}, _From, #state{proc_state = ?dead_proc_walking} = State) ->
+ shutting_down(State),
+ {reply, {error, connection_closing}, State};
+
handle_call({send_req, {Url, Headers, Method, Body, Options, Timeout}},
From, State) ->
send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State);
@@ -187,7 +215,7 @@ handle_info({ssl, _Sock, Data}, State) ->
handle_info({stream_next, Req_id}, #state{socket = Socket,
cur_req = #request{req_id = Req_id}} = State) ->
- do_setopts(Socket, [{active, once}], State),
+ _ = do_setopts(Socket, [{active, once}], State),
{noreply, set_inac_timer(State)};
handle_info({stream_next, _Req_id}, State) ->
@@ -200,51 +228,74 @@ handle_info({stream_next, _Req_id}, State) ->
{noreply, State};
handle_info({stream_close, _Req_id}, State) ->
- shutting_down(State),
- do_close(State),
- do_error_reply(State, closing_on_request),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ do_close(State_1),
+ do_error_reply(State_1, closing_on_request),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_info({tcp_closed, _Sock}, State) ->
do_trace("TCP connection closed by peer!~n", []),
- handle_sock_closed(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ handle_sock_closed(State_1, ?can_retry_pipelined_requests),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_info({ssl_closed, _Sock}, State) ->
do_trace("SSL connection closed by peer!~n", []),
- handle_sock_closed(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ handle_sock_closed(State_1, ?can_retry_pipelined_requests),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_info({tcp_error, _Sock, Reason}, State) ->
do_trace("Error on connection to ~1000.p:~1000.p -> ~1000.p~n",
[State#state.host, State#state.port, Reason]),
- handle_sock_closed(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ handle_sock_closed(State_1, ?dont_retry_pipelined_requests),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_info({ssl_error, _Sock, Reason}, State) ->
do_trace("Error on SSL connection to ~1000.p:~1000.p -> ~1000.p~n",
[State#state.host, State#state.port, Reason]),
- handle_sock_closed(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ handle_sock_closed(State_1, ?dont_retry_pipelined_requests),
+ delayed_stop_timer(),
+ {noreply, State_1};
-handle_info({req_timedout, From}, State) ->
- case lists:keymember(From, #request.from, queue:to_list(State#state.reqs)) of
+handle_info({req_timedout, From}, #state{reqs = Reqs} = State) ->
+ Reqs_list = queue:to_list(Reqs),
+ case lists:keysearch(From, #request.from, Reqs_list) of
false ->
{noreply, State};
- true ->
- shutting_down(State),
-%% do_error_reply(State, req_timedout),
- {stop, normal, State}
+ {value, #request{stream_to = StreamTo, req_id = ReqId}} ->
+ catch StreamTo ! {ibrowse_async_response_timeout, ReqId},
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ Reqs_1 = lists:filter(fun(#request{from = X_from}) ->
+ X_from /= From
+ end, Reqs_list),
+ State_2 = State_1#state{reqs = queue:from_list(Reqs_1)},
+ do_error_reply(State_2, req_timedout),
+ delayed_stop_timer(),
+ {noreply, State_2}
end;
handle_info(timeout, State) ->
do_trace("Inactivity timeout triggered. Shutting down connection~n", []),
- shutting_down(State),
- do_error_reply(State, req_timedout),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ do_error_reply(State_1, req_timedout),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_info({trace, Bool}, State) ->
put(my_trace_flag, Bool),
{noreply, State};
+handle_info(delayed_stop, State) ->
+ {stop, normal, State};
+
handle_info(Info, State) ->
io:format("Unknown message recvd for ~1000.p:~1000.p -> ~p~n",
[State#state.host, State#state.port, Info]),
@@ -256,8 +307,10 @@ handle_info(Info, State) ->
%% Description: Shutdown the server
%% Returns: any (ignored by gen_server)
%%--------------------------------------------------------------------
-terminate(_Reason, State) ->
+terminate(_Reason, #state{lb_ets_tid = Tid} = State) ->
do_close(State),
+ shutting_down(State),
+ (catch ets:select_delete(Tid, [{{{'_','_','$1'},'_'},[{'==','$1',{const,self()}}],[true]}])),
ok.
%%--------------------------------------------------------------------
@@ -277,23 +330,27 @@ code_change(_OldVsn, State, _Extra) ->
%%--------------------------------------------------------------------
handle_sock_data(Data, #state{status=idle}=State) ->
do_trace("Data recvd on socket in state idle!. ~1000.p~n", [Data]),
- shutting_down(State),
- do_error_reply(State, data_in_status_idle),
- do_close(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ do_error_reply(State_1, data_in_status_idle),
+ do_close(State_1),
+ delayed_stop_timer(),
+ {noreply, State_1};
handle_sock_data(Data, #state{status = get_header}=State) ->
case parse_response(Data, State) of
{error, _Reason} ->
- shutting_down(State),
- {stop, normal, State};
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ delayed_stop_timer(),
+ {noreply, State_1};
#state{socket = Socket, status = Status, cur_req = CurReq} = State_1 ->
- case {Status, CurReq} of
- {get_header, #request{caller_controls_socket = true}} ->
- do_setopts(Socket, [{active, once}], State_1);
- _ ->
- active_once(State_1)
- end,
+ _ = case {Status, CurReq} of
+ {get_header, #request{caller_controls_socket = true}} ->
+ do_setopts(Socket, [{active, once}], State_1);
+ _ ->
+ active_once(State_1)
+ end,
{noreply, set_inac_timer(State_1)}
end;
@@ -307,32 +364,36 @@ handle_sock_data(Data, #state{status = get_body,
true ->
case accumulate_response(Data, State) of
{error, Reason} ->
- shutting_down(State),
- fail_pipelined_requests(State,
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ fail_pipelined_requests(State_1,
{error, {Reason, {stat_code, StatCode}, Headers}}),
- {stop, normal, State};
+ delayed_stop_timer(),
+ {noreply, State_1};
State_1 ->
- active_once(State_1),
+ _ = active_once(State_1),
State_2 = set_inac_timer(State_1),
{noreply, State_2}
end;
_ ->
case parse_11_response(Data, State) of
{error, Reason} ->
- shutting_down(State),
- fail_pipelined_requests(State,
+ State_1 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_1),
+ fail_pipelined_requests(State_1,
{error, {Reason, {stat_code, StatCode}, Headers}}),
- {stop, normal, State};
+ delayed_stop_timer(),
+ {noreply, State_1};
#state{cur_req = #request{caller_controls_socket = Ccs},
interim_reply_sent = Irs} = State_1 ->
- case Irs of
- true ->
- active_once(State_1);
- false when Ccs == true ->
- do_setopts(Socket, [{active, once}], State);
- false ->
- active_once(State_1)
- end,
+ _ = case Irs of
+ true ->
+ active_once(State_1);
+ false when Ccs == true ->
+ do_setopts(Socket, [{active, once}], State);
+ false ->
+ active_once(State_1)
+ end,
State_2 = State_1#state{interim_reply_sent = false},
case Ccs of
true ->
@@ -342,7 +403,7 @@ handle_sock_data(Data, #state{status = get_body,
{noreply, set_inac_timer(State_2)}
end;
State_1 ->
- active_once(State_1),
+ _ = active_once(State_1),
State_2 = set_inac_timer(State_1),
{noreply, State_2}
end
@@ -425,7 +486,7 @@ accumulate_response(Data, #state{reply_buffer = RepBuf,
make_tmp_filename(true) ->
DownloadDir = ibrowse:get_config_value(download_dir, filename:absname("./")),
- {A,B,C} = now(),
+ {A,B,C} = os:timestamp(),
filename:join([DownloadDir,
"ibrowse_tmp_file_"++
integer_to_list(A) ++
@@ -443,11 +504,11 @@ file_mode(_Srtf) -> write.
%%--------------------------------------------------------------------
%% Handles the case when the server closes the socket
%%--------------------------------------------------------------------
-handle_sock_closed(#state{status=get_header} = State) ->
+handle_sock_closed(#state{status=get_header} = State, _) ->
shutting_down(State),
- do_error_reply(State, connection_closed);
+ do_error_reply(State, connection_closed_no_retry);
-handle_sock_closed(#state{cur_req=undefined} = State) ->
+handle_sock_closed(#state{cur_req=undefined} = State, _) ->
shutting_down(State);
%% We check for IsClosing because this the server could have sent a
@@ -461,10 +522,12 @@ handle_sock_closed(#state{reply_buffer = Buf, reqs = Reqs, http_status_code = SC
recvd_headers = Headers,
status_line = Status_line,
raw_headers = Raw_headers
- }=State) ->
+ }=State, Retry_state) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
- options = Options} = CurReq,
+ options = Options,
+ raw_req = Raw_req
+ } = CurReq,
case IsClosing of
true ->
{_, Reqs_1} = queue:out(Reqs),
@@ -475,39 +538,70 @@ handle_sock_closed(#state{reply_buffer = Buf, reqs = Reqs, http_status_code = SC
ok = file:close(Fd),
{file, TmpFilename}
end,
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false->
{ok, Status_line, Raw_headers, Body};
+ true ->
+ {ok, Status_line, Raw_headers, Body, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SC, Headers, Body};
false ->
- {ok, SC, Headers, Buf}
+ {ok, SC, Headers, Body, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
- ok = do_error_reply(State_1#state{reqs = Reqs_1}, connection_closed),
+ case Retry_state of
+ ?dont_retry_pipelined_requests ->
+ ok = do_error_reply(State_1#state{reqs = Reqs_1}, connection_closed_no_retry);
+ ?can_retry_pipelined_requests ->
+ ok = do_error_reply(State_1#state{reqs = Reqs_1}, connection_closed)
+ end,
State_1;
_ ->
- ok = do_error_reply(State, connection_closed),
+ case Retry_state of
+ ?dont_retry_pipelined_requests ->
+ ok = do_error_reply(State, connection_closed_no_retry);
+ ?can_retry_pipelined_requests ->
+ ok = do_error_reply(State, connection_closed)
+ end,
State
end.
-do_connect(Host, Port, Options, #state{socks5_host = SocksHost}=State, Timeout)
- when SocksHost /= undefined ->
- ProxyOptions = [
- {user, State#state.socks5_user},
- {password, State#state.socks5_password},
- {host, SocksHost},
- {port, State#state.socks5_port},
- {is_ssl, State#state.is_ssl},
- {ssl_opts, State#state.ssl_options}],
- ibrowse_socks5:connect(Host, Port, ProxyOptions,
- get_sock_options(SocksHost, Options, []),
- Timeout);
-do_connect(Host, Port, Options, #state{is_ssl = true,
- use_http_proxy = false,
- ssl_options = SSLOptions},
+do_connect(Host, Port, Options, #state{is_ssl = true,
+ use_proxy = false,
+ ssl_options = SSLOptions},
Timeout) ->
- ssl:connect(Host, Port, get_sock_options(Host, Options, SSLOptions), Timeout);
+ %% if a socks5 proxy is configured, open the socket separately
+ %% before upgrading the socket to a TLS connection.
+ case get_value(socks5_host, Options, undefined) of
+ %% no socks5 proxy is configured, connect directly with TLS:
+ undefined ->
+ Sock_options = get_sock_options(Host, Options, SSLOptions),
+ ssl:connect(Host, Port, Sock_options, Timeout);
+
+ %% proxy configuration is present: first establish a socket
+ %% and then upgrade:
+ _ ->
+ Sock_options = get_sock_options(Host, Options, []),
+ Conn = ibrowse_socks5:connect(Host, Port, Options,
+ Sock_options, Timeout),
+ case Conn of
+ {ok, Sock} ->
+ ssl:connect(Sock, SSLOptions, Timeout);
+ _ ->
+ error
+ end
+ end;
+
do_connect(Host, Port, Options, _State, Timeout) ->
- gen_tcp:connect(Host, Port, get_sock_options(Host, Options, []), Timeout).
+ Socks5Host = get_value(socks5_host, Options, undefined),
+ Sock_options = get_sock_options(Host, Options, []),
+ case Socks5Host of
+ undefined ->
+ gen_tcp:connect(Host, Port, Sock_options, Timeout);
+ _ ->
+ catch ibrowse_socks5:connect(Host, Port, Options, Sock_options, Timeout)
+ end.
get_sock_options(Host, Options, SSLOptions) ->
Caller_socket_options = get_value(socket_options, Options, []),
@@ -532,7 +626,7 @@ is_ipv6_host(Host) ->
{ok, {_, _, _, _}} ->
false;
_ ->
- case inet:gethostbyname(Host) of
+ case inet:gethostbyname(Host, inet6) of
{ok, #hostent{h_addrtype = inet6}} ->
true;
_ ->
@@ -554,42 +648,82 @@ filter_sock_options(Opts) ->
do_send(Req, #state{socket = Sock,
is_ssl = true,
- use_http_proxy = true,
+ use_proxy = true,
proxy_tunnel_setup = Pts}) when Pts /= done -> gen_tcp:send(Sock, Req);
do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req);
do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req).
-%% @spec do_send_body(Sock::socket_descriptor(), Source::source_descriptor(), IsSSL::boolean()) -> ok | error()
-%% source_descriptor() = fun_arity_0 |
-%% {fun_arity_0} |
-%% {fun_arity_1, term()}
-%% error() = term()
do_send_body(Source, State, TE) when is_function(Source) ->
do_send_body({Source}, State, TE);
do_send_body({Source}, State, TE) when is_function(Source) ->
- do_send_body1(Source, Source(), State, TE);
+ do_send_body_1(generate_body(Source),
+ State, TE, []);
do_send_body({Source, Source_state}, State, TE) when is_function(Source) ->
- do_send_body1(Source, Source(Source_state), State, TE);
+ do_send_body_1(generate_body({Source, Source_state}),
+ State, TE, []);
do_send_body(Body, State, _TE) ->
- do_send(Body, State).
+ case do_send(Body, State) of
+ ok ->
+ {ok, Body};
+ Ret ->
+ Ret
+ end.
-do_send_body1(Source, Resp, State, TE) ->
+generate_body({Source, Source_state} = In) when is_function(Source) ->
+ case Source(Source_state) of
+ {ok, Data, Source_state_1} ->
+ {{ok, Data, Source_state_1}, Source};
+ {eof, Source_state_1} ->
+ {{eof, Source_state_1}, Source};
+ eof ->
+ {eof, Source};
+ Ret ->
+ {Ret, In}
+ end;
+generate_body(Source) when is_function(Source) ->
+ {Source(), Source}.
+
+do_send_body_1({Resp, Source}, State, TE, Acc) when is_function(Source) ->
case Resp of
- {ok, Data} when Data == []; Data == <<>> ->
- do_send_body({Source}, State, TE);
+ {ok, Data} when Data == []; Data == <<>> ->
+ do_send_body_1(generate_body(Source), State, TE, Acc);
{ok, Data} ->
- do_send(maybe_chunked_encode(Data, TE), State),
- do_send_body({Source}, State, TE);
- {ok, Data, New_source_state} when Data == []; Data == <<>> ->
- do_send_body({Source, New_source_state}, State, TE);
+ Acc_1 = case TE of
+ true ->
+ ok = do_send(maybe_chunked_encode(Data, TE), State),
+ Acc;
+ false ->
+ [Data | Acc]
+ end,
+ do_send_body_1(generate_body(Source), State, TE, Acc_1);
+ {ok, Data, New_source_state} when Data == []; Data == <<>> ->
+ do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc);
{ok, Data, New_source_state} ->
- do_send(maybe_chunked_encode(Data, TE), State),
- do_send_body({Source, New_source_state}, State, TE);
+ Acc_1 = case TE of
+ true ->
+ ok = do_send(maybe_chunked_encode(Data, TE), State),
+ Acc;
+ false ->
+ [Data | Acc]
+ end,
+ do_send_body_1(generate_body({Source, New_source_state}), State, TE, Acc_1);
+ {eof, _New_source_state} ->
+ case TE of
+ true ->
+ ok = do_send(<<"0\r\n\r\n">>, State),
+ {ok, []};
+ _ ->
+ Body = list_to_binary(lists:reverse(Acc)),
+ ok = do_send(Body, State),
+ {ok, Body}
+ end;
eof when TE == true ->
- do_send(<<"0\r\n\r\n">>, State),
- ok;
+ ok = do_send(<<"0\r\n\r\n">>, State),
+ {ok, []};
eof ->
- ok;
+ Body = list_to_binary(lists:reverse(Acc)),
+ ok = do_send(Body, State),
+ {ok, Body};
Err ->
Err
end.
@@ -602,7 +736,7 @@ maybe_chunked_encode(Data, true) ->
do_close(#state{socket = undefined}) -> ok;
do_close(#state{socket = Sock,
is_ssl = true,
- use_http_proxy = true,
+ use_proxy = true,
proxy_tunnel_setup = Pts
}) when Pts /= done -> catch gen_tcp:close(Sock);
do_close(#state{socket = Sock, is_ssl = true}) -> catch ssl:close(Sock);
@@ -611,11 +745,11 @@ do_close(#state{socket = Sock, is_ssl = false}) -> catch gen_tcp:close(Sock).
active_once(#state{cur_req = #request{caller_controls_socket = true}}) ->
ok;
active_once(#state{socket = Socket} = State) ->
- do_setopts(Socket, [{active, once}], State).
+ _ = do_setopts(Socket, [{active, once}], State).
do_setopts(_Sock, [], _) -> ok;
do_setopts(Sock, Opts, #state{is_ssl = true,
- use_http_proxy = true,
+ use_proxy = true,
proxy_tunnel_setup = Pts}
) when Pts /= done -> inet:setopts(Sock, Opts);
do_setopts(Sock, Opts, #state{is_ssl = true}) -> ssl:setopts(Sock, Opts);
@@ -634,28 +768,17 @@ send_req_1(From,
port = Port} = Url,
Headers, Method, Body, Options, Timeout,
#state{socket = undefined} = State) ->
- ProxyHost = get_value(proxy_host, Options, false),
- ProxyProtocol = get_value(proxy_protocol, Options, http),
{Host_1, Port_1, State_1} =
- case {ProxyHost, ProxyProtocol} of
- {false, _} ->
+ case get_value(proxy_host, Options, false) of
+ false ->
{Host, Port, State};
- {_, http} ->
+ PHost ->
ProxyUser = get_value(proxy_user, Options, []),
ProxyPassword = get_value(proxy_password, Options, []),
- Digest = http_auth_digest(ProxyUser, ProxyPassword),
- {ProxyHost, get_value(proxy_port, Options, 80),
- State#state{use_http_proxy = true,
- http_proxy_auth_digest = Digest}};
- {_, socks5} ->
- ProxyUser = list_to_binary(get_value(proxy_user, Options, [])),
- ProxyPassword = list_to_binary(get_value(proxy_password, Options, [])),
- ProxyPort = get_value(proxy_port, Options, 1080),
- {Host, Port,
- State#state{socks5_host = ProxyHost,
- socks5_port = ProxyPort,
- socks5_user = ProxyUser,
- socks5_password = ProxyPassword}}
+ AuthBasic = http_auth_basic(ProxyUser, ProxyPassword),
+ {PHost, get_value(proxy_port, Options, 80),
+ State#state{use_proxy = true,
+ proxy_auth_basic = AuthBasic}}
end,
State_2 = check_ssl_options(Options, State_1),
do_trace("Connecting...~n", []),
@@ -667,10 +790,12 @@ send_req_1(From,
connect_timeout = Conn_timeout},
send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State_3);
Err ->
- shutting_down(State_2),
+ State_3 = State_2#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_3),
do_trace("Error connecting. Reason: ~1000.p~n", [Err]),
gen_server:reply(From, {error, {conn_failed, Err}}),
- {stop, normal, State_2}
+ delayed_stop_timer(),
+ {noreply, State_3}
end;
%% Send a CONNECT request.
@@ -686,7 +811,7 @@ send_req_1(From,
Headers, Method, Body, Options, Timeout,
#state{
proxy_tunnel_setup = false,
- use_http_proxy = true,
+ use_proxy = true,
is_ssl = true} = State) ->
Ref = case Timeout of
infinity ->
@@ -711,9 +836,9 @@ send_req_1(From,
case do_send(Req, State) of
ok ->
case do_send_body(Body_1, State_1, TE) of
- ok ->
+ {ok, _Sent_body} ->
trace_request_body(Body_1),
- active_once(State_1),
+ _ = active_once(State_1),
State_1_1 = inc_pipeline_counter(State_1),
State_2 = State_1_1#state{status = get_header,
cur_req = NewReq,
@@ -722,16 +847,20 @@ send_req_1(From,
State_3 = set_inac_timer(State_2),
{noreply, State_3};
Err ->
- shutting_down(State_1),
+ State_2 = State_1#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_2),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ delayed_stop_timer(),
+ {noreply, State_2}
end;
Err ->
- shutting_down(State_1),
+ State_2 = State_1#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_2),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ delayed_stop_timer(),
+ {noreply, State_2}
end;
send_req_1(From, Url, Headers, Method, Body, Options, Timeout,
@@ -775,6 +904,11 @@ send_req_1(From,
_ ->
erlang:send_after(Timeout, self(), {req_timedout, From})
end,
+ Headers_1 = maybe_modify_headers(Url, Method, Options, Headers, State),
+ {Req, Body_1} = make_request(Method,
+ Headers_1,
+ AbsPath, RelPath, Body, Options, State,
+ ReqId),
NewReq = #request{url = Url,
method = Method,
stream_to = StreamTo,
@@ -783,32 +917,31 @@ send_req_1(From,
options = Options,
req_id = ReqId,
save_response_to_file = SaveResponseToFile,
+ stream_full_chunks = get_value(stream_full_chunks, Options, false),
stream_chunk_size = get_stream_chunk_size(Options),
response_format = Resp_format,
from = From,
preserve_chunked_encoding = get_value(preserve_chunked_encoding, Options, false),
timer_ref = Ref
},
- State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)},
- Headers_1 = maybe_modify_headers(Url, Method, Options, Headers, State_1),
- {Req, Body_1} = make_request(Method,
- Headers_1,
- AbsPath, RelPath, Body, Options, State_1,
- ReqId),
trace_request(Req),
- do_setopts(Socket, Caller_socket_options, State_1),
+ ok = do_setopts(Socket, Caller_socket_options, State),
TE = is_chunked_encoding_specified(Options),
- case do_send(Req, State_1) of
+ case do_send(Req, State) of
ok ->
- case do_send_body(Body_1, State_1, TE) of
- ok ->
- trace_request_body(Body_1),
+ case do_send_body(Body_1, State, TE) of
+ {ok, Sent_body} ->
+ trace_request_body(Sent_body),
+ Raw_req = list_to_binary([Req, Sent_body]),
+ NewReq_1 = NewReq#request{raw_req = Raw_req},
+ State_1 = State#state{reqs=queue:in(NewReq_1, State#state.reqs)},
State_2 = inc_pipeline_counter(State_1),
- active_once(State_2),
+ _ = active_once(State_2),
State_3 = case Status of
idle ->
- State_2#state{status = get_header,
- cur_req = NewReq};
+ State_2#state{
+ status = get_header,
+ cur_req = NewReq_1};
_ ->
State_2
end,
@@ -816,21 +949,31 @@ send_req_1(From,
undefined ->
ok;
_ ->
- gen_server:reply(From, {ibrowse_req_id, ReqId})
+ gen_server:reply(From, {ibrowse_req_id, ReqId}),
+ case get_value(return_raw_request, Options, false) of
+ false ->
+ ok;
+ true ->
+ catch StreamTo ! {ibrowse_async_raw_req, Raw_req}
+ end
end,
State_4 = set_inac_timer(State_3),
{noreply, State_4};
Err ->
- shutting_down(State_1),
+ State_2 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_2),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ delayed_stop_timer(),
+ {noreply, State_2}
end;
Err ->
- shutting_down(State_1),
+ State_2 = State#state{proc_state = ?dead_proc_walking},
+ shutting_down(State_2),
do_trace("Send failed... Reason: ~p~n", [Err]),
gen_server:reply(From, {error, {send_failed, Err}}),
- {stop, normal, State_1}
+ delayed_stop_timer(),
+ {noreply, State_2}
end.
maybe_modify_headers(#url{}, connect, _, Headers, State) ->
@@ -867,31 +1010,31 @@ add_auth_headers(#url{username = User,
undefined ->
Headers;
{U,P} ->
- [{"Authorization", ["Basic ", http_auth_digest(U, P)]} | Headers]
+ [{"Authorization", ["Basic ", http_auth_basic(U, P)]} | Headers]
end;
_ ->
- [{"Authorization", ["Basic ", http_auth_digest(User, UPw)]} | Headers]
+ [{"Authorization", ["Basic ", http_auth_basic(User, UPw)]} | Headers]
end,
add_proxy_auth_headers(State, Headers_1).
-add_proxy_auth_headers(#state{use_http_proxy = false}, Headers) ->
+add_proxy_auth_headers(#state{use_proxy = false}, Headers) ->
Headers;
-add_proxy_auth_headers(#state{http_proxy_auth_digest = []}, Headers) ->
+add_proxy_auth_headers(#state{proxy_auth_basic = []}, Headers) ->
Headers;
-add_proxy_auth_headers(#state{http_proxy_auth_digest = Auth_digest}, Headers) ->
- [{"Proxy-Authorization", ["Basic ", Auth_digest]} | Headers].
+add_proxy_auth_headers(#state{proxy_auth_basic = Auth_basic}, Headers) ->
+ [{"Proxy-Authorization", ["Basic ", Auth_basic]} | Headers].
-http_auth_digest([], []) ->
+http_auth_basic([], []) ->
[];
-http_auth_digest(Username, Password) ->
+http_auth_basic(Username, Password) ->
ibrowse_lib:encode_base64(Username ++ [$: | Password]).
make_request(Method, Headers, AbsPath, RelPath, Body, Options,
- #state{use_http_proxy = UseHttpProxy, is_ssl = Is_ssl}, ReqId) ->
+ #state{use_proxy = UseProxy, is_ssl = Is_ssl}, ReqId) ->
HttpVsn = http_vsn_string(get_value(http_vsn, Options, {1,1})),
Fun1 = fun({X, Y}) when is_atom(X) ->
{to_lower(atom_to_list(X)), X, Y};
- ({X, Y}) when is_list(X) ->
+ ({X, Y}) when is_list(X); is_binary(X) ->
{to_lower(X), X, Y}
end,
Headers_0 = [Fun1(X) || X <- Headers],
@@ -930,7 +1073,7 @@ make_request(Method, Headers, AbsPath, RelPath, Body, Options,
Headers_2
end,
Headers_4 = cons_headers(Headers_3),
- Uri = case get_value(use_absolute_uri, Options, false) or UseHttpProxy of
+ Uri = case get_value(use_absolute_uri, Options, false) or UseProxy of
true ->
case Is_ssl of
true ->
@@ -977,7 +1120,7 @@ encode_headers(L) ->
encode_headers(L, []).
encode_headers([{http_vsn, _Val} | T], Acc) ->
encode_headers(T, Acc);
-encode_headers([{Name,Val} | T], Acc) when is_list(Name) ->
+encode_headers([{Name,Val} | T], Acc) when is_list(Name); is_binary(Name) ->
encode_headers(T, [[Name, ": ", fmt_val(Val), crnl()] | Acc]);
encode_headers([{Name,Val} | T], Acc) when is_atom(Name) ->
encode_headers(T, [[atom_to_list(Name), ": ", fmt_val(Val), crnl()] | Acc]);
@@ -1028,7 +1171,8 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
cur_req = CurReq} = State) ->
#request{from=From, stream_to=StreamTo, req_id=ReqId,
method=Method, response_format = Resp_format,
- options = Options, timer_ref = T_ref
+ options = Options, timer_ref = T_ref,
+ raw_req = Raw_req
} = CurReq,
MaxHeaderSize = ibrowse:get_config_value(max_headers_size, infinity),
case scan_header(Acc, Data) of
@@ -1038,7 +1182,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
{HttpVsn, StatCode, Headers_1, Status_line, Raw_headers} = parse_headers(Headers),
do_trace("HttpVsn: ~p StatusCode: ~p Headers_1 -> ~1000.p~n", [HttpVsn, StatCode, Headers_1]),
LCHeaders = [{to_lower(X), Y} || {X,Y} <- Headers_1],
- ConnClose = to_lower(get_value("connection", LCHeaders, "false")),
+ ConnClose = to_lower(get_header_value("connection", LCHeaders, "false")),
IsClosing = is_connection_closing(HttpVsn, ConnClose),
State_0 = case IsClosing of
true ->
@@ -1048,6 +1192,7 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
State
end,
Give_raw_headers = get_value(give_raw_headers, Options, false),
+ Give_raw_req = get_value(return_raw_request, Options, false),
State_1 = case Give_raw_headers of
true ->
State_0#state{recvd_headers=Headers_1, status=get_body,
@@ -1057,13 +1202,16 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
http_status_code=StatCode};
false ->
State_0#state{recvd_headers=Headers_1, status=get_body,
+ status_line = Status_line,
reply_buffer = <<>>,
http_status_code=StatCode}
end,
put(conn_close, ConnClose),
- TransferEncoding = to_lower(get_value("transfer-encoding", LCHeaders, "false")),
+ TransferEncodings = to_lower(get_header_value("transfer-encoding", LCHeaders, "false")),
+ IsChunked = lists:any(fun(Enc) -> string:strip(Enc) =:= "chunked" end,
+ string:tokens(TransferEncodings, ",")),
Head_response_with_body = lists:member({workaround, head_response_with_body}, Options),
- case get_value("content-length", LCHeaders, undefined) of
+ case get_header_value("content-length", LCHeaders, undefined) of
_ when Method == connect,
hd(StatCode) == $2 ->
{_, Reqs_1} = queue:out(Reqs),
@@ -1086,8 +1234,13 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
%% there was still a body. Issue #67 on Github
{_, Reqs_1} = queue:out(Reqs),
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
- State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
- {ok, StatCode, Headers_1, []}),
+ Reply = case Give_raw_req of
+ false ->
+ {ok, StatCode, Headers_1, []};
+ true ->
+ {ok, StatCode, Headers_1, [], Raw_req}
+ end,
+ State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
State_2 = reset_state(State_1_1),
State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
@@ -1106,13 +1259,18 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
%% RFC2616 - Sec 4.4
{_, Reqs_1} = queue:out(Reqs),
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
- State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
- {ok, StatCode, Headers_1, []}),
+ Reply = case Give_raw_req of
+ false ->
+ {ok, StatCode, Headers_1, []};
+ true ->
+ {ok, StatCode, Headers_1, [], Raw_req}
+ end,
+ State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
State_2 = reset_state(State_1_1),
State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
parse_response(Data_1, State_3);
- _ when TransferEncoding =:= "chunked" ->
+ _ when IsChunked ->
do_trace("Chunked encoding detected...~n",[]),
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
case parse_11_response(Data_1, State_1#state{transfer_encoding=chunked,
@@ -1129,7 +1287,26 @@ parse_response(Data, #state{reply_buffer = Acc, reqs = Reqs,
undefined when HttpVsn =:= "HTTP/1.0";
ConnClose =:= "close" ->
send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
- State_1#state{reply_buffer = Data_1};
+ accumulate_response(Data_1, State_1);
+ undefined when StatCode =:= "303" ->
+ %% Some servers send 303 requests without a body.
+ %% RFC2616 says that they SHOULD, but they dont.
+ case ibrowse:get_config_value(allow_303_with_no_body, false) of
+ false ->
+ fail_pipelined_requests(State_1,
+ {error, {content_length_undefined,
+ {stat_code, StatCode}, Headers}}),
+ {error, content_length_undefined};
+ true ->
+ {_, Reqs_1} = queue:out(Reqs),
+ send_async_headers(ReqId, StreamTo, Give_raw_headers, State_1),
+ State_1_1 = do_reply(State_1, From, StreamTo, ReqId, Resp_format,
+ {ok, StatCode, Headers_1, []}),
+ cancel_timer(T_ref, {eat_message, {req_timedout, From}}),
+ State_2 = reset_state(State_1_1),
+ State_3 = set_cur_request(State_2#state{reqs = Reqs_1}),
+ parse_response(Data_1, State_3)
+ end;
undefined ->
fail_pipelined_requests(State_1,
{error, {content_length_undefined,
@@ -1283,22 +1460,41 @@ parse_11_response(DataRecvd,
#state{transfer_encoding = chunked,
chunk_size = CSz,
recvd_chunk_size = Recvd_csz,
- rep_buf_size = RepBufSz} = State) ->
+ reply_buffer = RepBuf,
+ rep_buf_size = RepBufSz,
+ streamed_size = Streamed_size,
+ cur_req = CurReq} = State) ->
NeedBytes = CSz - Recvd_csz,
DataLen = size(DataRecvd),
do_trace("Recvd more data: size: ~p. NeedBytes: ~p~n", [DataLen, NeedBytes]),
case DataLen >= NeedBytes of
true ->
{RemChunk, RemData} = split_binary(DataRecvd, NeedBytes),
- do_trace("Recvd another chunk...~p~n", [RemChunk]),
- do_trace("RemData -> ~p~n", [RemData]),
- case accumulate_response(RemChunk, State) of
- {error, Reason} ->
- do_trace("Error accumulating response --> ~p~n", [Reason]),
- {error, Reason};
- #state{} = State_1 ->
- State_2 = State_1#state{chunk_size=tbd},
- parse_11_response(RemData, State_2)
+ case CurReq of
+ #request{stream_to = StreamTo, caller_controls_socket = false, req_id = ReqId, stream_full_chunks = true, response_format = Response_format} ->
+ Chunk = <<RepBuf/binary, RemChunk/binary>>,
+ do_trace("Recvd another chunk...~p~n", [Chunk]),
+ do_trace("RemData -> ~p~n", [RemData]),
+ do_interim_reply(StreamTo, Response_format, ReqId, Chunk),
+ State_1 = State#state{
+ reply_buffer = <<>>,
+ rep_buf_size = RepBufSz + size(RemChunk),
+ interim_reply_sent = true,
+ streamed_size = Streamed_size + CSz,
+ chunk_size = tbd,
+ recvd_chunk_size = 0},
+ parse_11_response(RemData, State_1);
+ _ ->
+ do_trace("Recvd another chunk...~p~n", [RemChunk]),
+ do_trace("RemData -> ~p~n", [RemData]),
+ case accumulate_response(RemChunk, State) of
+ {error, Reason} ->
+ do_trace("Error accumulating response --> ~p~n", [Reason]),
+ {error, Reason};
+ #state{} = State_1 ->
+ State_2 = State_1#state{chunk_size=tbd},
+ parse_11_response(RemData, State_2)
+ end
end;
false ->
accumulate_response(DataRecvd,
@@ -1335,7 +1531,8 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
tmp_file_name = TmpFilename,
tmp_file_fd = Fd,
options = Options,
- timer_ref = ReqTimer
+ timer_ref = ReqTimer,
+ raw_req = Raw_req
},
#state{http_status_code = SCode,
status_line = Status_line,
@@ -1355,19 +1552,26 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
_ ->
{file, TmpFilename}
end,
- {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(RespHeaders, Raw_headers, Options),
+ {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Status_line, RespHeaders, Raw_headers, Options),
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false ->
{ok, Status_line, Raw_headers_1, ResponseBody};
+ true ->
+ {ok, Status_line, Raw_headers_1, ResponseBody, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SCode, Resp_headers_1, ResponseBody};
false ->
- {ok, SCode, Resp_headers_1, ResponseBody}
+ {ok, SCode, Resp_headers_1, ResponseBody, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
set_cur_request(State_1);
handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
response_format = Resp_format,
- options = Options, timer_ref = ReqTimer},
+ options = Options, timer_ref = ReqTimer,
+ raw_req = Raw_req
+ },
#state{http_status_code = SCode,
status_line = Status_line,
raw_headers = Raw_headers,
@@ -1375,12 +1579,17 @@ handle_response(#request{from=From, stream_to=StreamTo, req_id=ReqId,
reply_buffer = RepBuf
} = State) ->
Body = RepBuf,
- {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Resp_headers, Raw_headers, Options),
+ {Resp_headers_1, Raw_headers_1} = maybe_add_custom_headers(Status_line, Resp_headers, Raw_headers, Options),
+ Give_raw_req = get_value(return_raw_request, Options, false),
Reply = case get_value(give_raw_headers, Options, false) of
- true ->
+ true when Give_raw_req == false ->
{ok, Status_line, Raw_headers_1, Body};
+ true ->
+ {ok, Status_line, Raw_headers_1, Body, Raw_req};
+ false when Give_raw_req == false ->
+ {ok, SCode, Resp_headers_1, Body};
false ->
- {ok, SCode, Resp_headers_1, Body}
+ {ok, SCode, Resp_headers_1, Body, Raw_req}
end,
State_1 = do_reply(State, From, StreamTo, ReqId, Resp_format, Reply),
cancel_timer(ReqTimer, {eat_message, {req_timedout, From}}),
@@ -1407,12 +1616,8 @@ set_cur_request(#state{reqs = Reqs, socket = Socket} = State) ->
empty ->
State#state{cur_req = undefined};
{value, #request{caller_controls_socket = Ccs} = NextReq} ->
- case Ccs of
- true ->
- do_setopts(Socket, [{active, once}], State);
- _ ->
- ok
- end,
+ _ = Ccs =:= true
+ andalso do_setopts(Socket, [{active, once}], State),
State#state{cur_req = NextReq}
end.
@@ -1586,6 +1791,7 @@ get_crlf_pos(<<>>, _) -> no.
fmt_val(L) when is_list(L) -> L;
fmt_val(I) when is_integer(I) -> integer_to_list(I);
fmt_val(A) when is_atom(A) -> atom_to_list(A);
+fmt_val(B) when is_binary(B) -> B;
fmt_val(Term) -> io_lib:format("~p", [Term]).
crnl() -> "\r\n".
@@ -1678,7 +1884,7 @@ send_async_headers(ReqId, StreamTo, Give_raw_headers,
recvd_headers = Headers, http_status_code = StatCode,
cur_req = #request{options = Opts}
}) ->
- {Headers_1, Raw_headers_1} = maybe_add_custom_headers(Headers, Raw_headers, Opts),
+ {Headers_1, Raw_headers_1} = maybe_add_custom_headers(Status_line, Headers, Raw_headers, Opts),
case Give_raw_headers of
false ->
catch StreamTo ! {ibrowse_async_headers, ReqId, StatCode, Headers_1};
@@ -1686,7 +1892,7 @@ send_async_headers(ReqId, StreamTo, Give_raw_headers,
catch StreamTo ! {ibrowse_async_headers, ReqId, Status_line, Raw_headers_1}
end.
-maybe_add_custom_headers(Headers, Raw_headers, Opts) ->
+maybe_add_custom_headers(Status_line, Headers, Raw_headers, Opts) ->
Custom_headers = get_value(add_custom_headers, Opts, []),
Headers_1 = Headers ++ Custom_headers,
Raw_headers_1 = case Custom_headers of
@@ -1696,7 +1902,12 @@ maybe_add_custom_headers(Headers, Raw_headers, Opts) ->
_ ->
Raw_headers
end,
- {Headers_1, Raw_headers_1}.
+ case get_value(preserve_status_line, Opts, false) of
+ true ->
+ {[{ibrowse_status_line, Status_line} | Headers_1], Raw_headers_1};
+ false ->
+ {Headers_1, Raw_headers_1}
+ end.
format_response_data(Resp_format, Body) ->
case Resp_format of
@@ -1712,6 +1923,11 @@ format_response_data(Resp_format, Body) ->
Body
end.
+%% dont message an unexisting server
+%% triggered by :stop or :tcp_closed on an unactive connection
+do_reply(State, undefined, undefined, _, _, _Msg) ->
+ dec_pipeline_counter(State);
+
do_reply(State, From, undefined, _, Resp_format, {ok, St_code, Headers, Body}) ->
Msg_1 = {ok, St_code, Headers, format_response_data(Resp_format, Body)},
gen_server:reply(From, Msg_1),
@@ -1828,8 +2044,15 @@ cancel_timer(Ref, {eat_message, Msg}) ->
end.
make_req_id() ->
- now().
+ case catch erlang:unique_integer() of
+ {'EXIT', _} ->
+ erlang:apply(erlang, now, []);
+ V ->
+ V
+ end.
+to_lower(Str) when is_binary(Str) ->
+ to_lower(binary_to_list(Str));
to_lower(Str) ->
to_lower(Str, []).
to_lower([H|T], Acc) when H >= $A, H =< $Z ->
@@ -1843,34 +2066,29 @@ shutting_down(#state{lb_ets_tid = undefined}) ->
ok;
shutting_down(#state{lb_ets_tid = Tid,
cur_pipeline_size = _Sz}) ->
- catch ets:delete(Tid, self()).
+ (catch ets:select_delete(Tid, [{{{'_', '_', '$1'},'_'},[{'==','$1',{const,self()}}],[true]}])).
inc_pipeline_counter(#state{is_closing = true} = State) ->
State;
inc_pipeline_counter(#state{lb_ets_tid = undefined} = State) ->
State;
-inc_pipeline_counter(#state{cur_pipeline_size = Pipe_sz,
- lb_ets_tid = Tid} = State) ->
- update_counter(Tid, self(), {2,1,99999,9999}),
+inc_pipeline_counter(#state{cur_pipeline_size = Pipe_sz} = State) ->
State#state{cur_pipeline_size = Pipe_sz + 1}.
-update_counter(Tid, Key, Args) ->
- ets:update_counter(Tid, Key, Args).
-
-dec_pipeline_counter(#state{is_closing = true} = State) ->
- State;
-dec_pipeline_counter(#state{lb_ets_tid = undefined} = State) ->
- State;
dec_pipeline_counter(#state{cur_pipeline_size = Pipe_sz,
- lb_ets_tid = Tid} = State) ->
- try
- update_counter(Tid, self(), {2,-1,0,0}),
- update_counter(Tid, self(), {3,-1,0,0})
- catch
- _:_ ->
- ok
- end,
- State#state{cur_pipeline_size = Pipe_sz - 1}.
+ lb_ets_tid = Tid,
+ proc_state = Proc_state} = State) when Tid /= undefined,
+ Proc_state /= ?dead_proc_walking ->
+ Ts = os:timestamp(),
+ catch ets:insert(Tid, {{Pipe_sz - 1, os:timestamp(), self()}, []}),
+ (catch ets:select_delete(Tid, [{{{'_', '$2', '$1'},'_'},
+ [{'==', '$1', {const,self()}},
+ {'<', '$2', {const,Ts}}
+ ],
+ [true]}])),
+ State#state{cur_pipeline_size = Pipe_sz - 1};
+dec_pipeline_counter(State) ->
+ State.
flatten([H | _] = L) when is_integer(H) ->
L;
@@ -1880,11 +2098,16 @@ flatten([]) ->
[].
get_stream_chunk_size(Options) ->
- case lists:keysearch(stream_chunk_size, 1, Options) of
- {value, {_, V}} when V > 0 ->
- V;
+ case get_value(stream_full_chunks, Options, false) of
+ true ->
+ infinity;
_ ->
- ?DEFAULT_STREAM_CHUNK_SIZE
+ case lists:keysearch(stream_chunk_size, 1, Options) of
+ {value, {_, V}} when V > 0 ->
+ V;
+ _ ->
+ ?DEFAULT_STREAM_CHUNK_SIZE
+ end
end.
set_inac_timer(State) ->
@@ -1941,5 +2164,20 @@ trace_request_body(Body) ->
ok
end.
-to_binary(X) when is_list(X) -> list_to_binary(X);
-to_binary(X) when is_binary(X) -> X.
+to_binary({X, _}) when is_function(X) -> to_binary(X);
+to_binary(X) when is_function(X) -> <<"body generated by function">>;
+to_binary(X) when is_list(X) -> list_to_binary(X);
+to_binary(X) when is_binary(X) -> X.
+
+get_header_value(Name, Headers, Default_val) ->
+ case lists:keysearch(Name, 1, Headers) of
+ false ->
+ Default_val;
+ {value, {_, Val}} when is_binary(Val) ->
+ binary_to_list(Val);
+ {value, {_, Val}} ->
+ Val
+ end.
+
+delayed_stop_timer() ->
+ erlang:send_after(500, self(), delayed_stop).
diff --git a/src/ibrowse_lb.erl b/src/ibrowse_lb.erl
index d98cf32..894d8ad 100644
--- a/src/ibrowse_lb.erl
+++ b/src/ibrowse_lb.erl
@@ -16,7 +16,7 @@
%% External exports
-export([
start_link/1,
- spawn_connection/5,
+ spawn_connection/6,
stop/1
]).
@@ -36,7 +36,6 @@
port,
max_sessions,
max_pipeline_size,
- num_cur_sessions = 0,
proc_state
}).
@@ -70,24 +69,25 @@ init([Host, Port]) ->
Max_pipe_sz = ibrowse:get_config_value({max_pipeline_size, Host, Port}, 10),
put(my_trace_flag, ibrowse_lib:get_trace_status(Host, Port)),
put(ibrowse_trace_token, ["LB: ", Host, $:, integer_to_list(Port)]),
- Tid = ets:new(ibrowse_lb, [public, ordered_set]),
- {ok, #state{parent_pid = whereis(ibrowse),
+ State = #state{parent_pid = whereis(ibrowse),
host = Host,
port = Port,
- ets_tid = Tid,
max_pipeline_size = Max_pipe_sz,
- max_sessions = Max_sessions}}.
+ max_sessions = Max_sessions},
+ State_1 = maybe_create_ets(State),
+ {ok, State_1}.
spawn_connection(Lb_pid, Url,
Max_sessions,
Max_pipeline_size,
- SSL_options)
+ SSL_options,
+ Process_options)
when is_pid(Lb_pid),
is_record(Url, url),
is_integer(Max_pipeline_size),
is_integer(Max_sessions) ->
gen_server:call(Lb_pid,
- {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options}).
+ {spawn_connection, Url, Max_sessions, Max_pipeline_size, SSL_options, Process_options}).
stop(Lb_pid) ->
case catch gen_server:call(Lb_pid, stop) of
@@ -112,34 +112,30 @@ handle_call(stop, _From, #state{ets_tid = undefined} = State) ->
{stop, normal, State};
handle_call(stop, _From, #state{ets_tid = Tid} = State) ->
- ets:foldl(fun({Pid, _, _}, Acc) ->
- ibrowse_http_client:stop(Pid),
- Acc
- end, [], Tid),
+ stop_all_conn_procs(Tid),
gen_server:reply(_From, ok),
{stop, normal, State};
handle_call(_, _From, #state{proc_state = shutting_down} = State) ->
{reply, {error, shutting_down}, State};
-%% Update max_sessions in #state with supplied value
-handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From,
- #state{num_cur_sessions = Num} = State)
- when Num >= Max_sess ->
- State_1 = maybe_create_ets(State),
- Reply = find_best_connection(State_1#state.ets_tid, Max_pipe),
- {reply, Reply, State_1#state{max_sessions = Max_sess,
- max_pipeline_size = Max_pipe}};
-
-handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options}, _From,
- #state{num_cur_sessions = Cur} = State) ->
- State_1 = maybe_create_ets(State),
- Tid = State_1#state.ets_tid,
- {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}),
- ets:insert(Tid, {Pid, 0, 0}),
- {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1,
- max_sessions = Max_sess,
- max_pipeline_size = Max_pipe}};
+handle_call({spawn_connection, Url, Max_sess, Max_pipe, SSL_options, Process_options}, _From,
+ State) ->
+ State_1 = maybe_create_ets(State),
+ Tid = State_1#state.ets_tid,
+ Tid_size = ets:info(Tid, size),
+ case Tid_size >= Max_sess of
+ true ->
+ Reply = find_best_connection(Tid, Max_pipe),
+ {reply, Reply, State_1#state{max_sessions = Max_sess,
+ max_pipeline_size = Max_pipe}};
+ false ->
+ {ok, Pid} = ibrowse_http_client:start({Tid, Url, SSL_options}, Process_options),
+ Ts = os:timestamp(),
+ ets:insert(Tid, {{1, Ts, Pid}, []}),
+ {reply, {ok, {1, Ts, Pid}}, State_1#state{max_sessions = Max_sess,
+ max_pipeline_size = Max_pipe}}
+ end;
handle_call(Request, _From, State) ->
Reply = {unknown_request, Request},
@@ -162,24 +158,6 @@ handle_cast(_Msg, State) ->
%% {noreply, State, Timeout} |
%% {stop, Reason, State} (terminate/2 is called)
%%--------------------------------------------------------------------
-handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) ->
- {stop, normal, State};
-
-handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) ->
- {noreply, State};
-
-handle_info({'EXIT', Pid, _Reason},
- #state{num_cur_sessions = Cur,
- ets_tid = Tid} = State) ->
- ets:match_delete(Tid, {{'_', Pid}, '_'}),
- Cur_1 = Cur - 1,
- case Cur_1 of
- 0 ->
- ets:delete(Tid),
- {noreply, State#state{ets_tid = undefined, num_cur_sessions = 0}, 10000};
- _ ->
- {noreply, State#state{num_cur_sessions = Cur_1}}
- end;
handle_info({trace, Bool}, #state{ets_tid = undefined} = State) ->
put(my_trace_flag, Bool),
@@ -215,9 +193,17 @@ handle_info(_Info, State) ->
%% Description: Shutdown the server
%% Returns: any (ignored by gen_server)
%%--------------------------------------------------------------------
-terminate(_Reason, _State) ->
+terminate(_Reason, #state{host = Host, port = Port, ets_tid = Tid} = _State) ->
+ catch ets:delete(ibrowse_lb, {Host, Port}),
+ stop_all_conn_procs(Tid),
ok.
+stop_all_conn_procs(Tid) ->
+ ets:foldl(fun({{_, _, Pid}, _}, Acc) ->
+ ibrowse_http_client:stop(Pid),
+ Acc
+ end, [], Tid).
+
%%--------------------------------------------------------------------
%% Func: code_change/3
%% Purpose: Convert process state when code is changed
@@ -230,23 +216,18 @@ code_change(_OldVsn, State, _Extra) ->
%%% Internal functions
%%--------------------------------------------------------------------
find_best_connection(Tid, Max_pipe) ->
- Res = find_best_connection(ets:first(Tid), Tid, Max_pipe),
- Res.
-
-find_best_connection('$end_of_table', _, _) ->
- {error, retry_later};
-find_best_connection(Pid, Tid, Max_pipe) ->
- case ets:lookup(Tid, Pid) of
- [{Pid, Cur_sz, Speculative_sz}] when Cur_sz < Max_pipe,
- Speculative_sz < Max_pipe ->
- ets:update_counter(Tid, Pid, {3, 1, 9999999, 9999999}),
- {ok, Pid};
+ case ets:first(Tid) of
+ {Spec_size, Ts, Pid} = First when Spec_size < Max_pipe ->
+ ets:delete(Tid, First),
+ ets:insert(Tid, {{Spec_size + 1, Ts, Pid}, []}),
+ {ok, First};
_ ->
- find_best_connection(ets:next(Tid, Pid), Tid, Max_pipe)
+ {error, retry_later}
end.
-maybe_create_ets(#state{ets_tid = undefined} = State) ->
+maybe_create_ets(#state{ets_tid = undefined, host = Host, port = Port} = State) ->
Tid = ets:new(ibrowse_lb, [public, ordered_set]),
+ ets:insert(ibrowse_lb, #lb_pid{host_port = {Host, Port}, pid = self(), ets_tid = Tid}),
State#state{ets_tid = Tid};
maybe_create_ets(State) ->
State.
diff --git a/src/ibrowse_lib.erl b/src/ibrowse_lib.erl
index 7b12cb3..6c1883d 100644
--- a/src/ibrowse_lib.erl
+++ b/src/ibrowse_lib.erl
@@ -20,6 +20,7 @@
get_trace_status/2,
do_trace/2,
do_trace/3,
+ log_msg/2,
url_encode/1,
decode_rfc822_date/1,
status_code/1,
@@ -28,7 +29,8 @@
get_value/2,
get_value/3,
parse_url/1,
- printable_date/0
+ printable_date/0,
+ printable_date/1
]).
get_trace_status(Host, Port) ->
@@ -363,13 +365,16 @@ parse_url([], State, Url, TmpAcc) ->
{invalid_uri_2, State, Url, TmpAcc}.
default_port(socks5) -> 1080;
-default_port(http) -> 80;
-default_port(https) -> 443;
-default_port(ftp) -> 21.
+default_port(http) -> 80;
+default_port(https) -> 443;
+default_port(ftp) -> 21.
printable_date() ->
- {{Y,Mo,D},{H, M, S}} = calendar:local_time(),
- {_,_,MicroSecs} = now(),
+ printable_date(os:timestamp()).
+
+printable_date(Now) ->
+ {{Y,Mo,D},{H, M, S}} = calendar:now_to_local_time(Now),
+ {_,_,MicroSecs} = Now,
[integer_to_list(Y),
$-,
integer_to_list(Mo),
@@ -394,13 +399,31 @@ do_trace(_, Fmt, Args) ->
get(ibrowse_trace_token) | Args]).
-else.
do_trace(true, Fmt, Args) ->
- io:format("~s -- (~s) - "++Fmt,
- [printable_date(),
- get(ibrowse_trace_token) | Args]);
+ Fmt_1 = "~s -- (~s) - "++Fmt,
+ Args_1 = [printable_date(),
+ get(ibrowse_trace_token) | Args],
+ case application:get_env(ibrowse, logger_mf) of
+ {ok, {M, F}} ->
+ log_msg(M, F, Fmt_1, Args_1);
+ _ ->
+ log_msg(io, format, Fmt_1, Args_1)
+ end;
do_trace(_, _, _) ->
ok.
-endif.
+log_msg(Fmt, Args) ->
+ case application:get_env(ibrowse, logger_mf) of
+ {ok, {M, F}} ->
+ log_msg(M, F, Fmt, Args),
+ ok;
+ _ ->
+ ok
+ end.
+
+log_msg(M, F, Fmt, Args) ->
+ catch apply(M, F, [Fmt, Args]).
+
-ifdef(EUNIT).
parse_url_test() ->
diff --git a/src/ibrowse_socks5.erl b/src/ibrowse_socks5.erl
index d00df44..2f0d1fc 100644
--- a/src/ibrowse_socks5.erl
+++ b/src/ibrowse_socks5.erl
@@ -30,16 +30,16 @@
-import(ibrowse_lib, [get_value/2, get_value/3]).
-connect(TargetHost, TargetPort, ProxyOptions, Options, Timeout) ->
- case gen_tcp:connect(get_value(host, ProxyOptions),
- get_value(port, ProxyOptions),
- Options, Timeout) of
+connect(Host, Port, Options, SockOptions, Timeout) ->
+ Socks5Host = get_value(socks5_host, Options),
+ Socks5Port = get_value(socks5_port, Options),
+ case gen_tcp:connect(Socks5Host, Socks5Port, SockOptions, Timeout) of
{ok, Socket} ->
case handshake(Socket, Options) of
ok ->
- case connect(TargetHost, TargetPort, Socket) of
+ case connect(Host, Port, Socket) of
ok ->
- maybe_ssl(Socket, ProxyOptions, Timeout);
+ {ok, Socket};
Else ->
gen_tcp:close(Socket),
Else
@@ -52,19 +52,30 @@ connect(TargetHost, TargetPort, ProxyOptions, Options, Timeout) ->
Else
end.
-handshake(Socket, ProxyOptions) when is_port(Socket) ->
- {Handshake, Success} = case get_value(user, ProxyOptions, <<>>) of
- <<>> ->
- {<<?VERSION, 1, ?NO_AUTH>>, ?NO_AUTH};
- User ->
- Password = get_value(password, ProxyOptions, <<>>),
- {<<?VERSION, 1, ?USERPASS, (byte_size(User)), User,
- (byte_size(Password)), Password>>, ?USERPASS}
- end,
- ok = gen_tcp:send(Socket, Handshake),
- case gen_tcp:recv(Socket, 0) of
- {ok, <<?VERSION, Success>>} ->
+handshake(Socket, Options) when is_port(Socket) ->
+ User = get_value(socks5_user, Options, <<>>),
+ Handshake_msg = case User of
+ <<>> ->
+ <<?VERSION, 1, ?NO_AUTH>>;
+ User ->
+ <<?VERSION, 1, ?USERPASS>>
+ end,
+ ok = gen_tcp:send(Socket, Handshake_msg),
+ case gen_tcp:recv(Socket, 2) of
+ {ok, <<?VERSION, ?NO_AUTH>>} ->
ok;
+ {ok, <<?VERSION, ?USERPASS>>} ->
+ Password = get_value(socks5_password, Options, <<>>),
+ Auth_msg = list_to_binary([1,
+ iolist_size(User), User,
+ iolist_size(Password), Password]),
+ ok = gen_tcp:send(Socket, Auth_msg),
+ case gen_tcp:recv(Socket, 2) of
+ {ok, <<1, ?SUCCEEDED>>} ->
+ ok;
+ _ ->
+ {error, unacceptable}
+ end;
{ok, <<?VERSION, ?UNACCEPTABLE>>} ->
{error, unacceptable};
{error, Reason} ->
@@ -75,10 +86,19 @@ connect(Host, Port, Via) when is_list(Host) ->
connect(list_to_binary(Host), Port, Via);
connect(Host, Port, Via) when is_binary(Host), is_integer(Port),
is_port(Via) ->
+ {AddressType, Address} = case inet:parse_address(binary_to_list(Host)) of
+ {ok, {IP1, IP2, IP3, IP4}} ->
+ {?ATYP_IPV4, <<IP1,IP2,IP3,IP4>>};
+ {ok, {IP1, IP2, IP3, IP4, IP5, IP6, IP7, IP8}} ->
+ {?ATYP_IPV6, <<IP1,IP2,IP3,IP4,IP5,IP6,IP7,IP8>>};
+ _ ->
+ HostLength = byte_size(Host),
+ {?ATYP_DOMAINNAME, <<HostLength,Host/binary>>}
+ end,
ok = gen_tcp:send(Via,
- <<?VERSION, ?CONNECT, ?RESERVED, ?ATYP_DOMAINNAME,
- (byte_size(Host)), Host/binary,
- (Port):16>>),
+ <<?VERSION, ?CONNECT, ?RESERVED,
+ AddressType, Address/binary,
+ (Port):16>>),
case gen_tcp:recv(Via, 0) of
{ok, <<?VERSION, ?SUCCEEDED, ?RESERVED, _/binary>>} ->
ok;
@@ -88,16 +108,6 @@ connect(Host, Port, Via) when is_binary(Host), is_integer(Port),
{error, Reason}
end.
-maybe_ssl(Socket, ProxyOptions, Timeout) ->
- IsSsl = get_value(is_ssl, ProxyOptions, false),
- SslOpts = get_value(ssl_opts, ProxyOptions, []),
- case IsSsl of
- false ->
- {ok, Socket};
- true ->
- ssl:connect(Socket, SslOpts, Timeout)
- end.
-
rep(0) -> succeeded;
rep(1) -> server_fail;
rep(2) -> disallowed_by_ruleset;
diff --git a/test/Makefile b/test/Makefile
new file mode 100644
index 0000000..50a292c
--- /dev/null
+++ b/test/Makefile
@@ -0,0 +1,16 @@
+IBROWSE_EBIN_PATH=../../ibrowse/ebin
+
+compile:
+ @erl -pa $(IBROWSE_EBIN_PATH) -make
+
+test: compile
+ @erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test local_unit_tests -s erlang halt
+
+old_tests: compile
+ @erl -noshell -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test unit_tests -s erlang halt
+
+test_shell: compile
+ erl -boot start_clean -pa $(IBROWSE_EBIN_PATH) -s ibrowse_test_server start_server
+
+clean:
+ @rm -f *.beam
diff --git a/test/ibrowse_lib_tests.erl b/test/ibrowse_lib_tests.erl
new file mode 100644
index 0000000..cc3bf71
--- /dev/null
+++ b/test/ibrowse_lib_tests.erl
@@ -0,0 +1,135 @@
+%%% File : ibrowse_lib.erl
+%%% Authors : Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>,
+%%% Filipe David Manana <fdmanana@apache.org>
+%%% Description : Tests for the module ibrowse_lib.erl
+%%% Created : 12 April 2011 by Filipe David Manana <fdmanana@apache.org>
+
+-module(ibrowse_lib_tests).
+-include_lib("eunit/include/eunit.hrl").
+-include("../include/ibrowse.hrl").
+
+
+parse_urls_test_() ->
+ {timeout, 60, [fun parse_urls/0]}.
+
+
+parse_urls() ->
+ ?assertMatch(#url{
+ abspath = "http://localhost",
+ host = "localhost",
+ host_type = hostname,
+ port = 80,
+ path = "/",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://localhost")),
+ ?assertMatch(#url{
+ abspath = "http://localhost:80/",
+ host = "localhost",
+ host_type = hostname,
+ port = 80,
+ path = "/",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://localhost:80/")),
+ ?assertMatch(#url{
+ abspath = "http://127.0.0.1:8000/",
+ host = "127.0.0.1",
+ host_type = ipv4_address,
+ port = 8000,
+ path = "/",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://127.0.0.1:8000/")),
+ ?assertMatch(#url{
+ abspath = "https://foo:bar@127.0.0.1:8000/test",
+ host = "127.0.0.1",
+ host_type = ipv4_address,
+ port = 8000,
+ path = "/test",
+ username = "foo",
+ password = "bar",
+ protocol = https
+ },
+ ibrowse_lib:parse_url("https://foo:bar@127.0.0.1:8000/test")),
+ ?assertMatch(#url{
+ abspath = "https://[::1]",
+ host = "::1",
+ host_type = ipv6_address,
+ port = 443,
+ path = "/",
+ username = undefined,
+ password = undefined,
+ protocol = https
+ },
+ ibrowse_lib:parse_url("https://[::1]")),
+ ?assertMatch(#url{
+ abspath = "http://[::1]:8080",
+ host = "::1",
+ host_type = ipv6_address,
+ port = 8080,
+ path = "/",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://[::1]:8080")),
+ ?assertMatch(#url{
+ abspath = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8081/index.html",
+ host = "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210",
+ host_type = ipv6_address,
+ port = 8081,
+ path = "/index.html",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8081/index.html")),
+ ?assertMatch(#url{
+ abspath = "http://[1080:0:0:0:8:800:200C:417A]/foo/bar",
+ host = "1080:0:0:0:8:800:200C:417A",
+ host_type = ipv6_address,
+ port = 80,
+ path = "/foo/bar",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://[1080:0:0:0:8:800:200C:417A]/foo/bar")),
+ ?assertMatch(#url{
+ abspath = "http://[1080:0:0:0:8:800:200C:417A]:8080/foo/bar",
+ host = "1080:0:0:0:8:800:200C:417A",
+ host_type = ipv6_address,
+ port = 8080,
+ path = "/foo/bar",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://[1080:0:0:0:8:800:200C:417A]:8080/foo/bar")),
+ ?assertMatch(#url{
+ abspath = "http://[::192.9.5.5]:6000/foo?q=bar",
+ host = "::192.9.5.5",
+ host_type = ipv6_address,
+ port = 6000,
+ path = "/foo?q=bar",
+ username = undefined,
+ password = undefined,
+ protocol = http
+ },
+ ibrowse_lib:parse_url("http://[::192.9.5.5]:6000/foo?q=bar")),
+ ?assertMatch({error, invalid_uri},
+ ibrowse_lib:parse_url("http://[:1080:0:0:0:8:800:200C:417A:]:6000/foo?q=bar")),
+ ?assertMatch({error, invalid_uri},
+ ibrowse_lib:parse_url("http://[12::z]")),
+ ?assertMatch({error, invalid_uri},
+ ibrowse_lib:parse_url("http://foo[1080:0:0:0:8:800:200C:417A]:6000")),
+ ?assertMatch({error, invalid_uri},
+ ibrowse_lib:parse_url("http://foo:[1080:0:0:0:8:800:200C:417A]:6000")),
+ ok.
diff --git a/test/ibrowse_load_test.erl b/test/ibrowse_load_test.erl
new file mode 100644
index 0000000..076c46b
--- /dev/null
+++ b/test/ibrowse_load_test.erl
@@ -0,0 +1,209 @@
+-module(ibrowse_load_test).
+%%-compile(export_all).
+-export([
+ random_seed/0,
+ start/3,
+ query_state/0,
+ shutdown/0,
+ start_1/3,
+ calculate_timings/0,
+ get_mmv/2,
+ spawn_workers/2,
+ spawn_workers/4,
+ wait_for_workers/1,
+ worker_loop/2,
+ update_unknown_counter/2
+ ]).
+
+-ifdef(new_rand).
+
+-define(RAND, rand).
+random_seed() ->
+ ok.
+
+-else.
+
+-define(RAND, random).
+random_seed() ->
+ random:seed(os:timestamp()).
+
+-endif.
+
+-define(ibrowse_load_test_counters, ibrowse_load_test_counters).
+
+start(Num_workers, Num_requests, Max_sess) ->
+ proc_lib:spawn(fun() ->
+ start_1(Num_workers, Num_requests, Max_sess)
+ end).
+
+query_state() ->
+ ibrowse_load_test ! query_state.
+
+shutdown() ->
+ ibrowse_load_test ! shutdown.
+
+start_1(Num_workers, Num_requests, Max_sess) ->
+ register(ibrowse_load_test, self()),
+ application:start(ibrowse),
+ application:set_env(ibrowse, inactivity_timeout, 5000),
+ Ulimit = os:cmd("ulimit -n"),
+ case catch list_to_integer(string:strip(Ulimit, right, $\n)) of
+ X when is_integer(X), X > 3000 ->
+ ok;
+ X ->
+ io:format("Load test not starting. {insufficient_value_for_ulimit, ~p}~n", [X]),
+ exit({insufficient_value_for_ulimit, X})
+ end,
+ ets:new(?ibrowse_load_test_counters, [named_table, public]),
+ ets:new(ibrowse_load_timings, [named_table, public]),
+ try
+ ets:insert(?ibrowse_load_test_counters, [{success, 0},
+ {failed, 0},
+ {timeout, 0},
+ {retry_later, 0},
+ {one_request_only, 0}
+ ]),
+ ibrowse:set_max_sessions("localhost", 8081, Max_sess),
+ Start_time = os:timestamp(),
+ Workers = spawn_workers(Num_workers, Num_requests),
+ erlang:send_after(1000, self(), print_diagnostics),
+ ok = wait_for_workers(Workers),
+ End_time = os:timestamp(),
+ Time_in_secs = trunc(round(timer:now_diff(End_time, Start_time) / 1000000)),
+ Req_count = Num_workers * Num_requests,
+ [{_, Success_count}] = ets:lookup(?ibrowse_load_test_counters, success),
+ case Success_count == Req_count of
+ true ->
+ io:format("Test success. All requests succeeded~n", []);
+ false when Success_count > 0 ->
+ io:format("Test failed. Some successes~n", []);
+ false ->
+ io:format("Test failed. ALL requests FAILED~n", [])
+ end,
+ case Time_in_secs > 0 of
+ true ->
+ io:format("Reqs/sec achieved : ~p~n", [trunc(round(Success_count / Time_in_secs))]);
+ false ->
+ ok
+ end,
+ io:format("Load test results:~n~p~n", [ets:tab2list(?ibrowse_load_test_counters)]),
+ io:format("Timings: ~p~n", [calculate_timings()])
+ catch Err ->
+ io:format("Err: ~p~n", [Err])
+ after
+ ets:delete(?ibrowse_load_test_counters),
+ ets:delete(ibrowse_load_timings),
+ unregister(ibrowse_load_test)
+ end.
+
+calculate_timings() ->
+ {Max, Min, Mean} = get_mmv(ets:first(ibrowse_load_timings), {0, 9999999, 0}),
+ Variance = trunc(round(ets:foldl(fun({_, X}, X_acc) ->
+ (X - Mean)*(X-Mean) + X_acc
+ end, 0, ibrowse_load_timings) / ets:info(ibrowse_load_timings, size))),
+ Std_dev = trunc(round(math:sqrt(Variance))),
+ {ok, [{max, Max},
+ {min, Min},
+ {mean, Mean},
+ {variance, Variance},
+ {standard_deviation, Std_dev}]}.
+
+get_mmv('$end_of_table', {Max, Min, Total}) ->
+ Mean = trunc(round(Total / ets:info(ibrowse_load_timings, size))),
+ {Max, Min, Mean};
+get_mmv(Key, {Max, Min, Total}) ->
+ [{_, V}] = ets:lookup(ibrowse_load_timings, Key),
+ get_mmv(ets:next(ibrowse_load_timings, Key), {max(Max, V), min(Min, V), Total + V}).
+
+
+spawn_workers(Num_w, Num_r) ->
+ spawn_workers(Num_w, Num_r, self(), []).
+
+spawn_workers(0, _Num_requests, _Parent, Acc) ->
+ lists:reverse(Acc);
+spawn_workers(Num_workers, Num_requests, Parent, Acc) ->
+ Pid_ref = spawn_monitor(fun() ->
+ random_seed(),
+ case catch worker_loop(Parent, Num_requests) of
+ {'EXIT', Rsn} ->
+ io:format("Worker crashed with reason: ~p~n", [Rsn]);
+ _ ->
+ ok
+ end
+ end),
+ spawn_workers(Num_workers - 1, Num_requests, Parent, [Pid_ref | Acc]).
+
+wait_for_workers([]) ->
+ ok;
+wait_for_workers([{Pid, Pid_ref} | T] = Pids) ->
+ receive
+ {done, Pid} ->
+ wait_for_workers(T);
+ {done, Some_pid} ->
+ wait_for_workers([{Pid, Pid_ref} | lists:keydelete(Some_pid, 1, T)]);
+ print_diagnostics ->
+ io:format("~1000.p~n", [ibrowse:get_metrics()]),
+ erlang:send_after(1000, self(), print_diagnostics),
+ wait_for_workers(Pids);
+ query_state ->
+ io:format("Waiting for ~p~n", [Pids]),
+ wait_for_workers(Pids);
+ shutdown ->
+ io:format("Shutting down on command. Still waiting for ~p workers~n", [length(Pids)]);
+ {'DOWN', _, process, _, normal} ->
+ wait_for_workers(Pids);
+ {'DOWN', _, process, Down_pid, Rsn} ->
+ io:format("Worker ~p died. Reason: ~p~n", [Down_pid, Rsn]),
+ wait_for_workers(lists:keydelete(Down_pid, 1, Pids));
+ X ->
+ io:format("Recvd unknown msg: ~p~n", [X]),
+ wait_for_workers(Pids)
+ end.
+
+worker_loop(Parent, 0) ->
+ Parent ! {done, self()};
+worker_loop(Parent, N) ->
+ Delay = ?RAND:uniform(100),
+ Url = case Delay rem 10 of
+ %% Change 10 to some number between 0-9 depending on how
+ %% much chaos you want to introduce into the server
+ %% side. The higher the number, the more often the
+ %% server will close a connection after serving the
+ %% first request, thereby forcing the client to
+ %% retry. Any number of 10 or higher will disable this
+ %% chaos mechanism
+ 10 ->
+ ets:update_counter(?ibrowse_load_test_counters, one_request_only, 1),
+ "http://localhost:8081/ibrowse_handle_one_request_only";
+ _ ->
+ "http://localhost:8081/blah"
+ end,
+ Start_time = os:timestamp(),
+ Res = ibrowse:send_req(Url, [], get),
+ End_time = os:timestamp(),
+ Time_taken = trunc(round(timer:now_diff(End_time, Start_time) / 1000)),
+ ets:insert(ibrowse_load_timings, {os:timestamp(), Time_taken}),
+ case Res of
+ {ok, "200", _, _} ->
+ ets:update_counter(?ibrowse_load_test_counters, success, 1);
+ {error, req_timedout} ->
+ ets:update_counter(?ibrowse_load_test_counters, timeout, 1);
+ {error, retry_later} ->
+ ets:update_counter(?ibrowse_load_test_counters, retry_later, 1);
+ {error, Reason} ->
+ update_unknown_counter(Reason, 1);
+ _ ->
+ io:format("~p -- Res: ~p~n", [self(), Res]),
+ ets:update_counter(?ibrowse_load_test_counters, failed, 1)
+ end,
+ timer:sleep(Delay),
+ worker_loop(Parent, N - 1).
+
+update_unknown_counter(Counter, Inc_val) ->
+ case catch ets:update_counter(?ibrowse_load_test_counters, Counter, Inc_val) of
+ {'EXIT', _} ->
+ ets:insert_new(?ibrowse_load_test_counters, {Counter, 0}),
+ update_unknown_counter(Counter, Inc_val);
+ _ ->
+ ok
+ end.
diff --git a/test/ibrowse_socks_server.erl b/test/ibrowse_socks_server.erl
new file mode 100644
index 0000000..8a17ad4
--- /dev/null
+++ b/test/ibrowse_socks_server.erl
@@ -0,0 +1,218 @@
+%%%-------------------------------------------------------------------
+%%% @author Chandru Mullaparthi <>
+%%% @copyright (C) 2016, Chandru Mullaparthi
+%%% @doc
+%%%
+%%% @end
+%%% Created : 19 Apr 2016 by Chandru Mullaparthi <>
+%%%-------------------------------------------------------------------
+-module(ibrowse_socks_server).
+
+-behaviour(gen_server).
+
+%% API
+-export([start/2, stop/1]).
+
+%% gen_server callbacks
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+ terminate/2, code_change/3]).
+
+-define(SERVER, ?MODULE).
+
+-record(state, {listen_port, listen_socket, auth_method}).
+
+-define(NO_AUTH, 0).
+-define(AUTH_USER_PW, 2).
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+
+start(Port, Auth_method) ->
+ Name = make_proc_name(Port),
+ gen_server:start({local, Name}, ?MODULE, [Port, Auth_method], []).
+
+stop(Port) ->
+ make_proc_name(Port) ! stop.
+
+make_proc_name(Port) ->
+ list_to_atom("ibrowse_socks_server_" ++ integer_to_list(Port)).
+
+%%%===================================================================
+%%% gen_server callbacks
+%%%===================================================================
+
+init([Port, Auth_method]) ->
+ State = #state{listen_port = Port, auth_method = Auth_method},
+ {ok, Sock} = gen_tcp:listen(State#state.listen_port, [{active, false}, binary, {reuseaddr, true}]),
+ self() ! accept_connection,
+ process_flag(trap_exit, true),
+ {ok, State#state{listen_socket = Sock}}.
+
+handle_call(_Request, _From, State) ->
+ Reply = ok,
+ {reply, Reply, State}.
+
+handle_cast(_Msg, State) ->
+ {noreply, State}.
+
+handle_info(accept_connection, State) ->
+ case gen_tcp:accept(State#state.listen_socket, 1000) of
+ {error, timeout} ->
+ self() ! accept_connection,
+ {noreply, State};
+ {ok, Socket} ->
+ Pid = proc_lib:spawn_link(fun() ->
+ socks_server_loop(Socket, State#state.auth_method)
+ end),
+ gen_tcp:controlling_process(Socket, Pid),
+ Pid ! ready,
+ self() ! accept_connection,
+ {noreply, State};
+ _Err ->
+ {stop, normal, State}
+ end;
+
+handle_info(stop, State) ->
+ {stop, normal, State};
+
+handle_info(_Info, State) ->
+ {noreply, State}.
+
+terminate(_Reason, _State) ->
+ ok.
+
+code_change(_OldVsn, State, _Extra) ->
+ {ok, State}.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+socks_server_loop(In_socket, Auth_method) ->
+ receive
+ ready ->
+ socks_server_loop(In_socket, Auth_method, <<>>, unauth)
+ end.
+
+socks_server_loop(In_socket, Auth_method, Acc, unauth) ->
+ inet:setopts(In_socket, [{active, once}]),
+ receive
+ {tcp, In_socket, Data} ->
+ Acc_1 = list_to_binary([Acc, Data]),
+ case Acc_1 of
+ <<5, ?NO_AUTH>> when Auth_method == ?NO_AUTH ->
+ ok = gen_tcp:send(In_socket, <<5, ?NO_AUTH>>),
+ socks_server_loop(In_socket, Auth_method, <<>>, auth_done);
+ <<5, Num_auth_methods, Auth_methods:Num_auth_methods/binary>> ->
+ case lists:member(Auth_method, binary_to_list(Auth_methods)) of
+ true ->
+ ok = gen_tcp:send(In_socket, <<5, Auth_method>>),
+ Conn_state = case Auth_method of
+ ?NO_AUTH -> auth_done;
+ _ -> auth_pending
+ end,
+ socks_server_loop(In_socket, Auth_method, <<>>, Conn_state);
+ false ->
+ ok = gen_tcp:send(In_socket, <<5, 16#ff>>),
+ gen_tcp:close(In_socket)
+ end;
+ _ ->
+ ok = gen_tcp:send(In_socket, <<5, 0>>),
+ gen_tcp:close(In_socket)
+ end;
+ {tcp_closed, In_socket} ->
+ ok;
+ {tcp_error, In_socket, _Rsn} ->
+ ok
+ end;
+socks_server_loop(In_socket, Auth_method, Acc, auth_pending) ->
+ inet:setopts(In_socket, [{active, once}]),
+ receive
+ {tcp, In_socket, Data} ->
+ Acc_1 = list_to_binary([Acc, Data]),
+ case Acc_1 of
+ <<1, U_len, Username:U_len/binary, P_len, Password:P_len/binary>> ->
+ case check_user_pw(Username, Password) of
+ ok ->
+ ok = gen_tcp:send(In_socket, <<1, 0>>),
+ socks_server_loop(In_socket, Auth_method, <<>>, auth_done);
+ notok ->
+ ok = gen_tcp:send(In_socket, <<1, 1>>),
+ gen_tcp:close(In_socket)
+ end;
+ _ ->
+ socks_server_loop(In_socket, Auth_method, Acc_1, auth_pending)
+ end;
+ {tcp_closed, In_socket} ->
+ ok;
+ {tcp_error, In_socket, _Rsn} ->
+ ok
+ end;
+socks_server_loop(In_socket, Auth_method, Acc, auth_done) ->
+ inet:setopts(In_socket, [{active, once}]),
+ receive
+ {tcp, In_socket, Data} ->
+ Acc_1 = list_to_binary([Acc, Data]),
+ case Acc_1 of
+ <<5, 1, 0, Addr_type, Dest_ip:4/binary, Dest_port:16>> when Addr_type == 1->
+ handle_connect(In_socket, Addr_type, Dest_ip, Dest_port);
+ <<5, 1, 0, Addr_type, Dest_len, Dest_hostname:Dest_len/binary, Dest_port:16>> when Addr_type == 3 ->
+ handle_connect(In_socket, Addr_type, Dest_hostname, Dest_port);
+ <<5, 1, 0, Addr_type, Dest_ip:16/binary, Dest_port:16>> when Addr_type == 4->
+ handle_connect(In_socket, Addr_type, Dest_ip, Dest_port);
+ _ ->
+ socks_server_loop(In_socket, Auth_method, Acc_1, auth_done)
+ end;
+ {tcp_closed, In_socket} ->
+ ok;
+ {tcp_error, In_socket, _Rsn} ->
+ ok
+ end.
+
+handle_connect(In_socket, Addr_type, Dest_host, Dest_port) ->
+ Dest_host_1 = case Addr_type of
+ 1 ->
+ list_to_tuple(binary_to_list(Dest_host));
+ 3 ->
+ binary_to_list(Dest_host);
+ 4 ->
+ list_to_tuple(binary_to_list(Dest_host))
+ end,
+ case gen_tcp:connect(Dest_host_1, Dest_port, [binary, {active, once}]) of
+ {ok, Out_socket} ->
+ Addr = case Addr_type of
+ 1 ->
+ <<Dest_host/binary, Dest_port:16>>;
+ 3 ->
+ Len = size(Dest_host),
+ <<Len, Dest_host/binary, Dest_port:16>>;
+ 4 ->
+ <<Dest_host/binary, Dest_port:16>>
+ end,
+ ok = gen_tcp:send(In_socket, <<5, 0, 0, Addr_type, Addr/binary>>),
+ inet:setopts(In_socket, [{active, once}]),
+ inet:setopts(Out_socket, [{active, once}]),
+ connected_loop(In_socket, Out_socket);
+ _Err ->
+ ok = gen_tcp:send(<<5, 1>>),
+ gen_tcp:close(In_socket)
+ end.
+
+check_user_pw(<<"user">>, <<"password">>) ->
+ ok;
+check_user_pw(_, _) ->
+ notok.
+
+connected_loop(In_socket, Out_socket) ->
+ receive
+ {tcp, In_socket, Data} ->
+ inet:setopts(In_socket, [{active, once}]),
+ ok = gen_tcp:send(Out_socket, Data),
+ connected_loop(In_socket, Out_socket);
+ {tcp, Out_socket, Data} ->
+ inet:setopts(Out_socket, [{active, once}]),
+ ok = gen_tcp:send(In_socket, Data),
+ connected_loop(In_socket, Out_socket);
+ _ ->
+ ok
+ end.
diff --git a/src/ibrowse_test.erl b/test/ibrowse_test.erl
similarity index 65%
rename from src/ibrowse_test.erl
rename to test/ibrowse_test.erl
index d97f76c..cd78049 100644
--- a/src/ibrowse_test.erl
+++ b/test/ibrowse_test.erl
@@ -5,14 +5,13 @@
-module(ibrowse_test).
-export([
- load_test/3,
+ load_test_/3,
send_reqs_1/3,
do_send_req/2,
+ local_unit_tests/0,
unit_tests/0,
- unit_tests/1,
- unit_tests_1/2,
- ue_test/0,
- ue_test/1,
+ unit_tests/2,
+ unit_tests_1/3,
verify_chunked_streaming/0,
verify_chunked_streaming/1,
test_chunked_streaming_once/0,
@@ -27,9 +26,108 @@
test_head_transfer_encoding/0,
test_head_transfer_encoding/1,
test_head_response_with_body/0,
- test_head_response_with_body/1
+ test_head_response_with_body/1,
+ test_303_response_with_no_body/0,
+ test_303_response_with_no_body/1,
+ test_303_response_with_a_body/0,
+ test_303_response_with_a_body/1,
+ test_preserve_status_line/0,
+ test_binary_headers/0,
+ test_binary_headers/1,
+ test_generate_body_0/0,
+ test_retry_of_requests/0,
+ test_retry_of_requests/1,
+ test_save_to_file_no_content_length/0,
+ socks5_noauth/0,
+ socks5_auth_succ/0,
+ socks5_auth_fail/0
]).
+-include_lib("ibrowse/include/ibrowse.hrl").
+
+%%------------------------------------------------------------------------------
+%% Unit Tests
+%%------------------------------------------------------------------------------
+-define(LOCAL_TESTS, [
+ {local_test_fun, socks5_noauth, []},
+ {local_test_fun, socks5_auth_succ, []},
+ {local_test_fun, socks5_auth_fail, []},
+ {local_test_fun, test_preserve_status_line, []},
+ {local_test_fun, test_save_to_file_no_content_length, []},
+ {local_test_fun, test_20122010, []},
+ {local_test_fun, test_pipeline_head_timeout, []},
+ {local_test_fun, test_head_transfer_encoding, []},
+ {local_test_fun, test_head_response_with_body, []},
+ {local_test_fun, test_303_response_with_a_body, []},
+ {local_test_fun, test_303_response_with_no_body, []},
+ {local_test_fun, test_binary_headers, []},
+ {local_test_fun, test_retry_of_requests, []},
+ {local_test_fun, verify_chunked_streaming, []},
+ {local_test_fun, test_chunked_streaming_once, []},
+ {local_test_fun, test_generate_body_0, []}
+ ]).
+
+-define(TEST_LIST, [{"http://intranet/messenger", get},
+ {"http://www.google.co.uk", get},
+ {"http://www.google.com", get},
+ {"http://www.google.com", options},
+ {"https://mail.google.com", get},
+ {"http://www.sun.com", get},
+ {"http://www.oracle.com", get},
+ {"http://www.bbc.co.uk", get},
+ {"http://www.bbc.co.uk", trace},
+ {"http://www.bbc.co.uk", options},
+ {"http://yaws.hyber.org", get},
+ {"http://jigsaw.w3.org/HTTP/ChunkedScript", get},
+ {"http://jigsaw.w3.org/HTTP/TE/foo.txt", get},
+ {"http://jigsaw.w3.org/HTTP/TE/bar.txt", get},
+ {"http://jigsaw.w3.org/HTTP/connection.html", get},
+ {"http://jigsaw.w3.org/HTTP/cc.html", get},
+ {"http://jigsaw.w3.org/HTTP/cc-private.html", get},
+ {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html", get},
+ {"http://jigsaw.w3.org/HTTP/cc-nocache.html", get},
+ {"http://jigsaw.w3.org/HTTP/h-content-md5.html", get},
+ {"http://jigsaw.w3.org/HTTP/h-retry-after.html", get},
+ {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html", get},
+ {"http://jigsaw.w3.org/HTTP/neg", get},
+ {"http://jigsaw.w3.org/HTTP/negbad", get},
+ {"http://jigsaw.w3.org/HTTP/400/toolong/", get},
+ {"http://jigsaw.w3.org/HTTP/300/", get},
+ {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
+ {"http://jigsaw.w3.org/HTTP/CL/", get},
+ {"http://www.httpwatch.com/httpgallery/chunked/", get},
+ {"https://github.com", get, [{ssl_options, [{depth, 2}]}]}
+ ]).
+
+socks5_noauth() ->
+ case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+ [{socks5_host, "localhost"}, {socks5_port, 8282}], 2000) of
+ {ok, "200", _, _} ->
+ success;
+ Err ->
+ Err
+ end.
+
+socks5_auth_succ() ->
+ case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+ [{socks5_host, "localhost"}, {socks5_port, 8383},
+ {socks5_user, <<"user">>}, {socks5_password, <<"password">>}], 2000) of
+ {ok, "200", _, _} ->
+ success;
+ Err ->
+ Err
+ end.
+
+socks5_auth_fail() ->
+ case ibrowse:send_req("http://localhost:8181/success", [], get, [],
+ [{socks5_host, "localhost"}, {socks5_port, 8282},
+ {socks5_user, <<"user">>}, {socks5_password, <<"wrong_password">>}], 2000) of
+ {error,{conn_failed,{error,unacceptable}}} ->
+ success;
+ Err ->
+ Err
+ end.
+
test_stream_once(Url, Method, Options) ->
test_stream_once(Url, Method, Options, 5000).
@@ -69,9 +167,10 @@ test_stream_once(Req_id) ->
{ibrowse_async_response_end, Req_id} ->
ok
end.
+
%% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to
%% tweak settings before running the load test. The defaults are 10 and 10.
-load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url),
+load_test_(Url, NumWorkers, NumReqsPerWorker) when is_list(Url),
is_integer(NumWorkers),
is_integer(NumReqsPerWorker),
NumWorkers > 0,
@@ -79,17 +178,19 @@ load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url),
proc_lib:spawn(?MODULE, send_reqs_1, [Url, NumWorkers, NumReqsPerWorker]).
send_reqs_1(Url, NumWorkers, NumReqsPerWorker) ->
- Start_time = now(),
+ Start_time = os:timestamp(),
ets:new(pid_table, [named_table, public]),
ets:new(ibrowse_test_results, [named_table, public]),
ets:new(ibrowse_errors, [named_table, public, ordered_set]),
+ ets:new(ibrowse_counter, [named_table, public, ordered_set]),
+ ets:insert(ibrowse_counter, {req_id, 1}),
init_results(),
process_flag(trap_exit, true),
log_msg("Starting spawning of workers...~n", []),
spawn_workers(Url, NumWorkers, NumReqsPerWorker),
log_msg("Finished spawning workers...~n", []),
do_wait(Url),
- End_time = now(),
+ End_time = os:timestamp(),
log_msg("All workers are done...~n", []),
log_msg("ibrowse_test_results table: ~n~p~n", [ets:tab2list(ibrowse_test_results)]),
log_msg("Start time: ~1000.p~n", [calendar:now_to_local_time(Start_time)]),
@@ -167,7 +268,7 @@ do_send_req_1(Url, NumReqs) ->
{error, retry_later} ->
ets:update_counter(ibrowse_test_results, retry_later, 1);
Err ->
- ets:insert(ibrowse_errors, {now(), Err}),
+ ets:insert(ibrowse_errors, {os:timestamp(), Err}),
ets:update_counter(ibrowse_test_results, other_error, 1),
ok
end,
@@ -178,7 +279,7 @@ dump_errors() ->
0 ->
ok;
_ ->
- {A, B, C} = now(),
+ {A, B, C} = os:timestamp(),
Filename = lists:flatten(
io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])),
case file:open(Filename, [write, delayed_write, raw]) of
@@ -197,57 +298,23 @@ dump_errors(Key, Iod) ->
file:write(Iod, io_lib:format("~p~n", [Term])),
dump_errors(ets:next(ibrowse_errors, Key), Iod).
-%%------------------------------------------------------------------------------
-%% Unit Tests
-%%------------------------------------------------------------------------------
--define(TEST_LIST, [{"http://intranet/messenger", get},
- {"http://www.google.co.uk", get},
- {"http://www.google.com", get},
- {"http://www.google.com", options},
- {"https://mail.google.com", get},
- {"http://www.sun.com", get},
- {"http://www.oracle.com", get},
- {"http://www.bbc.co.uk", get},
- {"http://www.bbc.co.uk", trace},
- {"http://www.bbc.co.uk", options},
- {"http://yaws.hyber.org", get},
- {"http://jigsaw.w3.org/HTTP/ChunkedScript", get},
- {"http://jigsaw.w3.org/HTTP/TE/foo.txt", get},
- {"http://jigsaw.w3.org/HTTP/TE/bar.txt", get},
- {"http://jigsaw.w3.org/HTTP/connection.html", get},
- {"http://jigsaw.w3.org/HTTP/cc.html", get},
- {"http://jigsaw.w3.org/HTTP/cc-private.html", get},
- {"http://jigsaw.w3.org/HTTP/cc-proxy-revalidate.html", get},
- {"http://jigsaw.w3.org/HTTP/cc-nocache.html", get},
- {"http://jigsaw.w3.org/HTTP/h-content-md5.html", get},
- {"http://jigsaw.w3.org/HTTP/h-retry-after.html", get},
- {"http://jigsaw.w3.org/HTTP/h-retry-after-date.html", get},
- {"http://jigsaw.w3.org/HTTP/neg", get},
- {"http://jigsaw.w3.org/HTTP/negbad", get},
- {"http://jigsaw.w3.org/HTTP/400/toolong/", get},
- {"http://jigsaw.w3.org/HTTP/300/", get},
- {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]},
- {"http://jigsaw.w3.org/HTTP/CL/", get},
- {"http://www.httpwatch.com/httpgallery/chunked/", get},
- {"https://github.com", get, [{ssl_options, [{depth, 2}]}]},
- {local_test_fun, test_20122010, []},
- {local_test_fun, test_pipeline_head_timeout, []},
- {local_test_fun, test_head_transfer_encoding, []},
- {local_test_fun, test_head_response_with_body, []}
- ]).
+local_unit_tests() ->
+ unit_tests([], ?LOCAL_TESTS).
unit_tests() ->
- unit_tests([]).
+ unit_tests([], ?TEST_LIST).
-unit_tests(Options) ->
+unit_tests(Options, Test_list) ->
+ error_logger:tty(false),
application:start(crypto),
+ application:start(asn1),
application:start(public_key),
application:start(ssl),
(catch ibrowse_test_server:start_server(8181, tcp)),
- ibrowse:start(),
+ application:start(ibrowse),
Options_1 = Options ++ [{connect_timeout, 5000}],
Test_timeout = proplists:get_value(test_timeout, Options, 60000),
- {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]),
+ {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1, Test_list]),
receive
{done, Pid} ->
ok;
@@ -258,16 +325,17 @@ unit_tests(Options) ->
io:format("Timed out waiting for tests to complete~n", [])
end,
catch ibrowse_test_server:stop_server(8181),
+ error_logger:tty(true),
ok.
-unit_tests_1(Parent, Options) ->
+unit_tests_1(Parent, Options, Test_list) ->
lists:foreach(fun({local_test_fun, Fun_name, Args}) ->
execute_req(local_test_fun, Fun_name, Args);
({Url, Method}) ->
execute_req(Url, Method, Options);
({Url, Method, X_Opts}) ->
execute_req(Url, Method, X_Opts ++ Options)
- end, ?TEST_LIST),
+ end, Test_list),
Parent ! {done, self()}.
verify_chunked_streaming() ->
@@ -294,7 +362,8 @@ verify_chunked_streaming(Options) ->
Res2 = compare_responses(Result_without_streaming, Async_response_list, Async_response_bin_once),
case {Res1, Res2} of
{success, success} ->
- io:format(" Chunked streaming working~n", []);
+ io:format(" Chunked streaming working~n", []),
+ success;
_ ->
ok
end.
@@ -309,7 +378,7 @@ test_chunked_streaming_once(Options) ->
io:format(" Fetching data with streaming as binary, {active, once}...~n", []),
case do_async_req_list(Url, get, [once, {response_format, binary} | Options]) of
{ok, _, _, _} ->
- io:format(" Success!~n", []);
+ success;
Err ->
io:format(" Fail: ~p~n", [Err])
end.
@@ -362,6 +431,8 @@ wait_for_resp(Pid) ->
{'EXIT', Reason};
{'DOWN', _, _, _, _} ->
wait_for_resp(Pid);
+ {'EXIT', _, normal} ->
+ wait_for_resp(Pid);
Msg ->
io:format("Recvd unknown message: ~p~n", [Msg]),
wait_for_resp(Pid)
@@ -416,8 +487,9 @@ maybe_stream_next(Req_id, Options) ->
end.
execute_req(local_test_fun, Method, Args) ->
- io:format(" ~-54.54w: ", [Method]),
+ reset_ibrowse(),
Result = (catch apply(?MODULE, Method, Args)),
+ io:format(" ~-54.54w: ", [Method]),
io:format("~p~n", [Result]);
execute_req(Url, Method, Options) ->
io:format("~7.7w, ~50.50s: ", [Method, Url]),
@@ -429,15 +501,6 @@ execute_req(Url, Method, Options) ->
io:format("~p~n", [Err])
end.
-ue_test() ->
- ue_test(lists:duplicate(1024, $?)).
-ue_test(Data) ->
- {Time, Res} = timer:tc(ibrowse_lib, url_encode, [Data]),
- io:format("Time -> ~p~n", [Time]),
- io:format("Data Length -> ~p~n", [length(Data)]),
- io:format("Res Length -> ~p~n", [length(Res)]).
-% io:format("Result -> ~s~n", [Res]).
-
log_msg(Fmt, Args) ->
io:format("~s -- " ++ Fmt,
[ibrowse_lib:printable_date() | Args]).
@@ -459,6 +522,28 @@ test_head_transfer_encoding(Url) ->
{test_failed, Res}
end.
+%%------------------------------------------------------------------------------
+%% Test what happens when the response to a HEAD request is a
+%% Chunked-Encoding response with a non-empty body. Issue #67 on
+%% Github
+%% ------------------------------------------------------------------------------
+test_binary_headers() ->
+ clear_msg_q(),
+ test_binary_headers("http://localhost:8181/ibrowse_echo_header").
+
+test_binary_headers(Url) ->
+ case ibrowse:send_req(Url, [{<<"x-binary">>, <<"x-header">>}], get) of
+ {ok, "200", Headers, _} ->
+ case proplists:get_value("x-binary", Headers) of
+ "x-header" ->
+ success;
+ V ->
+ {fail, V}
+ end;
+ Res ->
+ {test_failed, Res}
+ end.
+
%%------------------------------------------------------------------------------
%% Test what happens when the response to a HEAD request is a
%% Chunked-Encoding response with a non-empty body. Issue #67 on
@@ -476,6 +561,139 @@ test_head_response_with_body(Url) ->
{test_failed, Res}
end.
+%%------------------------------------------------------------------------------
+%% Test what happens when a 303 response has no body
+%% Github issue #97
+%% ------------------------------------------------------------------------------
+test_303_response_with_no_body() ->
+ clear_msg_q(),
+ test_303_response_with_no_body("http://localhost:8181/ibrowse_303_no_body_test").
+
+test_303_response_with_no_body(Url) ->
+ ibrowse:add_config([{allow_303_with_no_body, true}]),
+ case ibrowse:send_req(Url, [], post) of
+ {ok, "303", _, _} ->
+ success;
+ Res ->
+ {test_failed, Res}
+ end.
+
+%% Make sure we don't break requests that do have a body.
+test_303_response_with_a_body() ->
+ clear_msg_q(),
+ test_303_response_with_no_body("http://localhost:8181/ibrowse_303_with_body_test").
+
+test_303_response_with_a_body(Url) ->
+ ibrowse:add_config([{allow_303_with_no_body, true}]),
+ case ibrowse:send_req(Url, [], post) of
+ {ok, "303", _, "abcde"} ->
+ success;
+ Res ->
+ {test_failed, Res}
+ end.
+
+%% Test that the 'preserve_status_line' option works as expected
+test_preserve_status_line() ->
+ case ibrowse:send_req("http://localhost:8181/ibrowse_preserve_status_line", [], get, [],
+ [{preserve_status_line, true}]) of
+ {ok, "200", [{ibrowse_status_line,<<"HTTP/1.1 200 OKBlah">>} | _], _} ->
+ success;
+ Res ->
+ {test_failed, Res}
+ end.
+
+%%------------------------------------------------------------------------------
+%% Test that when the save_response_to_file option is used with a server which
+%% does not send the Content-Length header, the response is saved correctly to
+%% a file
+%%------------------------------------------------------------------------------
+test_save_to_file_no_content_length() ->
+ clear_msg_q(),
+ {{Y, M, D}, {H, Mi, S}} = calendar:local_time(),
+ Test_file = filename:join
+ ([".",
+ lists:flatten(
+ io_lib:format("test_save_to_file_no_content_length_~p~p~p_~p~p~p.txt", [Y, M, D, H, Mi, S]))]),
+ try
+ case ibrowse:send_req("http://localhost:8181/ibrowse_send_file_conn_close", [], get, [],
+ [{save_response_to_file, Test_file}]) of
+ {ok, "200", _, {file, Test_file}} ->
+ success;
+ Res ->
+ {test_failed, Res}
+ end
+ after
+ file:delete(Test_file)
+ end.
+
+%%------------------------------------------------------------------------------
+%% Test that retry of requests happens correctly, and that ibrowse doesn't retry
+%% if there is not enough time left
+%%------------------------------------------------------------------------------
+test_retry_of_requests() ->
+ clear_msg_q(),
+ test_retry_of_requests("http://localhost:8181/ibrowse_handle_one_request_only_with_delay").
+
+test_retry_of_requests(Url) ->
+ reset_ibrowse(),
+ Timeout_1 = 2050,
+ Res_1 = test_retry_of_requests(Url, Timeout_1),
+ case lists:filter(fun({_Pid, {ok, "200", _, _}}) ->
+ true;
+ (_) -> false
+ end, Res_1) of
+ [_|_] = X ->
+ Res_1_1 = Res_1 -- X,
+ case lists:all(
+ fun({_Pid, {error, retry_later}}) ->
+ true;
+ (_) ->
+ false
+ end, Res_1_1) of
+ true ->
+ ok;
+ false ->
+ exit({failed, Timeout_1, Res_1})
+ end;
+ _ ->
+ exit({failed, Timeout_1, Res_1})
+ end,
+ Timeout_2 = 2200,
+ Res_2 = test_retry_of_requests(Url, Timeout_2),
+ case lists:filter(fun({_Pid, {ok, "200", _, _}}) ->
+ true;
+ (_) -> false
+ end, Res_2) of
+ [_|_] = Res_2_X ->
+ Res_2_1 = Res_2 -- Res_2_X,
+ case lists:all(
+ fun({_Pid, {error, X_err_2}}) ->
+ (X_err_2 == retry_later) orelse (X_err_2 == req_timedout);
+ (_) ->
+ false
+ end, Res_2_1) of
+ true ->
+ ok;
+ false ->
+ exit({failed, {?MODULE, ?LINE}, Timeout_2, Res_2})
+ end;
+ _ ->
+ exit({failed, {?MODULE, ?LINE}, Timeout_2, Res_2})
+ end,
+ success.
+
+test_retry_of_requests(Url, Timeout) ->
+ #url{host = Host, port = Port} = ibrowse_lib:parse_url(Url),
+ ibrowse:set_max_sessions(Host, Port, 1),
+ Parent = self(),
+ Pids = lists:map(fun(_) ->
+ spawn(fun() ->
+ Res = (catch ibrowse:send_req(Url, [], get, [], [], Timeout)),
+ Parent ! {self(), Res}
+ end)
+ end, lists:seq(1,10)),
+ accumulate_worker_resp(Pids).
+
%%------------------------------------------------------------------------------
%% Test what happens when the request at the head of a pipeline times out
%%------------------------------------------------------------------------------
@@ -485,22 +703,27 @@ test_pipeline_head_timeout() ->
test_pipeline_head_timeout(Url) ->
{ok, Pid} = ibrowse:spawn_worker_process(Url),
+ Fixed_timeout = 2000,
Test_parent = self(),
Fun = fun({fixed, Timeout}) ->
- spawn(fun() ->
- do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout)
- end);
- (Timeout_mult) ->
- spawn(fun() ->
- Timeout = 1000 + Timeout_mult*1000,
- do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout)
- end)
- end,
- Pids = [Fun(X) || X <- [{fixed, 32000} | lists:seq(1,10)]],
+ X_pid = spawn(fun() ->
+ do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout)
+ end),
+ %% io:format("Pid ~p with a fixed timeout~n", [X_pid]),
+ X_pid;
+ (Timeout_mult) ->
+ Timeout = Fixed_timeout + Timeout_mult*1000,
+ X_pid = spawn(fun() ->
+ do_test_pipeline_head_timeout(Url, Pid, Test_parent, Timeout)
+ end),
+ %% io:format("Pid ~p with a timeout of ~p~n", [X_pid, Timeout]),
+ X_pid
+ end,
+ Pids = [Fun(X) || X <- [{fixed, Fixed_timeout} | lists:seq(1,10)]],
Result = accumulate_worker_resp(Pids),
case lists:all(fun({_, X_res}) ->
- X_res == {error,req_timedout}
- end, Result) of
+ (X_res == {error,req_timedout}) orelse (X_res == {error, connection_closed})
+ end, Result) of
true ->
success;
false ->
@@ -616,6 +839,44 @@ do_test_20122010_1(Expected_resp, Req_id, Acc) ->
exit({timeout, test_failed})
end.
+%%------------------------------------------------------------------------------
+%% Test requests where body is generated using a Fun
+%%------------------------------------------------------------------------------
+test_generate_body_0() ->
+ Tid = ets:new(ibrowse_test_state, [public]),
+ try
+ Body_1 = <<"Part 1 of the body">>,
+ Body_2 = <<"Part 2 of the body\r\n">>,
+ Size = size(Body_1) + size(Body_2),
+ Body = list_to_binary([Body_1, Body_2]),
+ Fun = fun() ->
+ case ets:lookup(Tid, body_gen_state) of
+ [] ->
+ ets:insert(Tid, {body_gen_state, 1}),
+ {ok, Body_1};
+ [{_, 1}]->
+ ets:insert(Tid, {body_gen_state, 2}),
+ {ok, Body_2};
+ [{_, 2}] ->
+ eof
+ end
+ end,
+ case ibrowse:send_req("http://localhost:8181/echo_body",
+ [{"Content-Length", Size}],
+ post,
+ Fun,
+ [{response_format, binary},
+ {http_vsn, {1,1}}]) of
+ {ok, "200", _, Body} ->
+ success;
+ Err ->
+ io:format("Test failed : ~p~n", [Err]),
+ {test_failed, Err}
+ end
+ after
+ ets:delete(Tid)
+ end.
+
do_trace(Fmt, Args) ->
do_trace(get(my_trace_flag), Fmt, Args).
@@ -623,3 +884,7 @@ do_trace(true, Fmt, Args) ->
io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]);
do_trace(_, _, _) ->
ok.
+
+reset_ibrowse() ->
+ application:stop(ibrowse),
+ application:start(ibrowse).
diff --git a/test/ibrowse_test_server.erl b/test/ibrowse_test_server.erl
new file mode 100644
index 0000000..3ba7c67
--- /dev/null
+++ b/test/ibrowse_test_server.erl
@@ -0,0 +1,363 @@
+%%% File : ibrowse_test_server.erl
+%%% Author : Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
+%%% Description : A server to simulate various test scenarios
+%%% Created : 17 Oct 2010 by Chandrashekhar Mullaparthi <chandrashekhar.mullaparthi@t-mobile.co.uk>
+
+-module(ibrowse_test_server).
+-export([
+ start_server/0,
+ start_server/2,
+ stop_server/1,
+ get_conn_pipeline_depth/0
+ ]).
+
+-ifdef(new_rand).
+-define(RAND, rand).
+-else.
+-define(RAND, random).
+-endif.
+
+-record(request, {method, uri, version, headers = [], body = [], state}).
+
+-define(dec2hex(X), erlang:integer_to_list(X, 16)).
+-define(ACCEPT_TIMEOUT_MS, 10000).
+-define(CONN_PIPELINE_DEPTH, conn_pipeline_depth).
+
+start_server() ->
+ start_server(8181, tcp).
+
+start_server(Port, Sock_type) ->
+ Fun = fun() ->
+ Proc_name = server_proc_name(Port),
+ case whereis(Proc_name) of
+ undefined ->
+ register(Proc_name, self()),
+ ets:new(?CONN_PIPELINE_DEPTH, [named_table, public, set]),
+ case do_listen(Sock_type, Port, [{active, false},
+ {reuseaddr, true},
+ {nodelay, true},
+ {packet, http}]) of
+ {ok, Sock} ->
+ do_trace("Server listening on port: ~p~n", [Port]),
+ accept_loop(Sock, Sock_type);
+ Err ->
+ erlang:error(
+ lists:flatten(
+ io_lib:format(
+ "Failed to start server on port ~p. ~p~n",
+ [Port, Err]))),
+ exit({listen_error, Err})
+ end;
+ _X ->
+ ok
+ end
+ end,
+ spawn_link(Fun),
+ ibrowse_socks_server:start(8282, 0), %% No auth
+ ibrowse_socks_server:start(8383, 2). %% Username/Password auth
+
+stop_server(Port) ->
+ catch server_proc_name(Port) ! stop,
+ ibrowse_socks_server:stop(8282),
+ ibrowse_socks_server:stop(8383),
+ timer:sleep(2000), % wait for server to receive msg and unregister
+ ok.
+
+get_conn_pipeline_depth() ->
+ ets:tab2list(?CONN_PIPELINE_DEPTH).
+
+server_proc_name(Port) ->
+ list_to_atom("ibrowse_test_server_"++integer_to_list(Port)).
+
+do_listen(tcp, Port, Opts) ->
+ gen_tcp:listen(Port, Opts);
+do_listen(ssl, Port, Opts) ->
+ application:start(crypto),
+ application:start(ssl),
+ ssl:listen(Port, Opts).
+
+do_accept(tcp, Listen_sock) ->
+ gen_tcp:accept(Listen_sock, ?ACCEPT_TIMEOUT_MS);
+do_accept(ssl, Listen_sock) ->
+ ssl:ssl_accept(Listen_sock, ?ACCEPT_TIMEOUT_MS).
+
+accept_loop(Sock, Sock_type) ->
+ case do_accept(Sock_type, Sock) of
+ {ok, Conn} ->
+ Pid = spawn_link(fun() -> connection(Conn, Sock_type) end),
+ set_controlling_process(Conn, Sock_type, Pid),
+ accept_loop(Sock, Sock_type);
+ {error, timeout} ->
+ receive
+ stop ->
+ ok
+ after 10 ->
+ accept_loop(Sock, Sock_type)
+ end;
+ Err ->
+ Err
+ end.
+
+connection(Conn, Sock_type) ->
+ catch ets:insert(?CONN_PIPELINE_DEPTH, {self(), 0}),
+ try
+ inet:setopts(Conn, [{packet, http}, {active, true}]),
+ server_loop(Conn, Sock_type, #request{})
+ after
+ catch ets:delete(?CONN_PIPELINE_DEPTH, self())
+ end.
+
+set_controlling_process(Sock, tcp, Pid) ->
+ gen_tcp:controlling_process(Sock, Pid);
+set_controlling_process(Sock, ssl, Pid) ->
+ ssl:controlling_process(Sock, Pid).
+
+setopts(Sock, tcp, Opts) ->
+ inet:setopts(Sock, Opts);
+setopts(Sock, ssl, Opts) ->
+ ssl:setopts(Sock, Opts).
+
+server_loop(Sock, Sock_type, #request{headers = Headers} = Req) ->
+ receive
+ {http, Sock, {http_request, HttpMethod, HttpUri, HttpVersion}} ->
+ catch ets:update_counter(?CONN_PIPELINE_DEPTH, self(), 1),
+ server_loop(Sock, Sock_type, Req#request{method = HttpMethod,
+ uri = HttpUri,
+ version = HttpVersion});
+ {http, Sock, {http_header, _, _, _, _} = H} ->
+ server_loop(Sock, Sock_type, Req#request{headers = [H | Headers]});
+ {http, Sock, http_eoh} ->
+ case process_request(Sock, Sock_type, Req) of
+ close_connection ->
+ gen_tcp:shutdown(Sock, read_write);
+ not_done ->
+ ok;
+ collect_body ->
+ server_loop(Sock, Sock_type, Req#request{state = collect_body});
+ _ ->
+ catch ets:update_counter(?CONN_PIPELINE_DEPTH, self(), -1)
+ end,
+ server_loop(Sock, Sock_type, #request{});
+ {http, Sock, {http_error, Packet}} when Req#request.state == collect_body ->
+ Req_1 = Req#request{body = list_to_binary([Packet, Req#request.body])},
+ case process_request(Sock, Sock_type, Req_1) of
+ close_connection ->
+ gen_tcp:shutdown(Sock, read_write);
+ ok ->
+ server_loop(Sock, Sock_type, #request{});
+ collect_body ->
+ server_loop(Sock, Sock_type, Req_1)
+ end;
+ {http, Sock, {http_error, Err}} ->
+ io:format("Error parsing HTTP request:~n"
+ "Req so far : ~p~n"
+ "Err : ~p", [Req, Err]),
+ exit({http_error, Err});
+ {setopts, Opts} ->
+ setopts(Sock, Sock_type, Opts),
+ server_loop(Sock, Sock_type, Req);
+ {tcp_closed, Sock} ->
+ do_trace("Client closed connection~n", []),
+ ok;
+ Other ->
+ io:format("Recvd unknown msg: ~p~n", [Other]),
+ exit({unknown_msg, Other})
+ after 120000 ->
+ do_trace("Timing out client connection~n", []),
+ ok
+ end.
+
+do_trace(Fmt, Args) ->
+ do_trace(get(my_trace_flag), Fmt, Args).
+
+do_trace(true, Fmt, Args) ->
+ io:format("~s -- " ++ Fmt, [ibrowse_lib:printable_date() | Args]);
+do_trace(_, _, _) ->
+ ok.
+
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = Headers,
+ uri = {abs_path, "/ibrowse_stream_once_chunk_pipeline_test"}} = Req) ->
+ Req_id = case lists:keysearch("X-Ibrowse-Request-Id", 3, Headers) of
+ false ->
+ "";
+ {value, {http_header, _, _, _, Req_id_1}} ->
+ Req_id_1
+ end,
+ Req_id_header = ["x-ibrowse-request-id: ", Req_id, "\r\n"],
+ do_trace("Recvd req: ~p~n", [Req]),
+ Body = string:join([integer_to_list(X) || X <- lists:seq(1,100)], "-"),
+ Chunked_body = chunk_request_body(Body, 50),
+ Resp_1 = [<<"HTTP/1.1 200 OK\r\n">>,
+ Req_id_header,
+ <<"Transfer-Encoding: chunked\r\n\r\n">>],
+ Resp_2 = Chunked_body,
+ do_send(Sock, Sock_type, Resp_1),
+ timer:sleep(100),
+ do_send(Sock, Sock_type, Resp_2);
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_inac_timeout_test"}} = Req) ->
+ do_trace("Recvd req: ~p. Sleeping for 30 secs...~n", [Req]),
+ timer:sleep(3000),
+ do_trace("...Sending response now.~n", []),
+ Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='HEAD',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_head_transfer_enc"}}) ->
+ Resp = <<"HTTP/1.1 400 Bad Request\r\nServer: Apache-Coyote/1.1\r\nContent-Length:5\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\n\r\nabcde">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='POST',
+ headers = Headers,
+ uri = {abs_path, "/echo_body"},
+ body = Body}) ->
+ Content_len = get_content_length(Headers),
+ case iolist_size(Body) == Content_len of
+ true ->
+ Resp = [<<"HTTP/1.1 200 OK\r\nContent-Length: ">>, integer_to_list(Content_len), <<"\r\nServer: ibrowse_test_server\r\n\r\n">>, Body],
+ do_send(Sock, Sock_type, list_to_binary(Resp));
+ false ->
+ collect_body
+ end;
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = Headers,
+ uri = {abs_path, "/ibrowse_echo_header"}}) ->
+ Tag = "x-binary",
+ Headers_1 = [{to_lower(X), to_lower(Y)} || {http_header, _, X, _, Y} <- Headers],
+ X_binary_header_val = case lists:keysearch(Tag, 1, Headers_1) of
+ false ->
+ "not_found";
+ {value, {_, V}} ->
+ V
+ end,
+ Resp = [<<"HTTP/1.1 200 OK\r\n">>,
+ <<"Server: ibrowse_test\r\n">>,
+ Tag, ": ", X_binary_header_val, "\r\n",
+ <<"Content-Length: 0\r\n\r\n">>],
+ do_send(Sock, Sock_type, Resp);
+
+process_request(Sock, Sock_type,
+ #request{method='HEAD',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_head_test"}}) ->
+ Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\Date: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='POST',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_303_no_body_test"}}) ->
+ Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='POST',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_303_with_body_test"}}) ->
+ Resp = <<"HTTP/1.1 303 See Other\r\nLocation: http://example.org\r\nContent-Length: 5\r\n\r\nabcde">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_preserve_status_line"}}) ->
+ Resp = <<"HTTP/1.1 200 OKBlah\r\nContent-Length: 5\r\n\r\nabcde">>,
+ do_send(Sock, Sock_type, Resp);
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_handle_one_request_only_with_delay"}}) ->
+ timer:sleep(2000),
+ Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp),
+ close_connection;
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_handle_one_request_only"}}) ->
+ Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp),
+ close_connection;
+process_request(Sock, Sock_type,
+ #request{method='GET',
+ headers = _Headers,
+ uri = {abs_path, "/ibrowse_send_file_conn_close"}}) ->
+ Resp = <<"HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: Wed, 04 Apr 2012 16:53:49 GMT\r\nConnection: close\r\n\r\nblahblah-">>,
+ do_send(Sock, Sock_type, Resp),
+ timer:sleep(1000),
+ do_send(Sock, Sock_type, <<"blahblah">>),
+ close_connection;
+process_request(_Sock, _Sock_type, #request{uri = {abs_path, "/never_respond"} } ) ->
+ not_done;
+process_request(Sock, Sock_type, Req) ->
+ do_trace("Recvd req: ~p~n", [Req]),
+ Resp = <<"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n">>,
+ do_send(Sock, Sock_type, Resp),
+ timer:sleep(?RAND:uniform(100)).
+
+do_send(Sock, tcp, Resp) ->
+ gen_tcp:send(Sock, Resp);
+do_send(Sock, ssl, Resp) ->
+ ssl:send(Sock, Resp).
+
+%%------------------------------------------------------------------------------
+%% Utility functions
+%%------------------------------------------------------------------------------
+
+chunk_request_body(Body, _ChunkSize) when is_tuple(Body) orelse
+ is_function(Body) ->
+ Body;
+chunk_request_body(Body, ChunkSize) ->
+ chunk_request_body(Body, ChunkSize, []).
+
+chunk_request_body(Body, _ChunkSize, Acc) when Body == <<>>; Body == [] ->
+ LastChunk = "0\r\n",
+ lists:reverse(["\r\n", LastChunk | Acc]);
+chunk_request_body(Body, ChunkSize, Acc) when is_binary(Body),
+ size(Body) >= ChunkSize ->
+ <<ChunkBody:ChunkSize/binary, Rest/binary>> = Body,
+ Chunk = [?dec2hex(ChunkSize),"\r\n",
+ ChunkBody, "\r\n"],
+ chunk_request_body(Rest, ChunkSize, [Chunk | Acc]);
+chunk_request_body(Body, _ChunkSize, Acc) when is_binary(Body) ->
+ BodySize = size(Body),
+ Chunk = [?dec2hex(BodySize),"\r\n",
+ Body, "\r\n"],
+ LastChunk = "0\r\n",
+ lists:reverse(["\r\n", LastChunk, Chunk | Acc]);
+chunk_request_body(Body, ChunkSize, Acc) when length(Body) >= ChunkSize ->
+ {ChunkBody, Rest} = split_list_at(Body, ChunkSize),
+ Chunk = [?dec2hex(ChunkSize),"\r\n",
+ ChunkBody, "\r\n"],
+ chunk_request_body(Rest, ChunkSize, [Chunk | Acc]);
+chunk_request_body(Body, _ChunkSize, Acc) when is_list(Body) ->
+ BodySize = length(Body),
+ Chunk = [?dec2hex(BodySize),"\r\n",
+ Body, "\r\n"],
+ LastChunk = "0\r\n",
+ lists:reverse(["\r\n", LastChunk, Chunk | Acc]).
+
+split_list_at(List, N) ->
+ split_list_at(List, N, []).
+
+split_list_at([], _, Acc) ->
+ {lists:reverse(Acc), []};
+split_list_at(List2, 0, List1) ->
+ {lists:reverse(List1), List2};
+split_list_at([H | List2], N, List1) ->
+ split_list_at(List2, N-1, [H | List1]).
+
+to_lower(X) when is_atom(X) ->
+ list_to_atom(to_lower(atom_to_list(X)));
+to_lower(X) when is_list(X) ->
+ string:to_lower(X).
+
+get_content_length([{http_header, _, 'Content-Length', _, V} | _]) ->
+ list_to_integer(V);
+get_content_length([{http_header, _, _X, _, _Y} | T]) ->
+ get_content_length(T);
+get_content_length([]) ->
+ undefined.
diff --git a/test/ibrowse_tests.erl b/test/ibrowse_tests.erl
new file mode 100644
index 0000000..594b8ed
--- /dev/null
+++ b/test/ibrowse_tests.erl
@@ -0,0 +1,174 @@
+%%% File : ibrowse_tests.erl
+%%% Authors : Benjamin Lee <http://github.com/benjaminplee>
+%%% Dan Schwabe <http://github.com/dfschwabe>
+%%% Brian Richards <http://github.com/richbria>
+%%% Description : Functional tests of the ibrowse library using a live test HTTP server
+%%% Created : 18 November 2014 by Benjamin Lee <yardspoon@gmail.com>
+
+-module(ibrowse_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+-define(PER_TEST_TIMEOUT_SEC, 60).
+-define(TIMEDTEST(Desc, Fun), {Desc, {timeout, ?PER_TEST_TIMEOUT_SEC, fun Fun/0}}).
+
+-define(SERVER_PORT, 8181).
+-define(BASE_URL, "http://localhost:" ++ integer_to_list(?SERVER_PORT)).
+-define(SHORT_TIMEOUT_MS, 5000).
+-define(LONG_TIMEOUT_MS, 30000).
+-define(PAUSE_FOR_CONNECTIONS_MS, 2000).
+
+%%-compile(export_all).
+
+setup() ->
+ application:start(crypto),
+ application:start(public_key),
+ application:start(ssl),
+ ibrowse_test_server:start_server(?SERVER_PORT, tcp),
+ ibrowse:start(),
+ ok.
+
+teardown(_) ->
+ ibrowse:stop(),
+ ibrowse_test_server:stop_server(?SERVER_PORT),
+ ok.
+
+running_server_fixture_test_() ->
+ {foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ ?TIMEDTEST("Simple request can be honored", simple_request),
+ ?TIMEDTEST("Slow server causes timeout", slow_server_timeout),
+ ?TIMEDTEST("Pipeline depth goes down with responses", pipeline_depth),
+ ?TIMEDTEST("Pipelines refill", pipeline_refill),
+ ?TIMEDTEST("Timeout closes pipe", closing_pipes),
+ ?TIMEDTEST("Requests are balanced over connections", balanced_connections),
+ ?TIMEDTEST("Pipeline too small signals retries", small_pipeline),
+ ?TIMEDTEST("Dest status can be gathered", status)
+ ]
+ }.
+
+simple_request() ->
+ ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [])).
+
+slow_server_timeout() ->
+ ?assertMatch({error, req_timedout}, ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [], 5000)).
+
+pipeline_depth() ->
+ MaxSessions = 2,
+ MaxPipeline = 2,
+ RequestsSent = 2,
+ EmptyPipelineDepth = 0,
+
+ ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()),
+
+ Fun = fun() -> ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end,
+ times(RequestsSent, fun() -> spawn_link(Fun) end),
+
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS),
+
+ Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()],
+ ?assertEqual(MaxSessions, length(Counts)),
+ ?assertEqual(lists:duplicate(MaxSessions, EmptyPipelineDepth), Counts).
+
+pipeline_refill() ->
+ MaxSessions = 2,
+ MaxPipeline = 2,
+ RequestsToFill = MaxSessions * MaxPipeline,
+
+ %% Send off enough requests to fill sessions and pipelines in rappid succession
+ Fun = fun() -> ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end,
+ times(RequestsToFill, fun() -> spawn_link(Fun) end),
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS),
+
+ % Verify that connections properly reported their completed responses and can still accept more
+ ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS)),
+
+ % and do it again to make sure we really are clear
+ times(RequestsToFill, fun() -> spawn_link(Fun) end),
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS),
+
+ % Verify that connections properly reported their completed responses and can still accept more
+ ?assertMatch({ok, "200", _, _}, ibrowse:send_req(?BASE_URL, [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS)).
+
+closing_pipes() ->
+ MaxSessions = 2,
+ MaxPipeline = 2,
+ RequestsSent = 2,
+ BalancedNumberOfRequestsPerConnection = 1,
+
+ ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()),
+
+ Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end,
+ times(RequestsSent, fun() -> spawn_link(Fun) end),
+
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS),
+
+ Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()],
+ ?assertEqual(MaxSessions, length(Counts)),
+ ?assertEqual(lists:duplicate(MaxSessions, BalancedNumberOfRequestsPerConnection), Counts),
+
+ timer:sleep(?SHORT_TIMEOUT_MS),
+
+ ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()).
+
+balanced_connections() ->
+ MaxSessions = 4,
+ MaxPipeline = 100,
+ RequestsSent = 80,
+ BalancedNumberOfRequestsPerConnection = 20,
+
+ ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()),
+
+ Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?LONG_TIMEOUT_MS) end,
+ times(RequestsSent, fun() -> spawn_link(Fun) end),
+
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS),
+
+ Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()],
+ ?assertEqual(MaxSessions, length(Counts)),
+
+ ?assertEqual(lists:duplicate(MaxSessions, BalancedNumberOfRequestsPerConnection), Counts).
+
+small_pipeline() ->
+ MaxSessions = 10,
+ MaxPipeline = 10,
+ RequestsSent = 100,
+ FullRequestsPerConnection = 10,
+
+ ?assertEqual([], ibrowse_test_server:get_conn_pipeline_depth()),
+
+ Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end,
+ times(RequestsSent, fun() -> spawn(Fun) end),
+
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), %% Wait for everyone to get in line
+
+ ibrowse:show_dest_status("localhost", 8181),
+ Counts = [Count || {_Pid, Count} <- ibrowse_test_server:get_conn_pipeline_depth()],
+ ?assertEqual(MaxSessions, length(Counts)),
+
+ ?assertEqual(lists:duplicate(MaxSessions, FullRequestsPerConnection), Counts),
+
+ Response = ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS),
+
+ ?assertEqual({error, retry_later}, Response).
+
+status() ->
+ MaxSessions = 10,
+ MaxPipeline = 10,
+ RequestsSent = 100,
+
+ Fun = fun() -> ibrowse:send_req(?BASE_URL ++ "/never_respond", [], get, [], [{max_sessions, MaxSessions}, {max_pipeline_size, MaxPipeline}], ?SHORT_TIMEOUT_MS) end,
+ times(RequestsSent, fun() -> spawn(Fun) end),
+
+ timer:sleep(?PAUSE_FOR_CONNECTIONS_MS), %% Wait for everyone to get in line
+
+ ibrowse:show_dest_status(),
+ ibrowse:show_dest_status("http://localhost:8181").
+
+
+times(0, _) ->
+ ok;
+times(X, Fun) ->
+ Fun(),
+ times(X - 1, Fun).
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment