Skip to content

Instantly share code, notes, and snippets.

@Wilfred Wilfred/exco-url-http-24.5.el Secret
Created Jun 17, 2015

Embed
What would you like to do?
excorporate URL fetching on Emacs 24.5
;;; exco-url-http-24.5.el --- Override some HTTP retrieval routines
;; Copyright (C) 1999, 2001, 2004-2014 Free Software Foundation, Inc.
;; Author: Bill Perry <wmperry@gnu.org>
;; Keywords: comm, data, processes
;; This file is part of GNU Emacs.
;;
;; GNU Emacs is free software: you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.
;; GNU Emacs is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
;;; Commentary:
;; Override url-http-parse-headers to clear Authorization elements
;; from url-http-extra-headers prior to executing a redirect. The
;; only change is to apply this backward-compatible patch:
;;
;; diff --git a/lisp/url/url-http.el b/lisp/url/url-http.el
;; index b0a3b68..80d9cca 100644
;; --- a/lisp/url/url-http.el
;; +++ b/lisp/url/url-http.el
;; @@ -617,6 +617,15 @@ should be shown to the user."
;; ;; compute the redirection relative to the URL of the proxy.
;; (setq redirect-uri
;; (url-expand-file-name redirect-uri url-http-target-url)))
;; + ;; Don't automatically include authorization header in redirect.
;; + ;; If needed it will be regenerated by the relevant auth scheme
;; + ;; when the new request happens.
;; + (setq url-http-extra-headers
;; + (let (result)
;; + (dolist (header url-http-extra-headers)
;; + (if (not (equal (car header) "Authorization"))
;; + (push header result)))
;; + (nreverse result)))
;; (let ((url-request-method url-http-method)
;; (url-request-data url-http-data)
;; (url-request-extra-headers url-http-extra-headers))
;;; Code:
(eval-when-compile (require 'cl))
(require 'url-gw)
(require 'url-parse)
(require 'url-cookie)
(require 'mail-parse)
(require 'url-auth)
(require 'url)
(autoload 'url-cache-create-filename "url-cache")
(require 'url-http)
(defun url-http-parse-headers ()
"Parse and handle HTTP specific headers.
Return t if and only if the current buffer is still active and
should be shown to the user."
;; The comments after each status code handled are taken from RFC
;; 2616 (HTTP/1.1)
(url-http-mark-connection-as-free (url-host url-current-object)
(url-port url-current-object)
url-http-process)
(if (or (not (boundp 'url-http-end-of-headers))
(not url-http-end-of-headers))
(error "Trying to parse headers in odd buffer: %s" (buffer-name)))
(goto-char (point-min))
(url-http-debug "url-http-parse-headers called in (%s)" (buffer-name))
(url-http-parse-response)
(mail-narrow-to-head)
;;(narrow-to-region (point-min) url-http-end-of-headers)
(let ((connection (mail-fetch-field "Connection")))
;; In HTTP 1.0, keep the connection only if there is a
;; "Connection: keep-alive" header.
;; In HTTP 1.1 (and greater), keep the connection unless there is a
;; "Connection: close" header
(cond
((string= url-http-response-version "1.0")
(unless (and connection
(string= (downcase connection) "keep-alive"))
(delete-process url-http-process)))
(t
(when (and connection
(string= (downcase connection) "close"))
(delete-process url-http-process)))))
(let* ((buffer (current-buffer))
(class (/ url-http-response-status 100))
(success nil)
;; other status symbols: jewelry and luxury cars
(status-symbol (cadr (assq url-http-response-status url-http-codes))))
(url-http-debug "Parsed HTTP headers: class=%d status=%d"
class url-http-response-status)
(when (url-use-cookies url-http-target-url)
(url-http-handle-cookies))
(pcase class
;; Classes of response codes
;;
;; 5xx = Server Error
;; 4xx = Client Error
;; 3xx = Redirection
;; 2xx = Successful
;; 1xx = Informational
(1 ; Information messages
;; 100 = Continue with request
;; 101 = Switching protocols
;; 102 = Processing (Added by DAV)
(url-mark-buffer-as-dead buffer)
(error "HTTP responses in class 1xx not supported (%d)"
url-http-response-status))
(2 ; Success
;; 200 Ok
;; 201 Created
;; 202 Accepted
;; 203 Non-authoritative information
;; 204 No content
;; 205 Reset content
;; 206 Partial content
;; 207 Multi-status (Added by DAV)
(pcase status-symbol
((or `no-content `reset-content)
;; No new data, just stay at the same document
(url-mark-buffer-as-dead buffer))
(_
;; Generic success for all others. Store in the cache, and
;; mark it as successful.
(widen)
(if (and url-automatic-caching (equal url-http-method "GET"))
(url-store-in-cache buffer))))
(setq success t))
(3 ; Redirection
;; 300 Multiple choices
;; 301 Moved permanently
;; 302 Found
;; 303 See other
;; 304 Not modified
;; 305 Use proxy
;; 307 Temporary redirect
(let ((redirect-uri (or (mail-fetch-field "Location")
(mail-fetch-field "URI"))))
(pcase status-symbol
(`multiple-choices ; 300
;; Quoth the spec (section 10.3.1)
;; -------------------------------
;; The requested resource corresponds to any one of a set of
;; representations, each with its own specific location and
;; agent-driven negotiation information is being provided so
;; that the user can select a preferred representation and
;; redirect its request to that location.
;; [...]
;; If the server has a preferred choice of representation, it
;; SHOULD include the specific URI for that representation in
;; the Location field; user agents MAY use the Location field
;; value for automatic redirection.
;; -------------------------------
;; We do not support agent-driven negotiation, so we just
;; redirect to the preferred URI if one is provided.
nil)
((or `moved-permanently `found `temporary-redirect) ; 301 302 307
;; If the 301|302 status code is received in response to a
;; request other than GET or HEAD, the user agent MUST NOT
;; automatically redirect the request unless it can be
;; confirmed by the user, since this might change the
;; conditions under which the request was issued.
(unless (member url-http-method '("HEAD" "GET"))
(setq redirect-uri nil)))
(`see-other ; 303
;; The response to the request can be found under a different
;; URI and SHOULD be retrieved using a GET method on that
;; resource.
(setq url-http-method "GET"
url-http-data nil))
(`not-modified ; 304
;; The 304 response MUST NOT contain a message-body.
(url-http-debug "Extracting document from cache... (%s)"
(url-cache-create-filename (url-view-url t)))
(url-cache-extract (url-cache-create-filename (url-view-url t)))
(setq redirect-uri nil
success t))
(`use-proxy ; 305
;; The requested resource MUST be accessed through the
;; proxy given by the Location field. The Location field
;; gives the URI of the proxy. The recipient is expected
;; to repeat this single request via the proxy. 305
;; responses MUST only be generated by origin servers.
(error "Redirection thru a proxy server not supported: %s"
redirect-uri))
(_
;; Treat everything like '300'
nil))
(when redirect-uri
;; Clean off any whitespace and/or <...> cruft.
(if (string-match "\\([^ \t]+\\)[ \t]" redirect-uri)
(setq redirect-uri (match-string 1 redirect-uri)))
(if (string-match "^<\\(.*\\)>$" redirect-uri)
(setq redirect-uri (match-string 1 redirect-uri)))
;; Some stupid sites (like sourceforge) send a
;; non-fully-qualified URL (ie: /), which royally confuses
;; the URL library.
(if (not (string-match url-nonrelative-link redirect-uri))
;; Be careful to use the real target URL, otherwise we may
;; compute the redirection relative to the URL of the proxy.
(setq redirect-uri
(url-expand-file-name redirect-uri url-http-target-url)))
;; Don't automatically include authorization header in redirect.
;; If needed it will be regenerated by the relevant auth scheme
;; when the new request happens.
(setq url-http-extra-headers
(let (result)
(dolist (header url-http-extra-headers)
(if (not (equal (car header) "Authorization"))
(push header result)))
(nreverse result)))
(let ((url-request-method url-http-method)
(url-request-data url-http-data)
(url-request-extra-headers url-http-extra-headers))
;; Check existing number of redirects
(if (or (< url-max-redirections 0)
(and (> url-max-redirections 0)
(let ((events (car url-callback-arguments))
(old-redirects 0))
(while events
(if (eq (car events) :redirect)
(setq old-redirects (1+ old-redirects)))
(and (setq events (cdr events))
(setq events (cdr events))))
(< old-redirects url-max-redirections))))
;; url-max-redirections hasn't been reached, so go
;; ahead and redirect.
(progn
;; Remember that the request was redirected.
(setf (car url-callback-arguments)
(nconc (list :redirect redirect-uri)
(car url-callback-arguments)))
;; Put in the current buffer a forwarding pointer to the new
;; destination buffer.
;; FIXME: This is a hack to fix url-retrieve-synchronously
;; without changing the API. Instead url-retrieve should
;; either simply not return the "destination" buffer, or it
;; should take an optional `dest-buf' argument.
(set (make-local-variable 'url-redirect-buffer)
(url-retrieve-internal
redirect-uri url-callback-function
url-callback-arguments
(url-silent url-current-object)
(not (url-use-cookies url-current-object))))
(url-mark-buffer-as-dead buffer))
;; We hit url-max-redirections, so issue an error and
;; stop redirecting.
(url-http-debug "Maximum redirections reached")
(setf (car url-callback-arguments)
(nconc (list :error (list 'error 'http-redirect-limit
redirect-uri))
(car url-callback-arguments)))
(setq success t))))))
(4 ; Client error
;; 400 Bad Request
;; 401 Unauthorized
;; 402 Payment required
;; 403 Forbidden
;; 404 Not found
;; 405 Method not allowed
;; 406 Not acceptable
;; 407 Proxy authentication required
;; 408 Request time-out
;; 409 Conflict
;; 410 Gone
;; 411 Length required
;; 412 Precondition failed
;; 413 Request entity too large
;; 414 Request-URI too large
;; 415 Unsupported media type
;; 416 Requested range not satisfiable
;; 417 Expectation failed
;; 422 Unprocessable Entity (Added by DAV)
;; 423 Locked
;; 424 Failed Dependency
(setq success
(pcase status-symbol
(`unauthorized ; 401
;; The request requires user authentication. The response
;; MUST include a WWW-Authenticate header field containing a
;; challenge applicable to the requested resource. The
;; client MAY repeat the request with a suitable
;; Authorization header field.
(url-http-handle-authentication nil))
(`payment-required ; 402
;; This code is reserved for future use
(url-mark-buffer-as-dead buffer)
(error "Somebody wants you to give them money"))
(`forbidden ; 403
;; The server understood the request, but is refusing to
;; fulfill it. Authorization will not help and the request
;; SHOULD NOT be repeated.
t)
(`not-found ; 404
;; Not found
t)
(`method-not-allowed ; 405
;; The method specified in the Request-Line is not allowed
;; for the resource identified by the Request-URI. The
;; response MUST include an Allow header containing a list of
;; valid methods for the requested resource.
t)
(`not-acceptable ; 406
;; The resource identified by the request is only capable of
;; generating response entities which have content
;; characteristics not acceptable according to the accept
;; headers sent in the request.
t)
(`proxy-authentication-required ; 407
;; This code is similar to 401 (Unauthorized), but indicates
;; that the client must first authenticate itself with the
;; proxy. The proxy MUST return a Proxy-Authenticate header
;; field containing a challenge applicable to the proxy for
;; the requested resource.
(url-http-handle-authentication t))
(`request-timeout ; 408
;; The client did not produce a request within the time that
;; the server was prepared to wait. The client MAY repeat
;; the request without modifications at any later time.
t)
(`conflict ; 409
;; The request could not be completed due to a conflict with
;; the current state of the resource. This code is only
;; allowed in situations where it is expected that the user
;; might be able to resolve the conflict and resubmit the
;; request. The response body SHOULD include enough
;; information for the user to recognize the source of the
;; conflict.
t)
(`gone ; 410
;; The requested resource is no longer available at the
;; server and no forwarding address is known.
t)
(`length-required ; 411
;; The server refuses to accept the request without a defined
;; Content-Length. The client MAY repeat the request if it
;; adds a valid Content-Length header field containing the
;; length of the message-body in the request message.
;;
;; NOTE - this will never happen because
;; `url-http-create-request' automatically calculates the
;; content-length.
t)
(`precondition-failed ; 412
;; The precondition given in one or more of the
;; request-header fields evaluated to false when it was
;; tested on the server.
t)
((or `request-entity-too-large `request-uri-too-large) ; 413 414
;; The server is refusing to process a request because the
;; request entity|URI is larger than the server is willing or
;; able to process.
t)
(`unsupported-media-type ; 415
;; The server is refusing to service the request because the
;; entity of the request is in a format not supported by the
;; requested resource for the requested method.
t)
(`requested-range-not-satisfiable ; 416
;; A server SHOULD return a response with this status code if
;; a request included a Range request-header field, and none
;; of the range-specifier values in this field overlap the
;; current extent of the selected resource, and the request
;; did not include an If-Range request-header field.
t)
(`expectation-failed ; 417
;; The expectation given in an Expect request-header field
;; could not be met by this server, or, if the server is a
;; proxy, the server has unambiguous evidence that the
;; request could not be met by the next-hop server.
t)
(_
;; The request could not be understood by the server due to
;; malformed syntax. The client SHOULD NOT repeat the
;; request without modifications.
t)))
;; Tell the callback that an error occurred, and what the
;; status code was.
(when success
(setf (car url-callback-arguments)
(nconc (list :error (list 'error 'http url-http-response-status))
(car url-callback-arguments)))))
(5
;; 500 Internal server error
;; 501 Not implemented
;; 502 Bad gateway
;; 503 Service unavailable
;; 504 Gateway time-out
;; 505 HTTP version not supported
;; 507 Insufficient storage
(setq success t)
(pcase url-http-response-status
(`not-implemented ; 501
;; The server does not support the functionality required to
;; fulfill the request.
nil)
(`bad-gateway ; 502
;; The server, while acting as a gateway or proxy, received
;; an invalid response from the upstream server it accessed
;; in attempting to fulfill the request.
nil)
(`service-unavailable ; 503
;; The server is currently unable to handle the request due
;; to a temporary overloading or maintenance of the server.
;; The implication is that this is a temporary condition
;; which will be alleviated after some delay. If known, the
;; length of the delay MAY be indicated in a Retry-After
;; header. If no Retry-After is given, the client SHOULD
;; handle the response as it would for a 500 response.
nil)
(`gateway-timeout ; 504
;; The server, while acting as a gateway or proxy, did not
;; receive a timely response from the upstream server
;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
;; auxiliary server (e.g. DNS) it needed to access in
;; attempting to complete the request.
nil)
(`http-version-not-supported ; 505
;; The server does not support, or refuses to support, the
;; HTTP protocol version that was used in the request
;; message.
nil)
(`insufficient-storage ; 507 (DAV)
;; The method could not be performed on the resource
;; because the server is unable to store the representation
;; needed to successfully complete the request. This
;; condition is considered to be temporary. If the request
;; which received this status code was the result of a user
;; action, the request MUST NOT be repeated until it is
;; requested by a separate user action.
nil))
;; Tell the callback that an error occurred, and what the
;; status code was.
(when success
(setf (car url-callback-arguments)
(nconc (list :error (list 'error 'http url-http-response-status))
(car url-callback-arguments)))))
(_
(error "Unknown class of HTTP response code: %d (%d)"
class url-http-response-status)))
(if (not success)
(url-mark-buffer-as-dead buffer)
(url-handle-content-transfer-encoding))
(url-http-debug "Finished parsing HTTP headers: %S" success)
(widen)
(goto-char (point-min))
success))
(provide 'exco-url-http-24.5)
;;; exco-url-http-24.5.el ends here
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.