Skip to content

Instantly share code, notes, and snippets.

@joewiz
Last active November 1, 2018 15:32
Show Gist options
  • Star 4 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save joewiz/5929809 to your computer and use it in GitHub Desktop.
Save joewiz/5929809 to your computer and use it in GitHub Desktop.
Access OAuth 1.0-based services like the Twitter v1.1 API, with XQuery. (See comments below for explanation.)
xquery version "3.0";
module namespace oauth="http://history.state.gov/ns/xquery/oauth";
(:~ A library module for signing and submitting OAuth requests such as the kind needed for the Twitter v1.1 API.
The EXPath Crypto library supplies the HMAC-SHA1 algorithm. The EXPath HTTP Client library makes the HTTP requests.
The OAuth standard requires a "nonce" parameter - a random string. Since there is no implementation-independent
nonce function in XQuery, we must rely on implementation-specific functions. For eXist-db we use util:uuid().
@see http://oauth.net/core/1.0/
@see http://tools.ietf.org/html/rfc5849
@see http://marktrapp.com/blog/2009/09/17/oauth-dummies
@see http://expath.org/spec/http-client
@see http://expath.org/spec/crypto
@see http://exist-db.org/exist/apps/fundocs/view.html?uri=http://exist-db.org/xquery/util
:)
import module namespace crypto="http://expath.org/ns/crypto";
import module namespace http="http://expath.org/ns/http-client";
import module namespace util = "http://exist-db.org/xquery/util";
declare function oauth:send-request(
$consumer-key,
$consumer-secret,
$access-token,
$access-token-secret,
$method,
$url,
$nonce,
$signature-method,
$timestamp,
$version
) {
let $base-url := if (contains($url, '?')) then substring-before($url, '?') else $url
let $query-string := if (contains($url, '?')) then substring-after($url, '?') else ()
let $query-string-params :=
for $param in tokenize($query-string, '&')
let $name := substring-before($param, '=')
let $value := substring-after($param, '=')
return
<param name="{$name}" value="{$value}"/>
let $params :=
(
$query-string-params,
<param name="oauth_consumer_key" value="{$consumer-key}"/>,
<param name="oauth_nonce" value="{$nonce}"/>,
<param name="oauth_signature_method" value="{$signature-method}"/>,
<param name="oauth_timestamp" value="{$timestamp}"/>,
<param name="oauth_token" value="{$access-token}"/>,
<param name="oauth_version" value="{$version}"/>
)
let $parameter-string := oauth:params-to-oauth-string($params, '&amp;')
let $signature-base-string :=
string-join(
(
upper-case($method),
encode-for-uri($base-url),
encode-for-uri($parameter-string)
)
,
'&amp;'
)
let $signing-key := concat(encode-for-uri($consumer-secret), '&amp;', encode-for-uri($access-token-secret))
let $oauth-signature := crypto:hmac($signature-base-string, $signing-key, 'HmacSha1', 'base64')
let $final-params :=
(
$params,
<param name="oauth_signature" value="{$oauth-signature}"/>
)
let $final-parameter-string := oauth:params-to-oauth-string($final-params, ', ')
let $authorization-header-value := concat('OAuth ', $final-parameter-string)
let $request :=
<http:request href="{$url}" method="{$method}">
<http:header name="Authorization" value="{$authorization-header-value}"/>
</http:request>
let $response := http:send-request($request)
return
(
$request
,
$response
)
};
declare function oauth:nonce() { util:uuid() };
declare variable $oauth:signature-method := 'HMAC-SHA1';
declare variable $oauth:oauth-version := '1.0';
(: Generates an OAuth timestamp, which takes the form of the number of seconds since the Unix Epoch.
You can test these values against http://www.epochconverter.com/.
@see http://en.wikipedia.org/wiki/Unix_time
:)
declare function oauth:timestamp() as xs:unsignedLong {
let $unix-epoch := xs:dateTime('1970-01-01T00:00:00Z')
let $now := current-dateTime()
let $duration-since-epoch := $now - $unix-epoch
let $seconds-since-epoch :=
days-from-duration($duration-since-epoch) * 86400 (: 60 * 60 * 24 :)
+
hours-from-duration($duration-since-epoch) * 3600 (: 60 * 60 :)
+
minutes-from-duration($duration-since-epoch) * 60
+
seconds-from-duration($duration-since-epoch)
return
xs:unsignedLong($seconds-since-epoch)
};
(: prepares OAuth authentication parameters :)
declare function oauth:params-to-oauth-string($params as element(param)+, $separator as xs:string) {
string-join(
for $param in $params
let $name := encode-for-uri($param/@name)
let $value := encode-for-uri($param/@value)
order by $name, $value
return
concat($name, '=', $value)
,
$separator
)
};
xquery version "3.1";
module namespace twitter-client = "http://history.state.gov/ns/xquery/twitter-client";
(:~ A library module for your application's Twitter credentials and any helper functions for processing
the raw results of Twitter requests. You can get your credentials from https://dev.twitter.com/apps/.
Twitter responds with JSON; despite being text, the HTTP Client returns JSON as binary, so we need
util:binary-to-text() to get the text. We use XQuery 3.1 to turn the JSON into XML.
@see http://exist-db.org/exist/apps/fundocs/view.html?uri=http://exist-db.org/xquery/util
@see http://github.com/joewiz/xqjson
:)
import module namespace twitter = "http://history.state.gov/ns/xquery/twitter" at "twitter.xq";
import module namespace util = "http://exist-db.org/xquery/util";
import module namespace ju = "http://joewiz.org/ns/xquery/json-util" at "https://gist.githubusercontent.com/joewiz/d986da715facaad633db/raw/12692fcd025e4a0572d6b7638577fa41f4a8166a/json-util.xqm";
declare variable $twitter-client:consumer-key := ''; (: insert your credentials :)
declare variable $twitter-client:consumer-secret := '';
declare variable $twitter-client:access-token := '';
declare variable $twitter-client:access-token-secret := '';
declare function twitter-client:echo-response($request-response as item()+) {
let $request := $request-response[1]
let $response-head := $request-response[2]
let $response-body := $request-response[3]
let $json := parse-json(util:binary-to-string($response-body))
let $xml := ju:json-to-xml($json)
return
(
$request,
$response-head,
ju:serialize-json($json),
$xml
)
};
xquery version "3.0";
module namespace twitter="http://history.state.gov/ns/xquery/twitter";
(:~ A library module for Twitter API methods.
@see https://dev.twitter.com/docs/api/1.1
:)
import module namespace oauth="http://history.state.gov/ns/xquery/oauth" at "oauth.xq";
declare variable $twitter:api-base-uri := 'https://api.twitter.com/1.1';
(:
Get the user timeline.
See https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline
:)
declare function twitter:user-timeline(
$consumer-key as xs:string,
$consumer-secret as xs:string,
$access-token as xs:string,
$access-token-secret as xs:string,
$user-id as xs:string?,
$screen-name as xs:string?,
$since-id as xs:unsignedLong?, (: IDs are too big for xs:integer :)
$count as xs:integer?,
$max-id as xs:unsignedLong?,
$trim-user as xs:boolean?,
$exclude-replies as xs:boolean?,
$contributor-details as xs:boolean?,
$include-rts as xs:boolean?
) {
let $api-method := '/statuses/user_timeline.json'
let $http-method := 'GET'
let $query-string :=
string-join(
(
if ($user-id) then concat('user_id=', $user-id) else (),
if ($screen-name) then concat('screen_name=', $screen-name) else (),
if ($since-id) then concat('since_id=', $since-id) else (),
if ($count) then concat('count=', $count) else (),
if ($max-id) then concat('max_id=', $max-id) else (),
if ($trim-user) then concat('trim_user=', $trim-user) else (),
if ($exclude-replies) then concat('exclude_replies=', $exclude-replies) else (),
if ($contributor-details) then concat('contributor_details=', $contributor-details) else (),
if ($include-rts) then concat('include_rts=', $include-rts) else ()
),
'&amp;'
)
let $api-url := concat($twitter:api-base-uri, $api-method, '?', $query-string)
return
oauth:send-request(
$consumer-key,
$consumer-secret,
$access-token,
$access-token-secret,
$http-method,
$api-url,
oauth:nonce(),
$oauth:signature-method,
oauth:timestamp(),
$oauth:oauth-version
)
};
xquery version "3.0";
(:~ A main module to retrieve the user timeline. :)
import module namespace twitter = "http://history.state.gov/ns/xquery/twitter" at "twitter.xq";
import module namespace twitter-client = "http://history.state.gov/ns/xquery/twitter-client" at "twitter-client.xq";
(: Parameters needed for the user timeline function. :)
let $user-id := ()
let $screen-name := ()
let $since-id := ()
let $count := 10
let $max-id := ()
let $trim-user := true()
let $exclude-replies := false()
let $contributor-details := false()
let $include-rts := false()
let $request-response :=
twitter:user-timeline(
$twitter-client:consumer-key,
$twitter-client:consumer-secret,
$twitter-client:access-token,
$twitter-client:access-token-secret,
$user-id,
$screen-name,
$since-id,
$count,
$max-id,
$trim-user,
$exclude-replies,
$contributor-details,
$include-rts
)
return
(: Echo the response so we can see the request, the response header, the response body (JSON), and the XML version of the JSON :)
twitter-client:echo-response($request-response)
@joewiz
Copy link
Author

joewiz commented Jul 4, 2013

Making OAuth requests with XQuery

The sample code here is intended to help get you started making requests to OAuth-based services like Twitter. The core is "oauth.xq", a library module that signing and submitting OAuth requests. I like to separate put all of my API's methods in a single library module: see "twitter.xq." I put application- and implementation-specific information like access credentials or functions for processing the raw results of requests in a separate library module (see "twitter-client.xq"). With all of these pieces in place, making an actual request is simple: define the request parameters, grab the credentials, and make the signed request (see "user-timeline.xq").

  1. oauth.xq A library module for signing and submitting OAuth requests such as the kind needed for the Twitter v1.1 API
  2. twitter.xq A library module with functions for each method defined by the Twitter API (here, I just have user-timeline.)
  3. twitter-client.xq A library module for your application's Twitter credentials and any helper functions for processing the raw results of Twitter requests
  4. user-timeline.xq A main module for retrieving the user timeline

Background on OAuth

There are two steps to working with OAuth-based services like the Twitter API: first, you need to get basic credentials for the service (the "consumer key" and "consumer secret") and an "access token" (not to be confused with a "request token"), and second you can start signing your requests with your credentials.

Getting credentials Every service will tell you your consumer key and consumer secret up front, but getting the access token is either simple or complicated. Some services like Twitter will let you generate an access token on their site, which you can then copy and paste into your code - very simple. But if the service you're working with doesn't let you generate an access token on their site, you need to perform a rather complex handshake - first requesting a request token and signing it with your consumer secret code. Frankly, I've had trouble getting this to work with XQuery; I'd like to get it to work but haven't been able to. Luckily, Twitter issues access tokens through their site.

Signing requests Once you have your credentials, you're ready to begin making requests. This involves constructing requests and signing them. You need to consult your service's API documentation to find out how to construct your requests properly, but once you have your request constructed (see "twitter.xq"), you need to sign them with an OAuth signature. The steps for signing OAuth requests are a little tricky, involving specific ways of ordering and encoding parameters and hashing the results with a cryptographic algorithm, all before submitting the request.

@joewiz
Copy link
Author

joewiz commented Jul 6, 2013

See also my post: Living in an OAuth & JSON World.

@joewiz
Copy link
Author

joewiz commented Nov 25, 2015

Updated for use with current release of EXPath Crypto module for eXist (0.3.x; needed for versions released since February 2014) and for use with XQuery 3.1's support.

@mathias-goebel
Copy link

nice work, thank you for sharing. is there a chance using the twitter client without the crypto module?
i wonder why u are calculating the timestamp on your own ( oauth:timestamp() ) instead of using datetime:timestamp() which returns millisec.

xquery version "3.0";

    let $unix-epoch := xs:dateTime('1970-01-01T00:00:00Z')
    let $now := current-dateTime()
    let $duration-since-epoch := $now - $unix-epoch
    let $seconds-since-epoch :=
        days-from-duration($duration-since-epoch) * 86400 (: 60 * 60 * 24 :)
        +
        hours-from-duration($duration-since-epoch) * 3600 (: 60 * 60 :)
        +
        minutes-from-duration($duration-since-epoch) * 60
        +
        seconds-from-duration($duration-since-epoch)
    return
        xs:unsignedLong($seconds-since-epoch),


let $time := datetime:timestamp()
return xs:unsignedLong( substring($time, 0, string-length($time)-2 ) )

@joewiz
Copy link
Author

joewiz commented Aug 29, 2016

Sorry @mathias-goebel, I didn't see your comment until now. Since twitter requires that all requests to its API be signed, we do need some crypto library for HMAC-SHA1. I don't think I was aware of the datetime:timestamp() function (an eXist-specific function), but I generally prefer to use implementation independent functions when possible.

@priyamenokey
Copy link

Hi ,
I have an angular application sending azure auth token to marklogic .Is there a way to authenticate the token in xquery .I have done the same in a Java app and it has client supported libraries .

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment