Proof of Concept: eXist DB module to commit files to github repository via github API

git-commit.xql

xquery version "3.0";

(:~ 
 : POC: eXist DB module to commit files to github reop via github API    
 : Prerequisites:
 : In order to run the module, you will need a github Authorization token.
 : When you create authorization token your OAuth Scope will need to include repo, allowing you to update files. 
 : @see https://github.com/blog/1509-personal-api-tokens
 :
 : @Notes 
 : Current version uses eXistdb httpclient as there were some issues POSTing json data using the EXpath http client.
 : It would be useful to iron out the issues, requires some additional testing. 
 : There is also a problem with how eXistdb handles base64 encoding. 
 : The suggested solution is to use EXpath http client.
 : @see http://exist.2174344.n4.nabble.com/Problem-converting-Base64Binary-data-to-raw-binary-data-for-Google-Drive-Api-td4665281.html
 :
 : Github API Steps for creating/updating repository contents:
 :   1. get the current commit object
 :   2. retrieve the tree it points to
 :   3. retrieve the content of the blob object that tree has for that particular file path
 :   4. change the content somehow and post a new blob object with that new content, getting a blob SHA back
 :   5. post a new tree object with that file path pointer replaced with your new blob SHA getting a tree SHA back
 :   6. create a new commit object with the current commit SHA as the parent and the new tree SHA, getting a commit SHA back
 :   7. update the reference of your branch to point to the new commit SHA
 :
 : Links to useful github api info
 : https://developer.github.com/v3/ 
 : https://developer.github.com/v3/git/
 : https://gist.github.com/jasonrudolph/10727108
 : http://www.mdswanson.com/blog/2011/07/23/digging-around-the-github-api-take-2.html
 : http://www.levibotelho.com/development/commit-a-file-with-the-github-api/
 : http://patrick-mckinley.com/tech/github-api-commit.html
 :
 : @author Winona Salesky
 : @version 0.1 
 :
 : @see https://github.com/joewiz/xqjson   
 : @see http://exist-db.org/xquery/util
 : @see http://exist-db.org/xquery/httpclient
 : 
 :)
 
import module namespace xqjson="http://xqilla.sourceforge.net/lib/xqjson";
import module namespace http="http://expath.org/ns/http-client";
declare namespace httpclient="http://exist-db.org/xquery/httpclient";

(: Pass updated file path and commit message via parameters :)
declare variable $path {request:get-parameter('path', 'README.md')};
declare variable $commit {request:get-parameter('commit', 'Update README. testing github API with parameters.')};

(:~
 : New content to submit 
 : @param $path path to updated file in eXist (relative to app root)
 : Should be passed with either, see notes on issue with this method:
 : util:base64-encode(doc($path))
 : util:base64-encode(util:binary-doc($path))
:)
let $new-content := 'Testing utf-8 text from xquery to github. Still problems with submitting binary files, or using expath http.'

(: Path to your github repository. eg: https://api.github.com/repos/wsalesky/blogs :)
let $repo := 'REPOSITORY-URL' 

(: 
 : You will need to have an github authorization token to run this script.
 : See https://developer.github.com/v3/oauth_authorizations/#create-a-new-authorization
 : Best practice would be to store this token in an environmental variable on 
 : your server and retrieve it via environment-variable()
:)
let $authorization-token := 'YOUR-AUTH-TOKEN'

(:~ 
: HTTP Headers for authorization
: @param $authorization-token your github authorization token
:)
let $headers := 
<httpclient:headers>
    <httpclient:header name="Authorization" value="{concat('token ',$authorization-token)}"/>
</httpclient:headers>

(: Get master branch of $repo :)
let $latest-commit := xqjson:parse-json(util:base64-decode(httpclient:get(xs:anyURI(concat($repo,'/git/refs/heads/master')),false(), $headers)/httpclient:body))
(: Get latest commit SHA from master branch  :)
let $get-latest-commit-sha := $latest-commit//pair[@name='sha']/text()

(: Get latest commit tree using $get-latest-commit-sha :)
let $get-latest-commit := xqjson:parse-json(util:base64-decode(httpclient:get(xs:anyURI(concat($repo,'/git/commits/',$get-latest-commit-sha)),false(), $headers)/httpclient:body))
(: Get latest commit tree SHA :) 
let $latest-commit-tree-sha := $get-latest-commit//pair[@name='tree']/pair[@name='sha']/text()

(: Get latest tree using $latest-commit-tree-sha :)
let $get-latest-tree := xqjson:parse-json(util:base64-decode(httpclient:get(xs:anyURI(concat($repo,'/git/trees/',$latest-commit-tree-sha)),false(), $headers)/httpclient:body))

(: Github documentation suggests getting the file SHA and content, but then never uses it. Currently commented out:)
(: Get file SHA from latest tree :)
(:let $get-file-sha-from-tree := $get-latest-tree//pair[@name='path'][. = {$path}]/following-sibling::pair[@name='sha'][1]/text():)

(: Get file blob from $get-file-sha-from-tree :)
(:
let $get-blob := xqjson:parse-json(util:base64-decode(httpclient:get(xs:anyURI(concat($repo,'/git/blobs/',$get-file-sha-from-tree)),false(), $headers)/httpclient:body))
let $blob-sha := $get-blob/pair[@name='sha']/text()
:)

(: Create new blob with new content base64/utf-8 :)
let $new-blob-content := 
xqjson:serialize-json(
    <pair name="object" type="object">
        <pair name="content" type="string">{$new-content}</pair>
        <pair name="encoding" type="string">utf-8</pair>
    </pair>)

(: Send new blob-content to Github API:)
let $new-blob := xqjson:parse-json(util:base64-decode(httpclient:post(xs:anyURI(concat($repo,'/git/blobs')), $new-blob-content, false(), $headers)/httpclient:body))    
(: New blob sha :)
let $new-blob-sha := $new-blob/pair[@name='sha']/text()

(: 
 : Create a new tree
 : New tree includes
 : base_tree: $latest-commit-tree-sha (the tree referenced in the latest commit, referenced in the master branch.)
 : path: path to the new/updated file relative to app/repo root.
 : sha: $new-blob-sha (SHA for the just created new content blob)
:)
let $new-tree-content := 
xqjson:serialize-json(
    <pair name="object" type="object">
        <pair name="base_tree" type="string">{$latest-commit-tree-sha}</pair>
        <pair name="tree" type="array">
            <pair name="object" type="object">
                <pair name="path" type="string">{$path}</pair>
                <pair name="mode" type="string">100644</pair>
                <pair name="type" type="string">blob</pair>
                <pair name="sha" type="string">{$new-blob-sha}</pair>
            </pair>
        </pair>
    </pair>)
(: Send new tree to Github API:)    
let $new-tree := xqjson:parse-json(util:base64-decode(httpclient:post(xs:anyURI(concat($repo,'/git/trees')), $new-tree-content, false(), $headers)/httpclient:body))    
(: New tree SHA :)    
let $new-tree-sha := $new-tree/pair[@name='sha']/text()

(: 
 : Create new commit 
 : message: commit message
 : tree: $new-tree-sha (the SHA of the tree you have just created for your new content)
 : parents: $get-latest-commit-sha (an array of parent SHA's, can be just one, should not be empty)
 :)
let $new-commit-content :=
xqjson:serialize-json(
    <pair name="object" type="object">
        <pair name="message" type="string">{$commit}</pair>
        <pair name="tree" type="string">{$new-tree-sha}</pair>
        <pair name="parents" type="array"><item type="string">{$get-latest-commit-sha}</item></pair>
    </pair>)
(: Send new commit to Github API:)  
let $new-commit := xqjson:parse-json(util:base64-decode(httpclient:post(xs:anyURI(concat($repo,'/git/commits')), $new-commit-content, false(), $headers)/httpclient:body))    
(: New commit SHA :)  
let $new-commit-sha := $new-commit/self::json[@type='object']/pair[@name='sha']/text()
    
(: Update refs in repository to your new commit SHA :)
let $commit-ref-data :=
xqjson:serialize-json(
    <pair name="object" type="object">
        <pair name="sha" type="string">{$new-commit-sha}</pair>
        <pair name="force" type="boolean">true</pair>
    </pair>)    
let $commit-ref := httpclient:post(xs:anyURI(concat($repo,'/git/refs/heads/master')), $commit-ref-data, false(), $headers)
let $commit-ref-message := xqjson:parse-json(util:base64-decode($commit-ref//httpclient:body))

(: Returns final commit message :)
return $commit-ref-message

Next steps are to investigate proper base64 content handling and posting json from EXPath http client.

Useful threads to follow up on:
http://comments.gmane.org/gmane.text.xml.basex.talk/5793
http://exist.2174344.n4.nabble.com/Problem-converting-Base64Binary-data-to-raw-binary-data-for-Google-Drive-Api-td4665281.html

Interested collaborators are welcome (encouraged, begged...) to fork and improve.