Skip to content

Instantly share code, notes, and snippets.

@zadean

zadean/xqerl_couch_query_server.md

Last active February 15, 2020 18:57
Show Gist options
  • Save zadean/1f75901f09a2aa2ba482a2ced00b2a63 to your computer and use it in GitHub Desktop.
Save zadean/1f75901f09a2aa2ba482a2ced00b2a63 to your computer and use it in GitHub Desktop.
Download ZIP
Using xqerl as a CouchDB Query Server
Raw
xqerl_couch_query_server.md

Using xqerl as a CouchDB Query Server

Assumptions:

  • xqerl and CouchDB are on the same machine.
  • The IP address of the machine is 192.168.178.36.
  • CouchDB is using port 5984.
  • xqerl and CouchDB are running on different Erlang nodes.
  • Both nodes are using the same cookie.
  • The CouchDB node name is couchdb@192.168.178.36 (set in /etc/vm.args of the CouchDB directory).
  • The xqerl node name is xqerl@192.168.178.36.
  • Native Query Servers is enabled in local.ini ([native_query_servers] enable_erlang_query_server = true).
  • The name of the Couch DB database is 'testDb'.

Link the nodes:

Call net_adm:ping('couchdb@192.168.178.36'). from the node running xqerl. This should return pong. If pang is returned, then the nodes are not using the correct names or have a different cookie.

Compile transformations in xqerl:

From the xqerl node, call: xqerl:compile("transform.xq"). xqerl:compile("transform_json.xq").

The returned values are needed as the value of Module in the rpc calls later!

They will look something like 'file____etc_transform_json.xq' depending on the file locations.

Contents of transform.xq

declare variable $data external;
declare variable $stamp external;
declare variable $value external;

let $json := parse-json($data)
let $stuff := $json('stuff')
return
map{
  'item_1' : $stuff(1),
  'item_2' : $stuff(2),
  'item_3' : $stuff(3),
  'item_4' : $stuff(4),
  'stamp'  : $stamp,
  'value'  : $value
}
=> serialize(map{'method':'json'})

Contents of transform_json.xq

declare variable $data external;

$data
  => json-to-xml()
  => serialize(map{'method':'xml'})

Set up a view

Add the following Design Doc to the database in CouchDB that is to be used.

{
  "_id": "_design/xqerl_design",
  "views": {
    "map-view": {
      "map": "fun({Doc}) ->\r\n    <<K,_/binary>> = proplists:get_value(<<\"_rev\">>, Doc, null),\r\n    Data = proplists:get_value(<<\"data\">>, Doc, null),\r\n    Stamp = proplists:get_value(<<\"stamp\">>, Doc, null), \r\n    Value = proplists:get_value(<<\"value\">>, Doc, null),\r\n    JsonData = jiffy:encode(Data),\r\n    Node = 'xqerl@192.168.178.36',\r\n    Module = 'file____git_zadean_xqerl_test_restxq_transform.xq',\r\n    Ctx = #{<<\"data\">> => JsonData,\r\n            <<\"stamp\">> => Stamp,\r\n            <<\"value\">> => Value},\r\n    V = rpc:call(Node, Module, main, [Ctx]), \r\n    Emit(<<K>>, jiffy:decode(V))\r\nend.\r\n"
    }
  },
  "lists": {
    "json_to_xml": "fun(Head, {Req}) -> \n    Start({[{<<\"headers\">>, {[{<<\"Content-Type\">>, <<\"application/xml\">>}]}}]}), \n    Fun = fun({Row}, Acc) ->\n            Val = couch_util:get_value(<<\"value\">>, Row),\n            Json = jiffy:encode(Val),\n            {ok, <<Acc/binary, \",\",Json/binary>>}\n          end,\n    {ok, <<\",\", Out/binary>>} = FoldRows(Fun, <<>>),\n    Out1 = <<\"[\", Out/binary, \"]\">>,\n    Node = 'xqerl@192.168.178.36',\n    Module = 'file____git_zadean_xqerl_test_restxq_transform_json.xq',\n    Ctx = #{<<\"data\">> => Out1},\n    V = rpc:call(Node, Module, main, [Ctx]),\n    Send(V),\n    []\nend."
  },
  "language": "erlang"
}

Formatted and commented for clarity

% map function for the view
fun({Doc}) ->
    % key for this view record
    <<K,_/binary>> = proplists:get_value(<<"_rev">>, Doc, null),
    % pull data, stamp, and value from the first level of the document
    Data = proplists:get_value(<<"data">>, Doc, null),
    Stamp = proplists:get_value(<<"stamp">>, Doc, null),
    Value = proplists:get_value(<<"value">>, Doc, null),
    % encode the data field back to a JSON string
    JsonData = jiffy:encode(Data),
    % the xqerl node
    Node = 'xqerl@192.168.178.36',
    % the compiled main-module name for this transform
    Module = 'file____git_zadean_xqerl_test_restxq_transform.xq',
    % set the external variables
    Ctx = #{<<"data">> => JsonData,
            <<"stamp">> => Stamp,
            <<"value">> => Value},
    % call the xqerl XQuery module
    V = rpc:call(Node, Module, main, [Ctx]), 
    % emit the returned value
    Emit(<<K>>, jiffy:decode(V))
end.

% list function that can further transform the view
fun(Head, {Req}) -> 
    % set the output header to XML
    Start({[{<<"headers">>, {[{<<"Content-Type">>, <<"application/xml">>}]}}]}), 
    % iterate the rows and join to a comma-separated list
    Fun = fun({Row}, Acc) ->
            Val = couch_util:get_value(<<"value">>, Row),
            Json = jiffy:encode(Val),
            {ok, <<Acc/binary, ",",Json/binary>>}
          end,
    % drop the leading comma
    {ok, <<",", Out/binary>>} = FoldRows(Fun, <<>>),
    % add array brackets to new large JSON
    Out1 = <<"[", Out/binary, "]">>,
    % set up the rpc call as above
    Node = 'xqerl@192.168.178.36',
    Module = 'file____git_zadean_xqerl_test_restxq_transform_json.xq',
    Ctx = #{<<"data">> => Out1},
    % get the transformed data, this is XML
    V = rpc:call(Node, Module, main, [Ctx]), 
    % send the results to the caller
    Send(V),
    []
end.

See what comes out

Compile and run the following XQuery to insert a few documents and call views that transform them.

declare namespace _ = 'http://xqerl.org/couchdb';

declare variable $_:DB := 'testDb';
declare variable $_:ROOT := 'http://192.168.178.36:5984/' || $_:DB;

declare function _:as-json($v)
{
  $v => serialize(map{'method' : 'json'})
};

declare function _:http-put($path, $content)
{
  http:send-request(
    <http:request method='put'>
      <http:body media-type='application/json'/>
    </http:request>,
    $_:ROOT || $path,
    $content => _:as-json()
  )
};

declare function _:put-doc($id, $content)
{
  let $path := '/' || $id
  return
  _:http-put($path, $content)
};

declare function _:get-view($path)
{
  http:send-request(
    <http:request method='get'/>,
    $_:ROOT||$path)
};

declare function _:dummy-doc()
{
  map{
    'stamp' : random:integer(99999999),
    'value' : random:integer(),
    'data'  : map{'stuff' : [random:integer(99999999), true(), 2e1, 'string']}
  }
};

declare function _:dummy-insert()
{
  for $i in (1 to 5)
  let $c := random:integer()
    , $d := _:dummy-doc()
  return
    _:put-doc($c, $d)
};

_:dummy-insert(),
_:get-view('/_design/xqerl_design/_view/map-view'),
_:get-view('/_design/xqerl_design/_list/json_to_xml/map-view')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment