-
-
Save bennettscience/2484d00b8d3f918c75b82bdef99e0bef to your computer and use it in GitHub Desktop.
Handling paginated REST API results in Google Apps Script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// To use the pagination utility, call it from a script. | |
function myFunc() { | |
let response = runner("GET", "endpoint") // returns an array | |
console.log(response) | |
// do more stuff | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Set your base URL here so it's easily used later. | |
// Follow the documentation for your use case to build the correct URL for requests. | |
const BASE_URL = "https://yourApi.com/" | |
/** | |
* Return a query string from a submitted object | |
* | |
* @params {object} opts JSON-formatted object with request params | |
*/ | |
const parseOpts = function(opts) { | |
if(!opts || (Object.keys(opts).length === 0 && opts.constructor === Object)) { return null } | |
// Iterate the object and create a properly formatted query string | |
var query = Object.keys(opts).map((key) => { | |
return key + "=" + opts[key]; | |
}).join('&'); | |
return query; | |
} | |
/** | |
* Perform the request | |
* | |
* @param {string} endpoint Request endpoint | |
* @param {string} query Query string for the HTTP endpoint | |
* @param {object} params Headers, HTTP method, and options for UrlFetchApp | |
* | |
* @returns {object} | |
*/ | |
const _get = function(endpoint, query, params) { | |
let fullUrl | |
query = query || null; | |
fullUrl = `https://yourUrl.com/${endpoint}`; | |
if(query !== null) { | |
fullUrl += `?${query}`; | |
} | |
var resp = UrlFetchApp.fetch(fullUrl, params); | |
return resp; | |
} | |
/** | |
* Format, send, and return data from the Canvas REST API | |
* | |
* @param {string} method HTTP request method | |
* @param {string} url REST endpoint | |
* @param {object} [opts=null] Optional request parameters | |
* @param {object[]} [data=null] Optional array to append paginated results | |
* @returns {object[]} | |
*/ | |
const runner = function(method, url, opts, data) { | |
let query; | |
// If data is sent, use it. Otherwise open an empty array | |
data = data || []; | |
// Process an options object into a querystring for the request | |
opts = opts || null; | |
if(opts !== null) { | |
query = parseOpts(opts); | |
} else { | |
query = "" | |
} | |
// If your API endpoint requires authorization, you'll probably need to set a header. | |
// Follow the docs for your use case. | |
const headers = { | |
"Authorization": "Bearer your_token" | |
} | |
var params = { | |
'headers': headers | |
'method': method, | |
'muteHttpExceptions': true, | |
} | |
//Retrieve data from Canvas | |
var request = _get(url, query, params); | |
let json = JSON.parse(request.getContentText()); | |
// Dump each response item into the array to return | |
json.forEach(function(el) { | |
data.push(el); | |
}); | |
// If there is a next header, run it again. | |
let req = hasNext(request.getHeaders()['Link']); | |
if(req.next) { | |
console.log('Next header present, requesting more...'); | |
// Return only the endpoint by removing the base URL | |
let endpoint = req.links.next.replace(BASE_URL, ""); | |
runner('GET', endpoint, null, data); | |
} | |
return data | |
} | |
/** | |
* Turn the Links header into an object | |
* The raw link header has this structure with each URL given a specific relationship | |
* Link: <https://<url>/endpoint.json?query="">; rel="[current, next,...], " | |
* | |
* This turns the Link header into an object so you can request any specific relationship | |
* in your code. | |
* | |
* @param {object} header HTTP header returned from a request | |
* @returns {object} | |
*/ | |
function hasNext(header) { | |
let next = false; | |
if (header.length === 0) { | |
throw new Error("input must not be of zero length"); | |
} | |
// Split parts by comma and open up an object | |
let parts = header.split(','); | |
let links = {}; | |
// Parse each part into a named link | |
for(var i=0; i<parts.length; i++) { | |
let section = parts[i].split(';'); | |
if (section.length !== 2) { | |
throw new Error("section could not be split on ';'"); | |
} | |
let url = section[0].replace(/<(.*)>/, '$1').trim(); | |
let name = section[1].replace(/rel="(.*)"/, '$1').trim(); | |
links[name] = url; | |
} | |
(links.next) ? next = true : next = false; | |
return { | |
next: next, | |
links: links | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment