Skip to content

Instantly share code, notes, and snippets.

@bennettscience
Last active November 30, 2021 17:04
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save bennettscience/2484d00b8d3f918c75b82bdef99e0bef to your computer and use it in GitHub Desktop.
Save bennettscience/2484d00b8d3f918c75b82bdef99e0bef to your computer and use it in GitHub Desktop.
Handling paginated REST API results in Google Apps Script
// To use the pagination utility, call it from a script.
function myFunc() {
let response = runner("GET", "endpoint") // returns an array
console.log(response)
// do more stuff
}
// Set your base URL here so it's easily used later.
// Follow the documentation for your use case to build the correct URL for requests.
const BASE_URL = "https://yourApi.com/"
/**
* Return a query string from a submitted object
*
* @params {object} opts JSON-formatted object with request params
*/
const parseOpts = function(opts) {
if(!opts || (Object.keys(opts).length === 0 && opts.constructor === Object)) { return null }
// Iterate the object and create a properly formatted query string
var query = Object.keys(opts).map((key) => {
return key + "=" + opts[key];
}).join('&');
return query;
}
/**
* Perform the request
*
* @param {string} endpoint Request endpoint
* @param {string} query Query string for the HTTP endpoint
* @param {object} params Headers, HTTP method, and options for UrlFetchApp
*
* @returns {object}
*/
const _get = function(endpoint, query, params) {
let fullUrl
query = query || null;
fullUrl = `https://yourUrl.com/${endpoint}`;
if(query !== null) {
fullUrl += `?${query}`;
}
var resp = UrlFetchApp.fetch(fullUrl, params);
return resp;
}
/**
* Format, send, and return data from the Canvas REST API
*
* @param {string} method HTTP request method
* @param {string} url REST endpoint
* @param {object} [opts=null] Optional request parameters
* @param {object[]} [data=null] Optional array to append paginated results
* @returns {object[]}
*/
const runner = function(method, url, opts, data) {
let query;
// If data is sent, use it. Otherwise open an empty array
data = data || [];
// Process an options object into a querystring for the request
opts = opts || null;
if(opts !== null) {
query = parseOpts(opts);
} else {
query = ""
}
// If your API endpoint requires authorization, you'll probably need to set a header.
// Follow the docs for your use case.
const headers = {
"Authorization": "Bearer your_token"
}
var params = {
'headers': headers
'method': method,
'muteHttpExceptions': true,
}
//Retrieve data from Canvas
var request = _get(url, query, params);
let json = JSON.parse(request.getContentText());
// Dump each response item into the array to return
json.forEach(function(el) {
data.push(el);
});
// If there is a next header, run it again.
let req = hasNext(request.getHeaders()['Link']);
if(req.next) {
console.log('Next header present, requesting more...');
// Return only the endpoint by removing the base URL
let endpoint = req.links.next.replace(BASE_URL, "");
runner('GET', endpoint, null, data);
}
return data
}
/**
* Turn the Links header into an object
* The raw link header has this structure with each URL given a specific relationship
* Link: <https://<url>/endpoint.json?query="">; rel="[current, next,...], "
*
* This turns the Link header into an object so you can request any specific relationship
* in your code.
*
* @param {object} header HTTP header returned from a request
* @returns {object}
*/
function hasNext(header) {
let next = false;
if (header.length === 0) {
throw new Error("input must not be of zero length");
}
// Split parts by comma and open up an object
let parts = header.split(',');
let links = {};
// Parse each part into a named link
for(var i=0; i<parts.length; i++) {
let section = parts[i].split(';');
if (section.length !== 2) {
throw new Error("section could not be split on ';'");
}
let url = section[0].replace(/<(.*)>/, '$1').trim();
let name = section[1].replace(/rel="(.*)"/, '$1').trim();
links[name] = url;
}
(links.next) ? next = true : next = false;
return {
next: next,
links: links
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment