Skip to content

Instantly share code, notes, and snippets.

@TaylorAckley
Last active August 15, 2021 04:54
Show Gist options
  • Save TaylorAckley/8977bc38659d3d0010a0135351de5d83 to your computer and use it in GitHub Desktop.
Save TaylorAckley/8977bc38659d3d0010a0135351de5d83 to your computer and use it in GitHub Desktop.
Simple Paging Example
require('dotenv').config();
const request = require('request-promise');
// Hoist our users array.
let users = [];
const API_ENDPOINT = `https://publicapi.knowledgeanywhere.com`;
const CLIENT_ID = process.env.LMS_CLIENT_ID; // Loaded from our .env file
const CLIENT_SECRET = process.env.LMS_CLIENT_SECRET; // Loaded from our .env file
const PAGE_SIZE = 1000; // How many records the API returns in a page.
const go = async () => {
const tokenReq = {
method: 'POST',
uri: `${API_ENDPOINT}/auth/token`,
json: true,
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
form: {
client_id: CLIENT_ID,
client_secret: CLIENT_SECRET,
grant_type: 'client_credentials'
}
};
const token = await request(tokenReq);
users = await getUsers(token);
// We're all done. Now we can do what we would like with our array of user objects.
console.log(`Total length of users is ${users.length}`);
}
const getUsers = async (token) => {
let records = [];
let keepGoing = true;
let offset = 0;
while (keepGoing) {
let response = await reqUsers(token, offset)
await records.push.apply(records, response);
offset += 1000;
// this may need to be adjusted to your api to handle the corner case where the last page size equal to PAGE_SIZE
// if the api either errors our the next call where the offset is greater than the amount of records or returns an empty array
// the behavior will be fine.
if (response.length < PAGE_SIZE) {
keepGoing = false;
return records;
}
}
}
const reqUsers = async (token, offset) => {
const userReq = {
uri: `${API_ENDPOINT}/v1/users?offset=${offset}`,
json: true,
headers: {
"Authorization": `Bearer ${token.access_token}`,
}
};
let payload = [];
try {
payload = await request(userReq);
} catch() {
console.log('An exception occured');
}
return payload;
}
go()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment