Simple Paging Example

pager.js

require('dotenv').config();
const request = require('request-promise');

// Hoist our users array.
let users = [];

const API_ENDPOINT = `https://publicapi.knowledgeanywhere.com`;
const CLIENT_ID = process.env.LMS_CLIENT_ID; // Loaded from our .env file
const CLIENT_SECRET = process.env.LMS_CLIENT_SECRET; // Loaded from our .env file
const PAGE_SIZE = 1000; // How many records the API returns in a page.

const go = async () => {
    const tokenReq = {
        method: 'POST',
        uri: `${API_ENDPOINT}/auth/token`,
        json: true,
        headers: {
            'content-type': 'application/x-www-form-urlencoded'
        },
        form: {
            client_id: CLIENT_ID,
            client_secret: CLIENT_SECRET,
            grant_type: 'client_credentials'
        }
    };
    const token = await request(tokenReq);
    users = await getUsers(token);
    // We're all done.   Now we can do what we would like with our array of user objects.
    console.log(`Total length of users is ${users.length}`);
}

const getUsers = async (token) => {
    let records = [];
    let keepGoing = true;
    let offset = 0;
    while (keepGoing) {
        let response = await reqUsers(token, offset)
        await records.push.apply(records, response);
        offset += 1000;
        // this may need to be adjusted to your api to handle the corner case where the last page size equal to PAGE_SIZE
        // if the api either errors our the next call where the offset is greater than the amount of records or returns an empty array
        // the behavior will be fine.
        if (response.length < PAGE_SIZE) {
            keepGoing = false;
            return records;
        }
    }
}

const reqUsers = async (token, offset) => {
    const userReq = {
        uri: `${API_ENDPOINT}/v1/users?offset=${offset}`,
        json: true,
        headers: {
            "Authorization": `Bearer ${token.access_token}`,
        }
    };
    let payload = [];
    try {
       payload = await request(userReq);
    } catch() {
       console.log('An exception occured');
    }

    return payload;
}

go()