Example query

scraper.js

const ckan = require('ckan');

var client = new ckan.Client('https://data.humdata.org/');
client.requestType = 'GET';

function getPages() {
  return new Promise((resolve, reject) => {
    const getPage = (offset, page, datasetsToDate) => {
      console.log(
        'requesting offset ',
        offset,
        'page',
        page,
        ' have got ',
        datasetsToDate.length,
        ' datasets',
      );
      client.action(
        'datastore_search',
        {
          resource_id: '_table_metadata',
          offset: offset,
        },
        (err, out) => {
          console.log(out);
          console.log(out.result.records.length);
          if (err || !out.success || out.result.records.length === 0) {
            resolve(datasetsToDate);
          } else {
            getPage(offset + 100, page + 1, [
              ...datasetsToDate,
              ...out.result.records,
            ]);
          }
        },
      );
    };
    getPage(0, 0, []);
  });
}

getPages()
  .then((result) => {
    console.log(result);
    console.log('have ', result.length, ' datasets ');
    return result;
  })
  .then((outputResults) => {
    outputResults.forEach((r) => console.log(r.name));
  });