test_scroll.js

const { Observable, Subject, BehaviorSubject, throwError, forkJoin, of, from, EMPTY } = require('./node_modules/rxjs');
const { map, catchError, tap, mergeMap, reduce, finalize, expand } = require("./node_modules/rxjs/operators");
const { ajax } = require('rxjs/ajax');
const {XMLHttpRequest} = require("xmlhttprequest");

function fetchAllGeneric(endpoint, qParams) {
  return fetchOne(endpoint, qParams).pipe(
    expand((data) => {
        return data.next ? fetchOne(endpoint, data.next) : EMPTY;
      }),
      reduce((acc, data) => {
        // console.group()
        // console.log(acc)
        // console.log(data)
        // console.groupEnd();
        return acc.concat(data);
      }, []),
      catchError(e => {
        console.log(e)
        throwError(e);
        return (new Observable());
      }),
      map((all_data) => {
        // console.log("end of API")
        // console.log(all_data)
        // last iteration returns undefined; filter out
        all_data = all_data.filter(d => d);
        return (all_data);
      })
  );
}

function fetchOne(endpoint, scrollID) {
  // console.log(params)
  return get(scrollID).pipe(
    map(response => {
      return {
        next: response["next"],
        results: response['results']
      };
      }),
    catchError(e => {
      console.log(e);
      throwError(e);
      return (new Observable());
    })
  );
}

function createXHR() {
  return new XMLHttpRequest();
}

function get(scrollID){
  var ct = 0;
  var url = "https://dev.cvisb.org/api/datadownload/query?q=__all__&fetch_all=true";
  if(scrollID){
    url = "https://dev.cvisb.org/api/datadownload/query?q=__all__&fetch_all=true&scroll_id="+scrollID;
  }
  return ajax({
    createXHR,
    url: url
  }).pipe(
    map(response => {
      var tmp = JSON.parse(response["xhr"]["responseText"]);
      console.log(url);
        return {
          next: tmp['_scroll_id'],
          results: tmp['hits'],
          ct: ct
        };
      }),
      catchError(e => {
        throwError(e);
	console.log(e);
        return (new Observable());
      })
    );
}

var a = fetchAllGeneric();
a.subscribe(_ => console.log(_));