Skip to content

Instantly share code, notes, and snippets.

@dipunm
Created March 26, 2018 20:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dipunm/99aa1607898ee89a65822baf31e0abc9 to your computer and use it in GitHub Desktop.
Save dipunm/99aa1607898ee89a65822baf31e0abc9 to your computer and use it in GitHub Desktop.
class Cursor {
constructor(cursor) {
this.innerCursor = cursor;
this.cached = [];
}
hasNext() {
if(cached.length) return Promise.resolve(true);
return this.innerCursor.fetchSome().then(some => {
this.cached.append(some);
return !!this.cached.length;
});
}
readNext(n) {
if(cached.length >= n) {
const toSend = this.cached.unshift(n);
this.cached = this.cached.tail(n);
return Promise.resolve(toSend);
}
this.innerCursor.fetchSome().then(some => {
if(some.length) {
this.cached.append(some);
return this.readNext(n);
}
// rest.
const cached = this.cached;
this.cached = [];
return cached;
});
}
}
class SolrClient {
sendData(data) {
return post(url, data).then(xhr => xhr.success);
}
}
function syncDataToSolr(lastUpdated) {
var query = new Query(lastUpdated);
const cursor = mongo.dispatch(query);
const timer = new Timer();
timer.start();
let count = 0;
let completed, error;
try {
while(await cursor.hasNext()) {
const data = await cursor.readNext(100);
const success = await solrClient.sendData(data);
if(!success) {
completed = false;
break;
} else {
count += data.length;
dateOfLastUpdated = data[data.length - 1].updatedDate;
}
}
} catch (err) {
completed = false;
error = err;
} finally {
completed = completed !== false;
timer.end();
}
return Status({
timeElapsed: timer.elapsed,
count,
dateOfLastUpdated,
completed,
error
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment