Created
March 26, 2018 20:48
-
-
Save dipunm/99aa1607898ee89a65822baf31e0abc9 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Cursor { | |
constructor(cursor) { | |
this.innerCursor = cursor; | |
this.cached = []; | |
} | |
hasNext() { | |
if(cached.length) return Promise.resolve(true); | |
return this.innerCursor.fetchSome().then(some => { | |
this.cached.append(some); | |
return !!this.cached.length; | |
}); | |
} | |
readNext(n) { | |
if(cached.length >= n) { | |
const toSend = this.cached.unshift(n); | |
this.cached = this.cached.tail(n); | |
return Promise.resolve(toSend); | |
} | |
this.innerCursor.fetchSome().then(some => { | |
if(some.length) { | |
this.cached.append(some); | |
return this.readNext(n); | |
} | |
// rest. | |
const cached = this.cached; | |
this.cached = []; | |
return cached; | |
}); | |
} | |
} | |
class SolrClient { | |
sendData(data) { | |
return post(url, data).then(xhr => xhr.success); | |
} | |
} | |
function syncDataToSolr(lastUpdated) { | |
var query = new Query(lastUpdated); | |
const cursor = mongo.dispatch(query); | |
const timer = new Timer(); | |
timer.start(); | |
let count = 0; | |
let completed, error; | |
try { | |
while(await cursor.hasNext()) { | |
const data = await cursor.readNext(100); | |
const success = await solrClient.sendData(data); | |
if(!success) { | |
completed = false; | |
break; | |
} else { | |
count += data.length; | |
dateOfLastUpdated = data[data.length - 1].updatedDate; | |
} | |
} | |
} catch (err) { | |
completed = false; | |
error = err; | |
} finally { | |
completed = completed !== false; | |
timer.end(); | |
} | |
return Status({ | |
timeElapsed: timer.elapsed, | |
count, | |
dateOfLastUpdated, | |
completed, | |
error | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment