Skip to content

Instantly share code, notes, and snippets.

@karussell
Created February 8, 2011 11:34
Show Gist options
  • Save karussell/816297 to your computer and use it in GitHub Desktop.
Save karussell/816297 to your computer and use it in GitHub Desktop.
merge indices
for (String fromIndex : indexList) {
SearchResponse rsp = client.prepareSearch(fromIndex).
setQuery(QueryBuilders.matchAllQuery()).setSize(hitsPerPage).
// This is important:
setSearchType(SearchType.QUERY_AND_FETCH).
setScroll(TimeValue.timeValueMinutes(30)).execute().actionGet();
try {
long total = rsp.hits().totalHits();
Collection<MyTweet> tweets = collectTweets(rsp);
bulkUpdate(tweets, intoIndex);
int page = 1;
int pages = (int) (total / hitsPerPage);
if (total % hitsPerPage > 0)
pages++;
int collectedResults = 0;
int currentResults;
while ((currentResults = rsp.hits().hits().length) > 0) {
collectedResults+=currentResults;
rsp = client.prepareSearchScroll(rsp.scrollId()).
setScroll(TimeValue.timeValueMinutes(30)).execute().actionGet();
tweets = collectTweets(rsp);
bulkUpdate(tweets, intoIndex);
logger.info("Progress " + page++ + "/"+ pages + " current=" + currentResults
+ " total=" + total + " fromIndex=" + fromIndex);
}
logger.info("Finished copying of index:"+fromIndex+". Total:" + total + " collected:" + collectedResults);
} catch (Exception ex) {
logger.error("Failed to copy data from index " + fromIndex + " into " + intoIndex + ".", ex);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment