/scroll-search.java Secret
Last active
August 29, 2015 14:00
Star
You must be signed in to star a gist
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public void search(final String type, final String source) { | |
final SearchOperationThreading searchOperationThreading = | |
SearchOperationThreading.THREAD_PER_SHARD; | |
final SearchType scan = SearchType.SCAN; | |
final TimeValue keepAlive = TimeValue.timeValueSeconds(10L); | |
final SearchResponse firstScanSearchResp = this.client | |
.prepareSearch(source) | |
.setTypes(type) | |
.setSearchType(scan) | |
.setOperationThreading(searchOperationThreading) | |
.setScroll(new Scroll(keepAlive)) | |
.setSize(this.pageSize) | |
.execute() | |
.actionGet(); | |
final String startingScrollId = firstScanSearchResp.getScrollId(); | |
final long totalHits = firstScanSearchResp.getHits().getTotalHits(); | |
LOGGER.info("starting scrollId: " + startingScrollId); | |
LOGGER.info("first search hits: " + firstScanSearchResp.getHits().getHits().length); | |
LOGGER.info("totalHits: " + totalHits); | |
String currentScrollId = startingScrollId; | |
int pageIndex = 1; | |
//to start | |
SearchResponse currentScanSearchResp = this.client | |
.prepareSearchScroll(currentScrollId) | |
.setScroll(new Scroll(keepAlive)) | |
.setOperationThreading(searchOperationThreading) | |
.execute() | |
.actionGet(); | |
SearchHit[] hits = currentScanSearchResp.getHits().getHits(); | |
while (hits.length > 0) { | |
LOGGER.info("hits: " + hits.length); | |
//ask for next page | |
final SearchResponse searchResponse = this.client | |
.prepareSearchScroll(currentScrollId) | |
.setScroll(new Scroll(keepAlive)) | |
.setOperationThreading(searchOperationThreading) | |
.execute() | |
.actionGet(); | |
hits = searchResponse.getHits().getHits(); | |
currentScrollId = currentScanSearchResp.getScrollId(); | |
LOGGER.info("processed '" + pageIndex + "' page of scroll search"); | |
pageIndex++; | |
try { | |
Thread.sleep(1000L); | |
} catch (InterruptedException e) { | |
e.printStackTrace(); | |
} | |
} | |
try { | |
Thread.sleep(10000L); | |
} catch (InterruptedException e) { | |
e.printStackTrace(); | |
} | |
this.client.prepareClearScroll().addScrollId(currentScrollId).execute().actionGet(); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment