Skip to content

Instantly share code, notes, and snippets.

@KidA78
Created February 20, 2013 03:56
Show Gist options
  • Save KidA78/4992749 to your computer and use it in GitHub Desktop.
Save KidA78/4992749 to your computer and use it in GitHub Desktop.
/**
* Recursive method that returns a Future of URLs
*
* @param url
* @param maxDepth
* @param currentDepth
* @return
*/
public Future<List<String>> crawl(final String url, final int maxDepth, final int currentDepth) {
if (maxDepth == currentDepth) {
return Futures.future(new Callable<List<String>>() {
@Override
public List<String> call() throws Exception {
return Collections.emptyList();
}
}, ec);
}
else {
// 1.
Future<List<String>> crawlResultsFuture = future(new Callable<List<String>>() {
public List<String> call() throws Exception {
Collection<String> foundUrls = MockWebCrawler.crawl(url)
return Arrays.asList(foundUrls.toArray(new String[0]));
}
}, ec);
// 2.
Future<Iterable<List<String>>> childCrawlsFuture = crawlResultsFuture.flatMap(new Mapper<List<String>, Future<Iterable<List<String>>>>() {
public Future<Iterable<List<String>>> apply(final List<String> foundUrls) {
List<Future<List<String>>> crawlFutures = new ArrayList<Future<List<String>>>();
crawlFutures.add(Futures.successful(foundUrls, ec));
for (String u : foundUrls) {
Future<List<String>> crawlFuture = crawl(u, maxDepth, currentDepth + 1);
crawlFutures.add(crawlFuture);
}
return Futures.sequence(crawlFutures, ec);
}
});
// 3.
Future<List<String>> crawlFuture = childCrawlsFuture.map(new Mapper<Iterable<List<String>>, List<String>>() {
@Override
public List<String> apply(Iterable<List<String>> childCrawls) {
List<String> result = new ArrayList<String>();
for (List<String> c : childCrawls) {
result.addAll(c);
}
return result;
}
});
return crawlFuture;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment