Skip to content

Instantly share code, notes, and snippets.

@louiszuckerman
Last active August 29, 2015 14:07
Show Gist options
  • Save louiszuckerman/98259a02a391a4bae778 to your computer and use it in GitHub Desktop.
Save louiszuckerman/98259a02a391a4bae778 to your computer and use it in GitHub Desktop.
Multithreaded (forkjoinpool) du in java
package com.peircean.glustest;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.*;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.*;
public class Du {
public static final String PREFIX =
"gluster://172.31.31.31:foo/usr";
private static final int THREADS = 4;
public static void main(String[] args) throws URISyntaxException, IOException {
ForkJoinPool pool = new ForkJoinPool(THREADS);
PathProcessor proc = new PathProcessor(Paths.get(new URI(PREFIX)));
long startTime = System.nanoTime();
pool.execute(proc);
Long total = proc.join();
pool.shutdown();
long endTime = System.nanoTime();
double time = (endTime - startTime) / 1000000.0;
System.out.println("TOTAL: " + total + " RUNTIME: " + time);
}
}
class PathProcessor extends RecursiveTask<Long> {
Path path;
public PathProcessor(Path path) {
this.path = path;
}
@Override
protected Long compute() {
long sum = 0;
List<PathProcessor> newTasks = new LinkedList<>();
DirectoryStream<Path> directoryStream;
try {
sum += Files.size(path);
directoryStream = Files.newDirectoryStream(path);
// System.out.println("SCANNING DIR: " + path);
} catch (IOException e) {
System.out.println("EXCEPTION: " + path);
e.printStackTrace();
return sum;
}
for (Path p : directoryStream) {
if (p.endsWith("/.") || p.endsWith("/..") || Files.isSymbolicLink(p)) {
continue;
}
if (Files.isDirectory(p, LinkOption.NOFOLLOW_LINKS)) {
// System.out.println("FORKING FOR DIR: " + p);
PathProcessor processor = new PathProcessor(p);
processor.fork();
newTasks.add(processor);
} else {
try {
sum += Files.size(p);
// System.out.println("ADDED " + p + " size now " + sum);
} catch (Exception e) {
System.out.println("EXCEPTION: " + p);
e.printStackTrace();
}
}
}
for (PathProcessor task : newTasks) {
sum += task.join();
}
return sum;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment