comp.lang.java: enumerating the first n files in a directory with many, many files.

ListFileTestThreaded2.java

package file;

import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;

public final class ListFileTestThreaded2 {

  private static final class CountFilterThread extends Thread implements FileFilter {

    private final File dir;
    private final int maxFiles;
    private final BlockingQueue<List<File>> queue;
    private List<File> filesSeen = new ArrayList<File>();

    public CountFilterThread(File dir, int maxFiles, BlockingQueue<List<File>> queue) {
      this.dir = dir;
      this.maxFiles = maxFiles;
      this.queue = queue;
    }

    @Override
    public void run() {
      try {
        dir.listFiles(this);

        if (filesSeen != null) {
          send();
        }
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }

    private void send() throws InterruptedException {
      queue.put(filesSeen);
      filesSeen = null;
    }

    @Override
    public boolean accept(final File f) {
      try {
        if (filesSeen != null) {
          filesSeen.add(f);

          if (filesSeen.size() == maxFiles) {
            send();
            assert filesSeen == null;
          }
        }

        return false;
      } catch (InterruptedException e) {
        throw new IllegalStateException(e);
      }
    }
  }

  private static final int[] LIMITS = { 10, 100, 1000, 10000, Integer.MAX_VALUE };

  public static void main(String[] args) throws InterruptedException {
    for (final String s : args) {
      System.out.println("Testing: " + s);
      final File dir = new File(s);

      if (dir.isDirectory()) {
        for (final int limit : LIMITS) {
          final SynchronousQueue<List<File>> queue = new SynchronousQueue<List<File>>();
          final CountFilterThread cf = new CountFilterThread(dir, limit, queue);
          cf.setDaemon(true);
          final long t1 = System.nanoTime();
          cf.start();
          final List<File> entries = queue.take();
          final long delta = System.nanoTime() - t1;
          System.out.printf("It took %20dus to retrieve %20d files, %20.5fus/file.\n",
              TimeUnit.NANOSECONDS.toMicros(delta), entries.size(), (double) TimeUnit.NANOSECONDS.toMicros(delta)
                  / entries.size());
        }
      } else {
        System.out.println("Not a directory.");
      }
    }

    System.out.println("done");
  }

}