gistfile1.java

package org.logpresso.misc;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;

public class SignatureTesterMain {
	public static void main(String[] args) throws IOException {
		File file = new File(args[0]);
		if (!file.exists()) {
			System.out.println("file not found");
			System.exit(1);
		}

		BufferedReader scanner = null;

		try {
			scanner = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
			HashMap<String, String> signatures = new HashMap<String, String>();
			long t1 = System.currentTimeMillis();
			long linecnt = 0;
			while (true) {
				String line = scanner.readLine();
				if (line == null)
					break;
				linecnt++;
				String sig = makeSignature(line);
				if (!signatures.containsKey(sig)) {
					signatures.put(sig, line);
					System.out.println("[" + sig + "]");
					System.out.println(line);
					System.out.flush();
				}

				if (linecnt == 1000000) {
					linecnt = 0;
					long t2 = System.currentTimeMillis();
					System.out.printf("%d logs/sec\n", (long) (1000000000.0 / (t2 - t1)));
					t1 = t2;
				}
			}
		} finally {
			scanner.close();
		}
	}

	private static String makeSignature(String line) {
		StringBuilder sb = new StringBuilder(line.length());
		boolean inQuote = false;
		for (int i = 0; i < line.length(); ++i) {
			char c = line.charAt(i);
			if (c == '\"')
				inQuote = !inQuote;
			if (Character.isLetterOrDigit(c))
				continue;
			if (!inQuote)
				sb.append(c);
		}
		return sb.toString();
	}
}