Skip to content

Instantly share code, notes, and snippets.

@v-cphillipson
Last active October 14, 2015 15:28
Show Gist options
  • Save v-cphillipson/e18a2360671f78046144 to your computer and use it in GitHub Desktop.
Save v-cphillipson/e18a2360671f78046144 to your computer and use it in GitHub Desktop.
A memory-efficient message handler for obtaining line counts for a file
@JsonPropertyOrder(value = { "run", "runParameters", "fileStatistics", "runSummary" })
public class RunStats {
public static final String TYPE = "type";
public static final String START_DATE = "start";
public static final String END_DATE = "end";
@JsonProperty("run")
private Integer run;
@JsonProperty("runParameters")
private final Map<String, Object> runParameters = new LinkedHashMap<>();
@JsonProperty("fileStatistics")
private final Map<String, Long> fileStatistics = new HashMap<>();
@JsonProperty("runSummary")
private final Map<String, Long> runSummary = new LinkedHashMap<>();
public RunStats(Integer run) {
this.run = run;
}
public void addFile(String filename, Long lineCount) {
fileStatistics.put(filename, lineCount);
}
public void addInputParameter(String parameterName, Object parameterValue) {
runParameters.put(parameterName, parameterValue);
}
public void addSummary(String status, Long lineCount) {
Long currentCount = runSummary.get(status);
if (currentCount == null) {
runSummary.put(status, lineCount);
} else {
runSummary.put(status, currentCount + lineCount);
}
}
public Integer getRun() {
return run;
}
public Map<String, Long> getFileStatistics() {
return Collections.unmodifiableMap(fileStatistics);
}
public Map<String, Object> getRunParameters() {
return Collections.unmodifiableMap(runParameters);
}
public Map<String, Long> getRunSummary() {
return Collections.unmodifiableMap(runSummary);
}
}
protected static class RunStatsHandler extends AbstractMessageHandler {
// Memory efficient routine, @see http://www.baeldung.com/java-read-lines-large-file
@Override
protected void handleMessageInternal(Message<?> message) throws Exception {
RunStats runStats = (RunStats) message.getHeaders().get("runStats");
if (runStats != null) {
File compressedFile = (File) message.getPayload();
String filename = compressedFile.getCanonicalPath();
LongAdder lineCount = new LongAdder();
// Streams and Scanner implement java.lang.AutoCloseable
try ( InputStream fs = new FileInputStream(compressedFile);
InputStream gzfs = new GZIPInputStream(fs);
Scanner sc = new Scanner(gzfs, "UTF-8");
) {
while (sc.hasNextLine()) {
sc.nextLine();
lineCount.increment();
}
// note that Scanner suppresses exceptions
if (sc.ioException() != null) {
throw sc.ioException();
}
runStats.addFile(filename, lineCount.longValue());
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment