Skip to content

Instantly share code, notes, and snippets.

@wesleyegberto
Created December 30, 2018 16:17
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wesleyegberto/fd76e03b1b836ba065681b4593f62435 to your computer and use it in GitHub Desktop.
Save wesleyegberto/fd76e03b1b836ba065681b4593f62435 to your computer and use it in GitHub Desktop.
Export big json file
private ObjectMapper jsonMapper = new ObjectMapper();
private ExecutorService executorService = Executors.newFixedThreadPool(5);
@Async
public ListenableFuture<Boolean> export(UUID customerId) {
try (PipedInputStream in = new PipedInputStream();
PipedOutputStream pipedOut = new PipedOutputStream(in);
GZIPOutputStream out = new GZIPOutputStream(pipedOut)) {
Stopwatch stopwatch = Stopwatch.createStarted();
ObjectWriter writer = jsonMapper.writer().withDefaultPrettyPrinter();
try(SequenceWriter sequenceWriter = writer.writeValues(out)) {
sequenceWriter.init(true);
Future<?> storageFuture = executorService.submit(() ->
storageProvider.storeFile(getFilePath(customerId), in));
int batchCounter = 0;
while (true) {
List<Record> batch = readDatabaseBatch(batchCounter++);
for (Record record : batch) {
sequenceWriter.write(entry);
}
if (batch.isEmpty()) {
// if there are no more batches, stop.
break;
}
}
// wait for storing to complete
storageFuture.get();
// send the customer a notification and a download link
notifyCustomer(customerId);
}
logger.info("Exporting took {} seconds", stopwatch.stop().elapsed(TimeUnit.SECONDS));
return AsyncResult.forValue(true);
} catch (Exception ex) {
logger.error("Failed to export data", ex);
return AsyncResult.forValue(false);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment