Created
December 30, 2018 16:17
-
-
Save wesleyegberto/fd76e03b1b836ba065681b4593f62435 to your computer and use it in GitHub Desktop.
Export big json file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
private ObjectMapper jsonMapper = new ObjectMapper(); | |
private ExecutorService executorService = Executors.newFixedThreadPool(5); | |
@Async | |
public ListenableFuture<Boolean> export(UUID customerId) { | |
try (PipedInputStream in = new PipedInputStream(); | |
PipedOutputStream pipedOut = new PipedOutputStream(in); | |
GZIPOutputStream out = new GZIPOutputStream(pipedOut)) { | |
Stopwatch stopwatch = Stopwatch.createStarted(); | |
ObjectWriter writer = jsonMapper.writer().withDefaultPrettyPrinter(); | |
try(SequenceWriter sequenceWriter = writer.writeValues(out)) { | |
sequenceWriter.init(true); | |
Future<?> storageFuture = executorService.submit(() -> | |
storageProvider.storeFile(getFilePath(customerId), in)); | |
int batchCounter = 0; | |
while (true) { | |
List<Record> batch = readDatabaseBatch(batchCounter++); | |
for (Record record : batch) { | |
sequenceWriter.write(entry); | |
} | |
if (batch.isEmpty()) { | |
// if there are no more batches, stop. | |
break; | |
} | |
} | |
// wait for storing to complete | |
storageFuture.get(); | |
// send the customer a notification and a download link | |
notifyCustomer(customerId); | |
} | |
logger.info("Exporting took {} seconds", stopwatch.stop().elapsed(TimeUnit.SECONDS)); | |
return AsyncResult.forValue(true); | |
} catch (Exception ex) { | |
logger.error("Failed to export data", ex); | |
return AsyncResult.forValue(false); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment