Last active
May 29, 2020 12:53
-
-
Save chakravarthyvp/6607e45648b77274c94fe97c59161585 to your computer and use it in GitHub Desktop.
code snippet to store invalid records in Google Cloud Storage
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
//get all error records per topic | |
final Map<String, GCSInvalidMessagesWrapper> topicMessagesMap = getMessagesToSerializePerTopic(errorRecords); | |
//serialize the messages in GCS for each topic - a file with all messages per topic is stored in GCS | |
topicMessagesMap.keySet().forEach(topicName -> { | |
final String fileName = UUID.randomUUID().toString(); | |
final String pathPrefix = gcsBasePathPrefix + "/" + topicName + "/" | |
+ getFormattedDatePrefix(Instant.now()) + "/"; | |
final BlobId blobId = BlobId.of(gcsBucket, pathPrefix + fileName); | |
final Map<String, String> metaDataMap = new HashMap<>(); | |
.... | |
final BlobInfo objectInfo = BlobInfo.newBuilder(blobId) | |
.setContentType("text/plain") | |
.setMetadata(metaDataMap).build(); | |
final Blob objectCreated = gcsStore.create(objectInfo, | |
topicMessagesMap.get(topicName).getBytes()); | |
}); | |
log.info("Pushing {} records to GCS success?: {}", errorRecords.size(), true); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment