-
-
Save rajeevprasanna/d12272c54a6a19793b6e3404ad4f3bdc to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.amazonaws.auth.profile.ProfileCredentialsProvider; | |
import com.amazonaws.services.s3.AmazonS3; | |
import com.amazonaws.services.s3.AmazonS3ClientBuilder; | |
import com.amazonaws.services.s3.model.*; | |
import lombok.val; | |
import java.io.*; | |
import java.nio.file.Files; | |
import java.nio.file.Paths; | |
import java.util.ArrayList; | |
import java.util.List; | |
import java.util.zip.CRC32; | |
import java.util.zip.ZipEntry; | |
import java.util.zip.ZipOutputStream; | |
public class MultiZipTest { | |
public static void main(String[] args) throws IOException { | |
String bucketName = "kp-snowflake-export-data"; | |
String keyName = "test/newf/result.zip"; | |
AmazonS3 s3Client = AmazonS3ClientBuilder.standard() | |
.withRegion("us-east-1") | |
.withCredentials(new ProfileCredentialsProvider("test")).build(); | |
List<PartETag> partETags = new ArrayList<PartETag>(); | |
ObjectMetadata metadata = new ObjectMetadata(); | |
metadata.setContentType("application/zip"); | |
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName, metadata); | |
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); | |
System.out.println("Upload response id => " + initResponse.getUploadId()); | |
uploadFile("/Users/rajeev/Desktop/test/data_0_1.csv", bucketName, keyName, s3Client, partETags, initResponse, 1); | |
uploadFile("/Users/rajeev/Desktop/test/data_0_2.csv", bucketName, keyName, s3Client, partETags, initResponse, 2); | |
uploadFile("/Users/rajeev/Desktop/test/x.pdf", bucketName, keyName, s3Client, partETags, initResponse, 3); | |
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName, | |
initResponse.getUploadId(), partETags); | |
s3Client.completeMultipartUpload(compRequest); | |
} | |
private static void uploadFile(String path, | |
String bucketName, | |
String keyName, | |
AmazonS3 s3Client, | |
List<PartETag> partETags, | |
InitiateMultipartUploadResult initResponse, | |
int partNumber) throws IOException { | |
val file = new File(path); | |
val fileinputstream = new FileInputStream(file); | |
val bufferedinputstream = new BufferedInputStream(fileinputstream); | |
ByteArrayOutputStream binaryOutputStream = new ByteArrayOutputStream(); | |
ZipOutputStream zipoutputstream = new ZipOutputStream(binaryOutputStream); | |
ZipEntry zipentry = new ZipEntry(file.getName()); | |
zipentry.setMethod(ZipEntry.STORED); | |
CRC32 checksum = new CRC32(); | |
byte[] array = Files.readAllBytes(Paths.get(path)); | |
checksum.update(array); | |
zipentry.setCrc(checksum.getValue()); | |
zipentry.setSize(file.length()); | |
zipoutputstream.putNextEntry(zipentry); | |
byte b[] = new byte[512]; | |
int len; | |
while ((len = bufferedinputstream.read(b)) != -1) { | |
zipoutputstream.write(b, 0, len); | |
} | |
//You need to flush and close the zipoutputstream before uploading (that is, before calling binaryOutputStream.toByteArray()). Right now you are uploading an incomplete zip file. – Mark Rotteveel | |
zipoutputstream.flush(); | |
zipoutputstream.closeEntry(); | |
zipoutputstream.close(); | |
byte[] partContent = binaryOutputStream.toByteArray(); | |
InputStream myInputStream = new ByteArrayInputStream(partContent); | |
UploadPartRequest uploadRequest = new UploadPartRequest() | |
.withBucketName(bucketName) | |
.withKey(keyName) | |
.withUploadId(initResponse.getUploadId()) | |
.withPartNumber(partNumber) | |
// .withFileOffset(offset) | |
.withInputStream(myInputStream) | |
.withPartSize(partContent.length); | |
// Upload the part and add the response's ETag to our list. | |
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest); | |
partETags.add(uploadResult.getPartETag()); | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Error: