Skip to content

Instantly share code, notes, and snippets.

@rajeevprasanna
Last active August 8, 2021 11:23
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rajeevprasanna/d12272c54a6a19793b6e3404ad4f3bdc to your computer and use it in GitHub Desktop.
Save rajeevprasanna/d12272c54a6a19793b6e3404ad4f3bdc to your computer and use it in GitHub Desktop.
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import lombok.val;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class MultiZipTest {
public static void main(String[] args) throws IOException {
String bucketName = "kp-snowflake-export-data";
String keyName = "test/newf/result.zip";
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
.withRegion("us-east-1")
.withCredentials(new ProfileCredentialsProvider("test")).build();
List<PartETag> partETags = new ArrayList<PartETag>();
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("application/zip");
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName, metadata);
InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
System.out.println("Upload response id => " + initResponse.getUploadId());
uploadFile("/Users/rajeev/Desktop/test/data_0_1.csv", bucketName, keyName, s3Client, partETags, initResponse, 1);
uploadFile("/Users/rajeev/Desktop/test/data_0_2.csv", bucketName, keyName, s3Client, partETags, initResponse, 2);
uploadFile("/Users/rajeev/Desktop/test/x.pdf", bucketName, keyName, s3Client, partETags, initResponse, 3);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, keyName,
initResponse.getUploadId(), partETags);
s3Client.completeMultipartUpload(compRequest);
}
private static void uploadFile(String path,
String bucketName,
String keyName,
AmazonS3 s3Client,
List<PartETag> partETags,
InitiateMultipartUploadResult initResponse,
int partNumber) throws IOException {
val file = new File(path);
val fileinputstream = new FileInputStream(file);
val bufferedinputstream = new BufferedInputStream(fileinputstream);
ByteArrayOutputStream binaryOutputStream = new ByteArrayOutputStream();
ZipOutputStream zipoutputstream = new ZipOutputStream(binaryOutputStream);
ZipEntry zipentry = new ZipEntry(file.getName());
zipentry.setMethod(ZipEntry.STORED);
CRC32 checksum = new CRC32();
byte[] array = Files.readAllBytes(Paths.get(path));
checksum.update(array);
zipentry.setCrc(checksum.getValue());
zipentry.setSize(file.length());
zipoutputstream.putNextEntry(zipentry);
byte b[] = new byte[512];
int len;
while ((len = bufferedinputstream.read(b)) != -1) {
zipoutputstream.write(b, 0, len);
}
//You need to flush and close the zipoutputstream before uploading (that is, before calling binaryOutputStream.toByteArray()). Right now you are uploading an incomplete zip file. – Mark Rotteveel
zipoutputstream.flush();
zipoutputstream.closeEntry();
zipoutputstream.close();
byte[] partContent = binaryOutputStream.toByteArray();
InputStream myInputStream = new ByteArrayInputStream(partContent);
UploadPartRequest uploadRequest = new UploadPartRequest()
.withBucketName(bucketName)
.withKey(keyName)
.withUploadId(initResponse.getUploadId())
.withPartNumber(partNumber)
// .withFileOffset(offset)
.withInputStream(myInputStream)
.withPartSize(partContent.length);
// Upload the part and add the response's ETag to our list.
UploadPartResult uploadResult = s3Client.uploadPart(uploadRequest);
partETags.add(uploadResult.getPartETag());
}
}
@rajeevprasanna
Copy link
Author

Error:
Screenshot 2021-08-08 at 4 33 15 PM

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment