Instantly share code, notes, and snippets.

Embed
What would you like to do?
A unit test suite exploring the main features of Azure block storage in Java
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.*;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.*;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.*;
import static junit.framework.TestCase.fail;
import static org.junit.Assert.*;
public class BlockBlobTest {
private static CloudBlobContainer container = null;
@BeforeClass
public static void setup() throws URISyntaxException, InvalidKeyException, StorageException, IOException {
InputStream inputStream = BlockBlobTest.class.getResourceAsStream("config.properties");
Properties properties = new Properties();
properties.load(inputStream);
String storageConnectionString = "DefaultEndpointsProtocol=https;" +
"AccountName=" + properties.getProperty("storage_account") + ";" +
"AccountKey=" + properties.getProperty("storage_account_key");
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
// Container names must be valid DNS names, i.e. only lower case letters, numbers and dashes (-)
container = blobClient.getContainerReference("testcontainer");
container.createIfNotExists();
// Delete any blobs from a previous run. We don't do this in teardown because it is sometimes
// useful to inspect the test artifacts after a run.
for (ListBlobItem item : container.listBlobs()) {
if (item instanceof CloudBlockBlob) {
CloudBlockBlob blockBlob = (CloudBlockBlob) item;
blockBlob.delete(DeleteSnapshotsOption.INCLUDE_SNAPSHOTS, null, null, null);
}
}
}
@AfterClass
public static void tearDown() {}
/******************************************************************************************************************/
@Test
public void basicUploadAndDownload() throws Exception {
URL imageUrl = BlockBlobTest.class.getResource("image.png");
File imageFile = new File(imageUrl.getFile());
CloudBlockBlob blob = container.getBlockBlobReference("basicUploadAndDownload.png");
// Upload the file, calculating the MD5 checksum.
MessageDigest uploadMD5 = MessageDigest.getInstance("MD5");
try (DigestInputStream inputStream = new DigestInputStream(new FileInputStream(imageFile), uploadMD5)) {
blob.upload(inputStream, imageFile.length());
}
assertTrue(blob.exists());
// The MD5 content calculated by the blob service should match our checksum.
BlobProperties properties = blob.getProperties();
assertEquals(properties.getContentMD5(), Base64.getEncoder().encodeToString(uploadMD5.digest()));
// Download the uploaded file to the temp directory, calculating the MD5 checksum again.
File tempFile = File.createTempFile("basicUploadAndDownload", ".png");
MessageDigest downloadMD5 = MessageDigest.getInstance("MD5");
try (DigestOutputStream outputStream = new DigestOutputStream(new FileOutputStream(tempFile.getAbsolutePath()), downloadMD5)) {
blob.download(outputStream);
}
// Downloaded file MD5 should match blob content MD5
assertEquals(properties.getContentMD5(), Base64.getEncoder().encodeToString(downloadMD5.digest()));
}
/******************************************************************************************************************/
@Test
public void basicCopy() throws Exception {
// Upload an image to our source blob.
URL imageUrl = BlockBlobTest.class.getResource("image.png");
File imageFile = new File(imageUrl.getFile());
CloudBlockBlob sourceBlob = container.getBlockBlobReference("basicCopySource.png");
try (FileInputStream inputStream = new FileInputStream(imageFile)) {
sourceBlob.upload(inputStream, imageFile.length());
}
assertTrue(sourceBlob.exists());
// Copy source blob to destination blob.
CloudBlockBlob destinationBlob = container.getBlockBlobReference("BasicCopyDest.png");
destinationBlob.startCopy(sourceBlob);
assertTrue(destinationBlob.exists());
// Check that source and destination match.
BlobProperties sourceProperties = sourceBlob.getProperties();
BlobProperties destinationProperties = destinationBlob.getProperties();
assertEquals(sourceProperties.getContentMD5(), destinationProperties.getContentMD5());
}
/******************************************************************************************************************/
@Test
public void setAndGetMetadata() throws Exception {
CloudBlockBlob blob = container.getBlockBlobReference("metadata.txt");
blob.uploadText("I have metadata.");
assertTrue(blob.exists());
// Create some metadata, arbitrary key-value pairs.
HashMap<String, String> metadata = new HashMap<>();
metadata.put("Timestamp", LocalDateTime.now().toString());
// setMetadata sets the local blob property and uploadMetadata commits it to the blob service.
blob.setMetadata(metadata);
blob.uploadMetadata();
// downloadAttributes refreshes the blob metadata (amongst other properties) from the blob service.
blob.downloadAttributes();
assertEquals(blob.getMetadata().get("Timestamp"), metadata.get("Timestamp"));
}
/******************************************************************************************************************/
@Test
public void uploadWithLease() throws Exception {
CloudBlockBlob blob = container.getBlockBlobReference("leaseTests.txt");
blob.uploadText("I am testing leases.");
assertTrue(blob.exists());
// Acquire the lease on our blob
String leaseId = blob.acquireLease();
AccessCondition leaseCondition = AccessCondition.generateLeaseCondition(leaseId);
// Attempt to upload without using the lease. This will fail.
try {
String testMessage = "This upload should fail.";
blob.uploadText(testMessage);
fail(testMessage);
}
catch (StorageException ex) {
// We expected a StorageException to be thrown, so just continue.
}
// Now upload correctly with the lease
String testMessage = "This upload with lease should succeed.";
blob.uploadText(testMessage, null, leaseCondition, null, null);
assertEquals(blob.downloadText(null, leaseCondition, null, null), testMessage);
// Release the lease and ensure that the blob is no longer locked.
blob.releaseLease(leaseCondition);
testMessage = "Once the lease is released, we can upload without one again.";
blob.uploadText(testMessage);
assertEquals(blob.downloadText(), testMessage);
}
/******************************************************************************************************************/
@Test
public void usingSnapshots() throws Exception {
CloudBlockBlob originalBlob = container.getBlockBlobReference("usingSnapshots.txt");
String originalText = "Original state.";
originalBlob.uploadText(originalText);
assertTrue(originalBlob.exists());
// Create a snapshot of the original blob, it should have been created in the same container.
CloudBlockBlob snapshot = (CloudBlockBlob) originalBlob.createSnapshot();
assertTrue(snapshot.exists());
assertTrue(snapshot.isSnapshot());
assertEquals(snapshot.getContainer(), container);
// Modify the original blob after snapshot-ing it, ensure it is modified.
String modifiedText = "Modified state.";
originalBlob.uploadText(modifiedText);
assertEquals(originalBlob.downloadText(), modifiedText);
// Restoring to a snapshot is just a normal blob-to-blob copy.
originalBlob.startCopy(snapshot);
assertEquals(originalBlob.downloadText(), originalText);
}
/******************************************************************************************************************/
@Test
public void invalidSnapshotOperations() throws Exception {
CloudBlockBlob originalBlob = container.getBlockBlobReference("invalidSnapshotOperations.txt");
originalBlob.uploadText("I am testing snapshots.");
assertTrue(originalBlob.exists());
CloudBlockBlob snapshot = (CloudBlockBlob) originalBlob.createSnapshot();
assertTrue(snapshot.exists());
assertTrue(snapshot.isSnapshot());
// Uploads are not allowed on snapshot blobs.
String text = "Should not be able to upload to snapshot.";
try {
snapshot.uploadText(text);
fail(text);
}
catch (IllegalArgumentException ex) {
// Expected exception was thrown, continue
}
// Metadata is now allowed either.
HashMap<String, String> metadata = new HashMap<>();
text = "Uploading metadata to a snapshot is not allowed.";
metadata.put("Invalid", text);
snapshot.setMetadata(metadata);
try {
snapshot.uploadMetadata();
fail(text);
}
catch (IllegalArgumentException ex) {
// Expected exception was thrown, continue
}
// Finally, you cannot create a snapshot of another snapshot.
try {
CloudBlob snapshotOfSnapshot = snapshot.createSnapshot();
fail("Should not be able to create snapshot of snapshot.");
}
catch (IllegalArgumentException ex) {
// Expected exception was thrown, continue
}
}
/******************************************************************************************************************/
/* A callable class that encapsulates a partial upload of a file. A number of instances are passed to an
ExecutorService to upload multiple portions of a file concurrently.
*/
private class BlockUploadTask implements Callable<Void> {
private final String id;
private final ByteArrayInputStream byteStream;
private final CloudBlockBlob blob;
BlockUploadTask(String id, ByteArrayInputStream byteStream, CloudBlockBlob blob) {
this.id = id;
this.byteStream = byteStream;
this.blob = blob;
}
public String getId() { return id; }
public Void call() {
try {
blob.uploadBlock(id, byteStream, -1);
} catch (StorageException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
// Close the bytestream cleanly after upload.
try {
byteStream.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
return null;
}
}
@Test
public void parallelBlockUpload() throws Exception {
CloudBlockBlob blob = container.getBlockBlobReference("parallelBlockUpload.jpg");
URL imageUrl = BlockBlobTest.class.getResource("largeimage.jpg");
File largeFile = new File(imageUrl.getFile());
/* This code won't work with files over (numThreads * 2GB) in size because we couldn't fit a portion of
the file into a byte array, whose maximum size would be Integer.MAX_VALUE (around 2GB worth of bytes).
Rather than over-complicate the code to support extreme files, I'm just not going to.
*/
int numThreads = 8;
if((largeFile.length() / numThreads) > Integer.MAX_VALUE)
fail("Source image file is too large.");
int bufferSize = (int) largeFile.length() / numThreads;
List<BlockUploadTask> uploadTasks = new ArrayList<>();
/* Read the image file into multiple byte streams, creating a list of callable upload tasks.
Also generate a base64-encoded ID for each block entry.
*/
MessageDigest uploadMD5 = MessageDigest.getInstance("MD5");
Base64.Encoder encoder = Base64.getEncoder();
try(DigestInputStream inputStream = new DigestInputStream(new FileInputStream(largeFile), uploadMD5)) {
int count = 1;
while(inputStream.available() > 0) {
String id = encoder.encodeToString(String.format("%d", count).getBytes());
byte[] bytes = new byte[bufferSize];
inputStream.read(bytes);
uploadTasks.add(new BlockUploadTask(id, new ByteArrayInputStream(bytes), blob));
count++;
}
}
// Perform the parallel execution of all the upload tasks.
ExecutorService executor = Executors.newFixedThreadPool(numThreads);
List<Future<Void>> results = executor.invokeAll(uploadTasks);
executor.shutdown(); // blocks until executor work is finished
/* Build the block list in the order that the uploaded blocks need to be assembled by the service,
this is basically an ordered list of the base64-encoded block ids. Then commit the list to complete
the upload.
*/
ArrayList<BlockEntry> blocks = new ArrayList<>();
uploadTasks.forEach((task) -> blocks.add(new BlockEntry(task.getId())));
blob.commitBlockList(blocks);
assertTrue(blob.exists());
/* The blob service won't calculate the MD5 content of a parallel block upload for us, like it does
for simple uploads. Let's download the file ourselves and make sure the MD5 matches.
*/
File tempFile = File.createTempFile("parallelBlockUpload", ".jpg");
MessageDigest downloadMD5 = MessageDigest.getInstance("MD5");
try (DigestOutputStream outputStream = new DigestOutputStream(new FileOutputStream(tempFile.getAbsolutePath()), downloadMD5)) {
blob.download(outputStream);
}
assertEquals(encoder.encodeToString(downloadMD5.digest()), encoder.encodeToString(uploadMD5.digest()));
}
/******************************************************************************************************************/
@Test
public void restrictingPermissions() throws Exception {
CloudBlockBlob blob = container.getBlockBlobReference("restrictingPermissions");
String blobContent = "Protected content.";
blob.uploadText(blobContent);
// Create a shared access policy which allows reads only and expires after five minutes.
SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
policy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ));
Date expiryTime = new Date(System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5));
policy.setSharedAccessExpiryTime(expiryTime);
// Generate a SAS token and shared access signature from the shared access policy.
String sasToken = blob.generateSharedAccessSignature(policy, null);
StorageCredentialsSharedAccessSignature credentials = new StorageCredentialsSharedAccessSignature(sasToken);
// From the SAS we can create a read-only version of the original blob.
CloudBlockBlob readOnlyBlob = new CloudBlockBlob(credentials.transformUri(blob.getUri()));
// Blob can be read successfully.
assertEquals(readOnlyBlob.downloadText(), blobContent);
// Blob cannot be written to.
try {
String text = "Should not be able to write to a blob with a SAS for reads only.";
readOnlyBlob.uploadText(text);
fail(text);
} catch (StorageException e) {
// Expected exception was thrown, continue
}
// Blob cannot be deleted.
try {
readOnlyBlob.delete();
fail("Should not be able to delete a blob with a SAS for reads only.");
} catch (StorageException e) {
// Expected exception was thrown, continue
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment