Merge pull request #484 from Azure/legacy-dev

Legacy dev

ChangeLog.txt

@@ -1,3 +1,13 @@
+2019.08.05 Version 8.4.0
+ * Support for 2019-02-02 REST version. Please see our REST API documentation and blogs for information about the related added features.
+ * Added support for setting rehydrate priority for SetBlobTier and CopyBlob APIs.
+ * Added support for PutRangeFromURL API to writes bytes from one Azure File endpoint into the specified range of another Azure File endpoint.
+ * Added setDirectoryProperties, createPermission, and getPermission APIs to the File package.
+ * Added required headers for creatFile, createDirectory, setFileProperties, getFileProperties, getDirectoryProperties, getFile APIs.
+ * Updated getFileProperties, getDirectoryProperties, and getFile calls to update SMB properties.
+ * Added support for setting access tier for PutBlob/PutBlockList and CopyBlob APIs.
+ * Added support for batching blob operations. Currently the only batchable apis are deleteBlob and setBlobTier.
+
 2019.04.23 Version 8.3.0
  * Fixed a bug in the range download to InputStream that would throw an exception if the source blob was empty.
  * Added support for List/Close File Handles APIs.
@@ -9,6 +19,7 @@
  * Fixed a bug in which putBlockFromURI accessConditions were indicated as being on the destination when in actuality they only apply to the source.
  * Added a method to get share stats in bytes.
  * Support for snapshot-level shared access signature tokens on blobs.
+ * Support for customer-provided keys with server-side encryption.
 
 2019.03.01 Version 8.1.0
  * Added support for the deep sync copy blob api.

README.md

@@ -39,7 +39,7 @@ To get the binaries of this library as distributed by Microsoft, ready for use w
 <dependency>
 	<groupId>com.microsoft.azure</groupId>
 	<artifactId>azure-storage</artifactId>
-	<version>8.3.0</version>
+	<version>8.4.0</version>
 </dependency>
 ```
 

microsoft-azure-storage-samples/pom.xml

@@ -26,7 +26,7 @@
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>
 			<artifactId>azure-storage</artifactId>
-			<version>8.3.0</version>
+			<version>8.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>com.microsoft.azure</groupId>

microsoft-azure-storage-samples/src/com/microsoft/azure/storage/blob/gettingstarted/BlobBatch.java

@@ -0,0 +1,115 @@
+package com.microsoft.azure.storage.blob.gettingstarted;
+
+import com.microsoft.azure.storage.BatchException;
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.blob.*;
+import com.microsoft.azure.storage.util.Utility;
+
+import java.net.InetSocketAddress;
+import java.net.Proxy;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public class BlobBatch {
+    private static final String SAMPLE_NAME = "BlobBatch";
+
+    public static void main(String[] args) throws URISyntaxException, InvalidKeyException, StorageException {
+        Utility.printSampleStartInfo(SAMPLE_NAME);
+
+        // Setup the cloud storage account.
+        CloudStorageAccount account = CloudStorageAccount.parse(Utility.storageConnectionString);
+
+        // Create a blob service client
+        CloudBlobClient blobClient = account.createCloudBlobClient();
+
+        // Get a reference to a container
+        // The container name must be lower case
+        // Append a random UUID to the end of the container name so that
+        // this sample can be run more than once in quick succession.
+        CloudBlobContainer container = blobClient.getContainerReference("blobbasicscontainer"
+                + UUID.randomUUID().toString().replace("-", ""));
+
+        try {
+            // create the container
+            container.create();
+
+            // add some blobs
+            final int NUM_BLOBS = 3;
+            List<CloudBlob> blobs = new ArrayList<>();
+            for (int i = 0; i < NUM_BLOBS; i++) {
+                // blobs have more relaxed naming constraints than containers
+                CloudBlockBlob blob = container.getBlockBlobReference("blobtobatchdelete" + UUID.randomUUID());
+                blob.uploadText("Sample data.");
+                blobs.add(blob);
+
+                System.out.println(String.format("Created blob %s", blob.getName()));
+            }
+
+            // create a blob object on client, but don't upload it to the service, to cause a bad request in the batch
+            // comment this line out to get a success state
+            blobs.add(container.getBlockBlobReference("blobtobatchdelete" + UUID.randomUUID()));
+
+            // assemble batch request, in this case: delete
+            BlobDeleteBatchOperation batch = new BlobDeleteBatchOperation();
+            for (CloudBlob blob : blobs) {
+                batch.addSubOperation(blob);
+
+                System.out.println(String.format("Added delete request for blob %s", blob.getName()));
+            }
+
+            try {
+                // send the batch request
+                Map<CloudBlob, Void> batchResponse = container.getServiceClient().executeBatch(batch);
+
+                // for each blob, view the result
+                // Delete returns void, so its batch response maps to Void, but other requests may return data to process
+                for (CloudBlob blob : blobs) {
+                    Void result = batchResponse.get(blob);
+
+                    System.out.println(String.format("Result from deleting blob %s: %s", blob.getName(), result));
+                }
+            }
+
+            // when one or more requests in the batch fail
+            catch (BatchException e) {
+                /*
+                 * Subclasses of java.lang.Throwable cannot be generic, so the collections of successful and
+                 * unsuccessful responses cannot be typed correctly on the class. However, the types will directly
+                 * relate to the batched operation. Meaning: for a BlobBatchOperation<P, R>,
+                 * e.getSuccessfulResponses() can be safely cast to Map<P, R>, and e.getExceptions() can be
+                 * safely cast to Map<P, StorageException>. BlobDeleteBatchOperation extends
+                 * BlobBatchOperation<CloudBlob, Void> so we cast as follows.
+                 */
+                Map<CloudBlob, Void> successes = (Map<CloudBlob, Void>) e.getSuccessfulResponses();
+                Map<CloudBlob, StorageException> failures = (Map<CloudBlob, StorageException>) e.getExceptions();
+
+                // handle successes
+                for (Map.Entry<CloudBlob, Void> entry : successes.entrySet()) {
+                    System.out.println(String.format("Result from deleting blob %s: %s",
+                            entry.getKey().getName(), entry.getValue()));
+                }
+
+                // handle failures
+                for (Map.Entry<CloudBlob, StorageException> entry : failures.entrySet()) {
+                    System.out.println(String.format("Failed to delete blob %s. Exception: %s",
+                            entry.getKey().getName(), entry.getValue().getErrorCode()));
+                }
+            }
+        }
+        catch (Throwable t) {
+            Utility.printException(t);
+        }
+        finally {
+            // clean up sample
+            container.deleteIfExists();
+        }
+
+        Utility.printSampleCompleteInfo(SAMPLE_NAME);
+    }
+}

microsoft-azure-storage-samples/src/com/microsoft/azure/storage/logging/pom.xml

@@ -26,7 +26,7 @@
         <dependency>
             <groupId>com.microsoft.azure</groupId>
             <artifactId>azure-storage</artifactId>
-            <version>8.3.0</version>
+            <version>8.4.0</version>
         </dependency>
         <dependency>
             <groupId>com.microsoft.azure</groupId>

microsoft-azure-storage-samples/src/com/microsoft/azure/storage/util/Utility.java

@@ -33,9 +33,9 @@ public final class Utility {
      * 
      * Stores the storage connection string.
      */
-    public static final String storageConnectionString = "DefaultEndpointsProtocol=https;"
-            + "AccountName=[MY_ACCOUNT_NAME];"
-            + "AccountKey=[MY_ACCOUNT_KEY]";
+    public static final String storageConnectionString = "DefaultEndpointsProtocol=http;"
+            + "AccountName=jamesschreppler;"
+            + "AccountKey=5POtFgqT5vsDYmIUOSsEpoXnbd3iw+mpBiXB4jiMvdqOcprfPX3gZXGU/sGZviLbIIFDMq+5c00w7O2eG0UT6Q==";
 
     /**
      * You only need to modify the following values if you want to run the

microsoft-azure-storage-test/res/TestConfigurations.xml

@@ -30,6 +30,7 @@
     <ActiveDirectoryApplicationId>[APPLICATION_ID]</ActiveDirectoryApplicationId>
     <ActiveDirectoryApplicationSecret>[APPLICATION_SECRET]</ActiveDirectoryApplicationSecret>
     <ActiveDirectoryTenantId>[TENANT_ID]</ActiveDirectoryTenantId>
+    <ActiveDirectoryAuthEndpoint>[ENDPOINT]</ActiveDirectoryAuthEndpoint>
   </TenantConfiguration>
 </TenantConfigurations>
 </TestConfigurations>

microsoft-azure-storage-test/src/com/microsoft/azure/storage/OAuthTests.java

@@ -40,7 +40,7 @@ public void testOAuthTokenWithBlobClient() throws StorageException, URISyntaxExc
             blobClient.downloadServiceProperties();
             fail();
         } catch (StorageException e) {
-            assertEquals("AuthenticationFailed", e.getErrorCode());
+            assertEquals("InvalidAuthenticationInfo", e.getErrorCode());
         }
 
         // change the token again to see it succeed

microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/BlobTestHelper.java

@@ -21,10 +21,13 @@
 import com.microsoft.azure.storage.TestHelper;
 import com.microsoft.azure.storage.ServiceProperties;
 
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -479,4 +482,26 @@ public static HashMap<String, String> generateSampleMetadata(final int entries)
 
         return result;
     }
+
+    public static BlobCustomerProvidedKey generateCPK() {
+        KeyGenerator keyGen;
+        try {
+            keyGen = KeyGenerator.getInstance("AES");
+        }
+        catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+
+        keyGen.init(256);
+        SecretKey key = keyGen.generateKey();
+
+        BlobCustomerProvidedKey result;
+        try {
+            result = new BlobCustomerProvidedKey(key.getEncoded());
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+
+        return result;
+    }
 }

microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudAppendBlobTests.java

@@ -33,7 +33,10 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.HttpURLConnection;
+import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.InvalidKeyException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Calendar;
@@ -1404,4 +1407,122 @@ public void testAppendBlobAppendFromStreamWithLength() throws StorageException,
         blob.downloadAttributes();
         assertEquals(15 * 1024 * 1024, blob.getProperties().getLength());
     }
+
+    @Test
+    public void testAppendBlockCPK() throws URISyntaxException, StorageException, IOException {
+
+        final int BLOB_SIZE = 128 * Constants.KB;
+
+        // load CPK into options
+        BlobRequestOptions options = new BlobRequestOptions();
+        options.setCustomerProvidedKey(BlobTestHelper.generateCPK());
+
+        // get blob reference
+        CloudAppendBlob blob = this.container.getAppendBlobReference(
+                BlobTestHelper.generateRandomBlobNameWithPrefix("testAppendBlob"));
+        // force https
+        blob = new CloudAppendBlob(
+                new URL(blob.getUri().toString().replace("http", "https")).toURI(),
+                container.getServiceClient().getCredentials());
+
+        OperationContext operationContext = new OperationContext();
+        blob.createOrReplace(null, options, operationContext);
+
+        // validate response
+        assertTrue(operationContext.getRequestResults().get(0).isRequestServiceEncrypted());
+        assertEquals(
+                options.getCustomerProvidedKey().getKeySHA256(),
+                operationContext.getRequestResults().get(0).getEncryptionKeySHA256());
+
+        // generate random data
+        byte[] buffer = BlobTestHelper.getRandomBuffer(BLOB_SIZE);
+        ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
+
+        // append blocks with CPK
+        operationContext = new OperationContext();
+        blob.appendBlock(stream, BLOB_SIZE, null, options, operationContext);
+        stream.close();
+
+        // validate response
+        assertTrue(operationContext.getRequestResults().get(0).isRequestServiceEncrypted());
+        assertEquals(
+                options.getCustomerProvidedKey().getKeySHA256(),
+                operationContext.getRequestResults().get(0).getEncryptionKeySHA256());
+    }
+
+    @Test
+    public void testAppendBlockFromURLCPK() throws URISyntaxException, StorageException, IOException, InvalidKeyException {
+        // CPK on source blobs for append block from URL is not yet supported
+        // TODO uncomment the marked comments AND the work in the actual web request factories when the feature is ready
+
+        ////// SETUP
+
+        final int BLOB_SIZE = 128 * Constants.KB;
+
+        // make key for this blob
+        // NOT YET SUPPORTED // BlobCustomerProvidedKey srcBlobKey = BlobTestHelper.generateCPK();
+
+        // load CPK into srcOptions
+        BlobRequestOptions srcOptions = new BlobRequestOptions();
+        // NOT YET SUPPORTED // srcOptions.setCustomerProvidedKey(srcBlobKey);
+
+        // get blob reference
+        CloudBlockBlob temp = this.container.getBlockBlobReference(
+                BlobTestHelper.generateRandomBlobNameWithPrefix("testAppendBlob"));
+        // force https
+        temp = new CloudBlockBlob(
+                new URL(temp.getUri().toString().replace("http", "https")).toURI(),
+                container.getServiceClient().getCredentials());
+
+        // generate random data
+        byte[] buffer = BlobTestHelper.getRandomBuffer(BLOB_SIZE);
+        ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
+
+        // upload blob with CPK
+        temp.upload(stream, buffer.length, null, srcOptions, null);
+
+        // save blob URI with SAS authentication (important for later when this is a source URL)
+        SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy();
+        policy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.WRITE));
+        Calendar cal = Calendar.getInstance();
+        cal.add(Calendar.MINUTE, 10);
+        policy.setSharedAccessExpiryTime(cal.getTime());
+        String sas = temp.generateSharedAccessSignature(policy, null);
+        URI srcBlobURI = new URI(temp.getUri().toString() + "?" + sas);
+
+        // get blob reference
+        CloudAppendBlob blob = this.container.getAppendBlobReference(
+                BlobTestHelper.generateRandomBlobNameWithPrefix("testAppendBlob"));
+        // force https
+        blob = new CloudAppendBlob(
+                new URL(blob.getUri().toString().replace("http", "https")).toURI(),
+                container.getServiceClient().getCredentials());
+
+        ////// ACT
+
+        // load CPK into srcOptions
+        BlobCustomerProvidedKey blobKey = BlobTestHelper.generateCPK();
+        BlobRequestOptions options = new BlobRequestOptions();
+        options.setCustomerProvidedKey(blobKey);
+
+        // create append blob
+        OperationContext operationContext = new OperationContext();
+        blob.createOrReplace(null, options, operationContext);
+
+        // validate response
+        assertTrue(operationContext.getRequestResults().get(0).isRequestServiceEncrypted());
+        assertEquals(
+                options.getCustomerProvidedKey().getKeySHA256(),
+                operationContext.getRequestResults().get(0).getEncryptionKeySHA256());
+
+        // append blocks from URI
+        operationContext = new OperationContext();
+        blob.appendBlockFromURI(srcBlobURI, null, null, null, null, null, options, operationContext);
+
+        // validate response
+        assertTrue(operationContext.getRequestResults().get(0).isRequestServiceEncrypted());
+        assertEquals(
+                options.getCustomerProvidedKey().getKeySHA256(),
+                operationContext.getRequestResults().get(0).getEncryptionKeySHA256());
+    }
 }