From 6facc438caa9b7570e3dc39c5295d05632fb8078 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:44:12 -0700 Subject: [PATCH] test(#300): determine streaming range issue b/w regular & encryption --- pom.xml | 6 +- .../s3/examples/TestEndOfStreamBehavior.java | 136 ++++++++++++++++++ 2 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java diff --git a/pom.xml b/pom.xml index f11c0f1b5..888a07ac3 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ software.amazon.awssdk bom - 2.20.38 + 2.26.7 true pom import @@ -68,13 +68,13 @@ software.amazon.awssdk s3 - 2.20.38 + 2.26.7 software.amazon.awssdk kms - 2.20.38 + 2.26.7 true diff --git a/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java new file mode 100644 index 000000000..a3155a566 --- /dev/null +++ b/src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java @@ -0,0 +1,136 @@ +package software.amazon.encryption.s3.examples; + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.S3EncryptionClient; +import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources; + +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.*; + +public class TestEndOfStreamBehavior { + private static final Region DEFAULT_REGION = KMS_REGION; + private static final String KEY = "GHI-300.txt"; + @SuppressWarnings("SpellCheckingInspection") + private static final byte[] CONTENT = new String(new char[4]) + .replace("\0", "abcdefghijklmnopqrstuvwxyz0123456789") + .getBytes(); + /** The encryption key to use in client-side encryption tests. */ + protected static final KeyPair KEY_PAIR; + + static { + try { + KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static Stream clientProvider() { + return Stream.of( + getClient(DEFAULT_REGION), + getEncryptionClient(KEY_PAIR, DEFAULT_REGION)); + } + + @ParameterizedTest + @MethodSource("clientProvider") + void testEndOfStreamBehavior(final S3Client client) throws Exception { + // Delete the data if it exists + final DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .build(); + + client.deleteObject(deleteRequest); + + // Upload the data + final PutObjectRequest uploadRequest = + PutObjectRequest.builder().bucket(BUCKET).key(KEY).build(); + client.putObject(uploadRequest, RequestBody.fromBytes(CONTENT)); + // wait 5 seconds for the data to be uploaded + Thread.sleep(5000); + + // Actual test + final GetObjectRequest downloadRequest = + GetObjectRequest.builder() + .bucket(BUCKET) + .key(KEY) + .range("bytes=0-15") + .build(); + + final InputStream stream = client.getObject(downloadRequest); + + // Buffer capacity matters !!! + // Behavior difference when the capacity is same as the content length (i.e. 16) of the ranged query + final ByteBuffer buffer = ByteBuffer.allocate(16); + final byte[] underlyingBuffer = buffer.array(); + final int capacity = buffer.capacity(); + + final int END_OF_STREAM = -1; + int byteRead = 0; + int startPosition = 0; + while (byteRead != END_OF_STREAM) { + int lenToRead = capacity - startPosition; + System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead); + // @NathanEckert , about https://github.com/aws/amazon-s3-encryption-client-java/issues/300 + // Crypto Tools SOMETIMES got an Assertion Error from + // https://github.com/aws/aws-sdk-java-v2/blob/2.20.38/utils/src/main/java/software/amazon/awssdk/utils/async/InputStreamSubscriber.java#L110 + // when using the Encryption Client. + // If we bump our Java SDK dependencies to the latest, which today is 2.26.7, + // than we never get the Assertion Error. + // Here is the PR that changes InputStreamSubscriber b/w 2.20.38 and 2.26.7: + // https://github.com/aws/aws-sdk-java-v2/pull/5201 + // This makes us suspect that something else is going wrong. + // Otherwise, we cannot detect a difference in behavior between + // the S3EC V3 Client and the S3 V2 Client with respect to this code. + byteRead = stream.read(underlyingBuffer, startPosition, lenToRead); + System.out.println("Read " + byteRead + " bytes"); + startPosition += byteRead; + if (byteRead == 0) { + // Crypto Tools cannot get this case to ever occur. + System.out.println("Looping indefinitely with an encryption client, as startPosition is not increasing"); + break; + } + } + } + + public static S3Client getEncryptionClient(final KeyPair keyPair, final Region region) { + return S3EncryptionClient.builder() + .rsaKeyPair(keyPair) + .enableLegacyUnauthenticatedModes(true) + .wrappedClient(getClient(region)) + .wrappedAsyncClient(getAsyncClient(region)) + .build(); + } + + public static S3Client getClient(final Region region) { + return S3Client.builder() + .region(region) + .credentialsProvider(CREDENTIALS) + .httpClient(HTTP_CLIENT) + .build(); + } + + public static S3AsyncClient getAsyncClient(final Region region) { + final SdkAsyncHttpClient nettyHttpClient = + NettyNioAsyncHttpClient.builder().maxConcurrency(100).build(); + return S3AsyncClient.builder() + .region(region) + .credentialsProvider(CREDENTIALS) + .httpClient(nettyHttpClient) + .build(); + } +}