[DO NOT MERGE] Feat flex checksum

config/load_options.go

@@ -224,6 +224,10 @@ type LoadOptions struct {
 
 	// Specifies if response checksum should be validated
 	ResponseChecksumValidation aws.ResponseChecksumValidation
+
+	// Service endpoint override. This value is not necessarily final and is
+	// passed to the service's EndpointResolverV2 for further delegation.
+	BaseEndpoint string
 }
 
 func (o LoadOptions) getDefaultsMode(ctx context.Context) (aws.DefaultsMode, bool, error) {
@@ -299,6 +303,19 @@ func (o LoadOptions) getResponseChecksumValidation(ctx context.Context) (aws.Res
 	return o.ResponseChecksumValidation, o.ResponseChecksumValidation > 0, nil
 }
 
+func (o LoadOptions) getBaseEndpoint(context.Context) (string, bool, error) {
+	return o.BaseEndpoint, o.BaseEndpoint != "", nil
+}
+
+// GetServiceBaseEndpoint satisfies (internal/configsources).ServiceBaseEndpointProvider.
+//
+// The sdkID value is unused because LoadOptions only supports setting a GLOBAL
+// endpoint override. In-code, per-service endpoint overrides are performed via
+// functional options in service client space.
+func (o LoadOptions) GetServiceBaseEndpoint(context.Context, string) (string, bool, error) {
+	return o.BaseEndpoint, o.BaseEndpoint != "", nil
+}
+
 // WithRegion is a helper function to construct functional options
 // that sets Region on config's LoadOptions. Setting the region to
 // an empty string, will result in the region value being ignored.

feature/s3/manager/integ_upload_test.go

@@ -161,8 +161,9 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
 	}{
 		"auto single part": {
 			"no checksum": {
-				payload:    bytes.NewReader(singlePartBytes),
-				expectETag: singlePartETag,
+				payload:             bytes.NewReader(singlePartBytes),
+				expectChecksumCRC32: singlePartCRC32,
+				expectETag:          singlePartETag,
 			},
 			"CRC32": {
 				algorithm:           s3types.ChecksumAlgorithmCrc32,
@@ -215,30 +216,13 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
 				expectETag:           singlePartETag,
 			},
 			"MD5": {
-				payload:    bytes.NewReader(singlePartBytes),
-				contentMD5: singlePartMD5,
-				expectETag: singlePartETag,
+				payload:             bytes.NewReader(singlePartBytes),
+				contentMD5:          singlePartMD5,
+				expectChecksumCRC32: singlePartCRC32,
+				expectETag:          singlePartETag,
 			},
 		},
 		"auto multipart part": {
-			"no checksum": {
-				payload: bytes.NewReader(multiPartBytes),
-				expectParts: []s3types.CompletedPart{
-					{
-						ETag:       aws.String(singlePartETag),
-						PartNumber: aws.Int32(1),
-					},
-					{
-						ETag:       aws.String(singlePartETag),
-						PartNumber: aws.Int32(2),
-					},
-					{
-						ETag:       aws.String(multiPartTailETag),
-						PartNumber: aws.Int32(3),
-					},
-				},
-				expectETag: multiPartETag,
-			},
 			"CRC32": {
 				algorithm: s3types.ChecksumAlgorithmCrc32,
 				payload:   bytes.NewReader(multiPartBytes),

service/internal/checksum/algorithms.go

@@ -30,6 +30,9 @@ const (
 
 	// AlgorithmSHA256 represents SHA256 hash algorithm
 	AlgorithmSHA256 Algorithm = "SHA256"
+
+	// AlgorithmCRC64NVME represents CRC64NVME hash algorithm
+	AlgorithmCRC64NVME Algorithm = "CRC64NVME"
 )
 
 var supportedAlgorithms = []Algorithm{

service/internal/checksum/middleware_compute_input_checksum.go

@@ -7,6 +7,7 @@ import (
 	"hash"
 	"io"
 	"strconv"
+	"strings"
 
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	internalcontext "github.com/aws/aws-sdk-go-v2/internal/context"
@@ -129,13 +130,12 @@ func (m *computeInputPayloadChecksum) HandleFinalize(
 		})
 	}()
 
-	// If the checksum header is already set nothing to do.
-	for _, supportedAlgorithm := range supportedAlgorithms {
-		header := AlgorithmHTTPHeader(supportedAlgorithm)
-		if cs := req.Header.Get(header); cs != "" {
-			// need to add header value to metadata
-			algorithm = supportedAlgorithm
-			checksum = cs
+	// If any checksum header is already set nothing to do.
+	for header := range req.Header {
+		h := strings.ToUpper(header)
+		if strings.HasPrefix(h, "X-AMZ-CHECKSUM-") {
+			algorithm = Algorithm(strings.TrimPrefix(h, "X-AMZ-CHECKSUM-"))
+			checksum = req.Header.Get(header)
 			return next.HandleFinalize(ctx, in)
 		}
 	}
@@ -180,7 +180,9 @@ func (m *computeInputPayloadChecksum) HandleFinalize(
 	// Only seekable streams are supported for non-trailing checksums, because
 	// the stream needs to be rewound before the handler can continue.
 	if stream != nil && !req.IsStreamSeekable() {
-		return next.HandleFinalize(ctx, in)
+		return out, metadata, computeInputHeaderChecksumError{
+			Msg: "unseekable stream is not supported without TLS and trailing checksum",
+		}
 	}
 
 	var sha256Checksum string
@@ -274,10 +276,9 @@ func (m *addInputChecksumTrailer) HandleFinalize(
 		}
 	}
 
-	// If the checksum header is already set nothing to do.
-	for _, supportedAlgorithm := range supportedAlgorithms {
-		header := AlgorithmHTTPHeader(supportedAlgorithm)
-		if req.Header.Get(header) != "" {
+	// If any checksum header is already set nothing to do.
+	for header := range req.Header {
+		if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") {
 			return next.HandleFinalize(ctx, in)
 		}
 	}

service/internal/checksum/middleware_compute_input_checksum_test.go

@@ -138,6 +138,52 @@ func TestComputeInputPayloadChecksum(t *testing.T) {
 				expectHeader:        http.Header{},
 				expectPayload:       []byte("hello world"),
 			},
+			"http crc64 checksum header preset": {
+				initContext: func(ctx context.Context) context.Context {
+					return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32))
+				},
+				buildInput: middleware.BuildInput{
+					Request: func() *smithyhttp.Request {
+						r := smithyhttp.NewStackRequest().(*smithyhttp.Request)
+						r.URL, _ = url.Parse("http://example.aws")
+						r.ContentLength = 11
+						r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=")
+						r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world"))))
+						return r
+					}(),
+				},
+				expectHeader: http.Header{
+					"X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="},
+				},
+				expectContentLength: 11,
+				expectPayload:       []byte("hello world"),
+				expectChecksumMetadata: map[string]string{
+					"CRC64NVME": "S2Zv/ZHmbVs=",
+				},
+			},
+			"https crc64 checksum header preset": {
+				initContext: func(ctx context.Context) context.Context {
+					return internalcontext.SetChecksumInputAlgorithm(ctx, string(AlgorithmCRC32))
+				},
+				buildInput: middleware.BuildInput{
+					Request: func() *smithyhttp.Request {
+						r := smithyhttp.NewStackRequest().(*smithyhttp.Request)
+						r.URL, _ = url.Parse("https://example.aws")
+						r.ContentLength = 11
+						r.Header.Set(AlgorithmHTTPHeader(AlgorithmCRC64NVME), "S2Zv/ZHmbVs=")
+						r = requestMust(r.SetStream(bytes.NewReader([]byte("hello world"))))
+						return r
+					}(),
+				},
+				expectHeader: http.Header{
+					"X-Amz-Checksum-Crc64nvme": []string{"S2Zv/ZHmbVs="},
+				},
+				expectContentLength: 11,
+				expectPayload:       []byte("hello world"),
+				expectChecksumMetadata: map[string]string{
+					"CRC64NVME": "S2Zv/ZHmbVs=",
+				},
+			},
 		},
 
 		"build handled": {
@@ -449,9 +495,8 @@ func TestComputeInputPayloadChecksum(t *testing.T) {
 						return r
 					}(),
 				},
-				expectContentLength: -1,
-				expectHeader:        http.Header{},
-				expectPayload:       []byte("hello world"),
+				expectErr:      "unseekable stream is not supported",
+				expectBuildErr: true,
 			},
 			"http stream read error": {
 				initContext: func(ctx context.Context) context.Context {
@@ -504,9 +549,8 @@ func TestComputeInputPayloadChecksum(t *testing.T) {
 						return r
 					}(),
 				},
-				expectContentLength: -1,
-				expectHeader:        http.Header{},
-				expectPayload:       []byte("hello world"),
+				expectErr:      "unseekable stream is not supported",
+				expectBuildErr: true,
 			},
 		},
 

service/internal/integrationtest/s3/checksum_test.go

@@ -66,7 +66,14 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 				},
 				getObjectChecksumMode: s3types.ChecksumModeEnabled,
 				expectPayload:         []byte("abc123"),
-				expectLogged:          "Response has no supported checksum.",
+				expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{
+					ComputedChecksums: map[string]string{
+						"CRC32": "zwK7XA==",
+					},
+				},
+				expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{
+					AlgorithmsUsed: []string{"CRC32"},
+				},
 			},
 			"preset checksum": {
 				params: &s3.PutObjectInput{
@@ -170,7 +177,14 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 				},
 				getObjectChecksumMode: s3types.ChecksumModeEnabled,
 				expectPayload:         []byte("abc123"),
-				expectLogged:          "Response has no supported checksum.",
+				expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{
+					ComputedChecksums: map[string]string{
+						"CRC32": "zwK7XA==",
+					},
+				},
+				expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{
+					AlgorithmsUsed: []string{"CRC32"},
+				},
 			},
 			"preset checksum": {
 				params: &s3.PutObjectInput{
@@ -251,7 +265,14 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 			"no checksum": {
 				params:                &s3.PutObjectInput{},
 				getObjectChecksumMode: s3types.ChecksumModeEnabled,
-				expectLogged:          "Response has no supported checksum.",
+				expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{
+					ComputedChecksums: map[string]string{
+						"CRC32": "AAAAAA==",
+					},
+				},
+				expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{
+					AlgorithmsUsed: []string{"CRC32"},
+				},
 			},
 			"preset checksum": {
 				params: &s3.PutObjectInput{
@@ -304,7 +325,14 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 					Body: bytes.NewBuffer([]byte{}),
 				},
 				getObjectChecksumMode: s3types.ChecksumModeEnabled,
-				expectLogged:          "Response has no supported checksum.",
+				expectComputedChecksums: &s3.ComputedInputChecksumsMetadata{
+					ComputedChecksums: map[string]string{
+						"CRC32": "AAAAAA==",
+					},
+				},
+				expectAlgorithmsUsed: &s3.ChecksumValidationMetadata{
+					AlgorithmsUsed: []string{"CRC32"},
+				},
 			},
 			"preset checksum": {
 				params: &s3.PutObjectInput{
@@ -394,7 +422,7 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 					}
 					// assert computed input checksums metadata
 					computedChecksums, ok := s3.GetComputedInputChecksumsMetadata(putResult.ResultMetadata)
-					if e, a := ok, (c.expectComputedChecksums != nil); e != a {
+					if e, a := (c.expectComputedChecksums != nil), ok; e != a {
 						t.Fatalf("expect computed checksum metadata %t, got %t, %v", e, a, computedChecksums)
 					}
 					if c.expectComputedChecksums != nil {
@@ -442,7 +470,7 @@ func TestInteg_ObjectChecksums(t *testing.T) {
 
 					// assert checksum validation metadata
 					algorithmsUsed, ok := s3.GetChecksumValidationMetadata(getResult.ResultMetadata)
-					if e, a := ok, (c.expectAlgorithmsUsed != nil); e != a {
+					if e, a := (c.expectAlgorithmsUsed != nil), ok; e != a {
 						t.Fatalf("expect algorithms used metadata %t, got %t, %v", e, a, algorithmsUsed)
 					}
 					if c.expectAlgorithmsUsed != nil {
@@ -468,7 +496,7 @@ func TestInteg_RequireChecksum(t *testing.T) {
 		expectComputedChecksums []string
 	}{
 		"no algorithm": {
-			expectComputedChecksums: []string{"MD5"},
+			expectComputedChecksums: []string{"CRC32"},
 		},
 		"with algorithm": {
 			checksumAlgorithm:       types.ChecksumAlgorithmCrc32c,