Merge pull request #2907 from redpanda-data/s3-output-checksum-algori…

…thm-field

Add checksum_algorithm to aws_s3 output

CHANGELOG.md

@@ -3,6 +3,12 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## 4.38.0 - TBD
+
+### Added
+
+- Field `checksum_algorithm` added to the `aws_s3` output. (@dom-lee-naimuri)
+
 ## 4.37.0 - 2024-09-26
 
 ### Added

docs/modules/components/pages/outputs/aws_s3.adoc

@@ -77,6 +77,7 @@ output:
       exclude_prefixes: []
     storage_class: STANDARD
     kms_key_id: ""
+    checksum_algorithm: ""
     server_side_encryption: ""
     force_path_style_urls: false
     max_in_flight: 64
@@ -329,6 +330,22 @@ An optional server side encryption key.
 
 *Default*: `""`
 
+=== `checksum_algorithm`
+
+The algorithm used to create the checksum for each object.
+
+
+*Type*: `string`
+
+*Default*: `""`
+
+Options:
+`CRC32`
+, `CRC32C`
+, `SHA1`
+, `SHA256`
+.
+
 === `server_side_encryption`
 
 An optional server side encryption algorithm.

internal/impl/aws/output_s3.go

@@ -40,6 +40,7 @@ const (
 	s3oFieldForcePathStyleURLs      = "force_path_style_urls"
 	s3oFieldPath                    = "path"
 	s3oFieldTags                    = "tags"
+	s3oFieldChecksumAlgorithm       = "checksum_algorithm"
 	s3oFieldContentType             = "content_type"
 	s3oFieldContentEncoding         = "content_encoding"
 	s3oFieldCacheControl            = "cache_control"
@@ -68,6 +69,7 @@ type s3oConfig struct {
 	ContentType             *service.InterpolatedString
 	ContentEncoding         *service.InterpolatedString
 	CacheControl            *service.InterpolatedString
+	ChecksumAlgorithm       string
 	ContentDisposition      *service.InterpolatedString
 	ContentLanguage         *service.InterpolatedString
 	ContentMD5              *service.InterpolatedString
@@ -126,6 +128,9 @@ func s3oConfigFromParsed(pConf *service.ParsedConfig) (conf s3oConfig, err error
 	if conf.ContentMD5, err = pConf.FieldInterpolatedString(s3oFieldContentMD5); err != nil {
 		return
 	}
+	if conf.ChecksumAlgorithm, err = pConf.FieldString(s3oFieldChecksumAlgorithm); err != nil {
+		return
+	}
 	if conf.WebsiteRedirectLocation, err = pConf.FieldInterpolatedString(s3oFieldWebsiteRedirectLocation); err != nil {
 		return
 	}
@@ -270,6 +275,12 @@ output:
 				Description("An optional server side encryption key.").
 				Default("").
 				Advanced(),
+			service.NewStringEnumField(s3oFieldChecksumAlgorithm,
+				"CRC32", "CRC32C", "SHA1", "SHA256",
+			).
+				Description("The algorithm used to create the checksum for each object.").
+				Default("").
+				Advanced(),
 			service.NewStringField(s3oFieldServerSideEncryption).
 				Description("An optional server side encryption algorithm.").
 				Version("3.63.0").
@@ -448,6 +459,10 @@ func (a *amazonS3Writer) WriteBatch(wctx context.Context, msg service.MessageBat
 			uploadInput.SSEKMSKeyId = &a.conf.KMSKeyID
 		}
 
+		if a.conf.ChecksumAlgorithm != "" {
+			uploadInput.ChecksumAlgorithm = types.ChecksumAlgorithm(a.conf.ChecksumAlgorithm)
+		}
+
 		// NOTE: This overrides the ServerSideEncryption set above. We need this to preserve
 		// backwards compatibility, where it is allowed to only set kms_key_id in the config and
 		// the ServerSideEncryption value of "aws:kms" is implied.