feat(s3): store and return checksum headers for additional checksum algorithms (#8914)
* feat(s3): store and return checksum headers for additional checksum algorithms When clients upload with --checksum-algorithm (SHA256, CRC32, etc.), SeaweedFS validated the checksum but discarded it. The checksum was never stored in metadata or returned in PUT/HEAD/GET responses. Now the checksum is computed alongside MD5 during upload, stored in entry extended attributes, and returned as the appropriate x-amz-checksum-* header in all responses. Fixes #8911 * fix(s3): address review feedback and CI failures for checksum support - Gate GET/HEAD checksum response headers on x-amz-checksum-mode: ENABLED per AWS S3 spec, fixing FlexibleChecksumError on ranged GETs and multipart copies - Verify computed checksum against client-provided header value for non-chunked uploads, returning BadDigest on mismatch - Add nil check for getCheckSumWriter to prevent panic - Handle comma-separated values in X-Amz-Trailer header - Use ordered slice instead of map for deterministic checksum header selection; extract shared mappings into package-level vars * fix(s3): skip checksum header for ranged GET responses The stored checksum covers the full object. Returning it for ranged (partial) responses causes SDK checksum validation failures because the SDK validates the header value against the partial content received. Skip emitting x-amz-checksum-* headers when a Range request header is present, fixing PyArrow large file read failures. * fix(s3): reject unsupported checksum algorithm with 400 detectRequestedChecksumAlgorithm now returns an error code when x-amz-sdk-checksum-algorithm or x-amz-checksum-algorithm contains an unsupported value, instead of silently ignoring it. * feat(s3): compute composite checksum for multipart uploads Store the checksum algorithm during CreateMultipartUpload, then during CompleteMultipartUpload compute a composite checksum from per-part checksums following the AWS S3 spec: concatenate raw per-part checksums, hash with the same algorithm, format as "base64-N" where N is part count. The composite checksum is persisted on the final object entry and returned in HEAD/GET responses (gated on x-amz-checksum-mode: ENABLED). Reuses existing per-part checksum storage from putToFiler and the getCheckSumWriter/checksumHeaders infrastructure. * fix(s3): validate checksum algorithm in CreateMultipartUpload, error on missing part checksums - Move detectRequestedChecksumAlgorithm call before mkdir callback so an unsupported algorithm returns 400 before the upload is created - Change computeCompositeChecksum to return an error when a part is missing its checksum (the upload was initiated with a checksum algorithm, so all parts must have checksums) - Propagate the error as ErrInvalidPart in CompleteMultipartUpload * fix(s3): return checksum header in CompleteMultipartUpload response, validate per-part algorithm - Add ChecksumHeaderName/ChecksumValue fields to CompleteMultipartUploadResult and set the x-amz-checksum-* HTTP response header in the handler, matching the AWS S3 CompleteMultipartUpload response spec - Validate that each part's stored checksum algorithm matches the upload's expected algorithm before assembling the composite checksum; return an error if a part was uploaded with a different algorithm
This commit is contained in:
@@ -65,6 +65,12 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
|||||||
|
|
||||||
uploadIdString = uploadIdString + "_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
uploadIdString = uploadIdString + "_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||||
|
|
||||||
|
// Validate checksum algorithm before creating the upload directory
|
||||||
|
_, checksumHeaderName, checksumErrCode := detectRequestedChecksumAlgorithm(r)
|
||||||
|
if checksumErrCode != s3err.ErrNone {
|
||||||
|
return nil, checksumErrCode
|
||||||
|
}
|
||||||
|
|
||||||
// Prepare error handling outside callback scope
|
// Prepare error handling outside callback scope
|
||||||
var encryptionError error
|
var encryptionError error
|
||||||
|
|
||||||
@@ -95,6 +101,12 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
|||||||
}
|
}
|
||||||
s3a.applyMultipartEncryptionConfig(entry, encryptionConfig)
|
s3a.applyMultipartEncryptionConfig(entry, encryptionConfig)
|
||||||
|
|
||||||
|
// Store the requested checksum algorithm so CompleteMultipartUpload can compute
|
||||||
|
// a composite checksum from per-part checksums
|
||||||
|
if checksumHeaderName != "" {
|
||||||
|
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(checksumHeaderName)
|
||||||
|
}
|
||||||
|
|
||||||
// Extract and store object lock metadata from request headers
|
// Extract and store object lock metadata from request headers
|
||||||
// This ensures object lock settings from create_multipart_upload are preserved
|
// This ensures object lock settings from create_multipart_upload are preserved
|
||||||
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
|
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
|
||||||
@@ -129,6 +141,11 @@ type CompleteMultipartUploadResult struct {
|
|||||||
Bucket *string `xml:"Bucket,omitempty"`
|
Bucket *string `xml:"Bucket,omitempty"`
|
||||||
Key *string `xml:"Key,omitempty"`
|
Key *string `xml:"Key,omitempty"`
|
||||||
ETag *string `xml:"ETag,omitempty"`
|
ETag *string `xml:"ETag,omitempty"`
|
||||||
|
|
||||||
|
// Checksum fields — returned as HTTP response headers, not in the XML body
|
||||||
|
ChecksumHeaderName string `xml:"-"`
|
||||||
|
ChecksumValue string `xml:"-"`
|
||||||
|
|
||||||
// VersionId is NOT included in XML body - it should only be in x-amz-version-id HTTP header
|
// VersionId is NOT included in XML body - it should only be in x-amz-version-id HTTP header
|
||||||
|
|
||||||
// Store the VersionId internally for setting HTTP header, but don't marshal to XML
|
// Store the VersionId internally for setting HTTP header, but don't marshal to XML
|
||||||
@@ -173,15 +190,17 @@ type multipartPartBoundary struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type multipartCompletionState struct {
|
type multipartCompletionState struct {
|
||||||
deleteEntries []*filer_pb.Entry
|
deleteEntries []*filer_pb.Entry
|
||||||
partEntries map[int][]*filer_pb.Entry
|
partEntries map[int][]*filer_pb.Entry
|
||||||
pentry *filer_pb.Entry
|
pentry *filer_pb.Entry
|
||||||
mime string
|
mime string
|
||||||
finalParts []*filer_pb.FileChunk
|
finalParts []*filer_pb.FileChunk
|
||||||
offset int64
|
offset int64
|
||||||
partBoundaries []multipartPartBoundary
|
partBoundaries []multipartPartBoundary
|
||||||
multipartETag string
|
multipartETag string
|
||||||
entityWithTtl bool
|
entityWithTtl bool
|
||||||
|
checksumHeaderName string // e.g. "X-Amz-Checksum-Crc32", empty if no checksum
|
||||||
|
checksumValue string // composite base64 checksum with "-N" suffix
|
||||||
}
|
}
|
||||||
|
|
||||||
func completeMultipartResult(r *http.Request, input *s3.CompleteMultipartUploadInput, etag string, entry *filer_pb.Entry) *CompleteMultipartUploadResult {
|
func completeMultipartResult(r *http.Request, input *s3.CompleteMultipartUploadInput, etag string, entry *filer_pb.Entry) *CompleteMultipartUploadResult {
|
||||||
@@ -346,16 +365,36 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Compute composite checksum from per-part checksums if the upload
|
||||||
|
// was initiated with a checksum algorithm (stored in upload dir entry)
|
||||||
|
checksumHeaderName := ""
|
||||||
|
checksumValue := ""
|
||||||
|
if pentry.Extended != nil {
|
||||||
|
if algoName, ok := pentry.Extended[s3_constants.ExtChecksumAlgorithm]; ok {
|
||||||
|
checksumHeaderName = string(algoName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if checksumHeaderName != "" {
|
||||||
|
var checksumErr error
|
||||||
|
checksumValue, checksumErr = computeCompositeChecksum(checksumHeaderName, partEntries, completedPartNumbers)
|
||||||
|
if checksumErr != nil {
|
||||||
|
glog.Errorf("completeMultipartUpload: composite checksum computation failed: %v", checksumErr)
|
||||||
|
return nil, nil, s3err.ErrInvalidPart
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return &multipartCompletionState{
|
return &multipartCompletionState{
|
||||||
deleteEntries: deleteEntries,
|
deleteEntries: deleteEntries,
|
||||||
partEntries: partEntries,
|
partEntries: partEntries,
|
||||||
pentry: pentry,
|
pentry: pentry,
|
||||||
mime: mime,
|
mime: mime,
|
||||||
finalParts: finalParts,
|
finalParts: finalParts,
|
||||||
offset: offset,
|
offset: offset,
|
||||||
partBoundaries: partBoundaries,
|
partBoundaries: partBoundaries,
|
||||||
multipartETag: calculateMultipartETag(partEntries, completedPartNumbers),
|
multipartETag: calculateMultipartETag(partEntries, completedPartNumbers),
|
||||||
entityWithTtl: entityWithTtl,
|
entityWithTtl: entityWithTtl,
|
||||||
|
checksumHeaderName: checksumHeaderName,
|
||||||
|
checksumValue: checksumValue,
|
||||||
}, nil, s3err.ErrNone
|
}, nil, s3err.ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,6 +481,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
|
|
||||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||||
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||||
|
// Store composite checksum if computed from per-part checksums
|
||||||
|
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||||
|
versionEntry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||||
|
versionEntry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||||
|
}
|
||||||
|
|
||||||
// Preserve ALL SSE metadata from the first part (if any)
|
// Preserve ALL SSE metadata from the first part (if any)
|
||||||
// SSE metadata is stored in individual parts, not the upload directory
|
// SSE metadata is stored in individual parts, not the upload directory
|
||||||
@@ -490,11 +534,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
// For versioned buckets, all content is stored in .versions directory
|
// For versioned buckets, all content is stored in .versions directory
|
||||||
// The latest version information is tracked in the .versions directory metadata
|
// The latest version information is tracked in the .versions directory metadata
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
VersionId: aws.String(versionId),
|
VersionId: aws.String(versionId),
|
||||||
|
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||||
|
ChecksumValue: completionState.checksumValue,
|
||||||
}
|
}
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
}
|
}
|
||||||
@@ -534,6 +580,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
}
|
}
|
||||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||||
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||||
|
// Store composite checksum if computed from per-part checksums
|
||||||
|
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||||
|
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||||
|
entry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||||
|
}
|
||||||
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
||||||
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
||||||
} else if completionState.mime != "" {
|
} else if completionState.mime != "" {
|
||||||
@@ -547,10 +598,12 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
|
|
||||||
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
|
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||||
|
ChecksumValue: completionState.checksumValue,
|
||||||
// VersionId field intentionally omitted for suspended versioning
|
// VersionId field intentionally omitted for suspended versioning
|
||||||
}
|
}
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
@@ -589,6 +642,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
}
|
}
|
||||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||||
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||||
|
// Store composite checksum if computed from per-part checksums
|
||||||
|
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||||
|
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||||
|
entry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||||
|
}
|
||||||
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
||||||
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
||||||
} else if completionState.mime != "" {
|
} else if completionState.mime != "" {
|
||||||
@@ -606,10 +664,12 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
|||||||
|
|
||||||
// For non-versioned buckets, return response without VersionId
|
// For non-versioned buckets, return response without VersionId
|
||||||
output = &CompleteMultipartUploadResult{
|
output = &CompleteMultipartUploadResult{
|
||||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||||
Bucket: input.Bucket,
|
Bucket: input.Bucket,
|
||||||
ETag: aws.String(etagQuote),
|
ETag: aws.String(etagQuote),
|
||||||
Key: objectKey(input.Key),
|
Key: objectKey(input.Key),
|
||||||
|
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||||
|
ChecksumValue: completionState.checksumValue,
|
||||||
}
|
}
|
||||||
return s3err.ErrNone
|
return s3err.ErrNone
|
||||||
})
|
})
|
||||||
@@ -1030,6 +1090,72 @@ func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPart
|
|||||||
return fmt.Sprintf("%x-%d", md5.Sum(etags), len(completedPartNumbers))
|
return fmt.Sprintf("%x-%d", md5.Sum(etags), len(completedPartNumbers))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// computeCompositeChecksum computes a composite checksum from per-part checksums.
|
||||||
|
// It concatenates the raw (decoded) per-part checksums, hashes the result with the
|
||||||
|
// same algorithm, and returns the value as "base64-N" where N is the part count.
|
||||||
|
// This follows the AWS S3 multipart checksum specification.
|
||||||
|
// Returns an error if a part is missing its checksum (the upload was initiated with
|
||||||
|
// a checksum algorithm, so all parts must have been uploaded with checksums).
|
||||||
|
func computeCompositeChecksum(checksumHeaderName string, partEntries map[int][]*filer_pb.Entry, completedPartNumbers []int) (string, error) {
|
||||||
|
// Determine the algorithm from the header name
|
||||||
|
algo := checksumAlgorithmFromHeaderName(checksumHeaderName)
|
||||||
|
if algo == ChecksumAlgorithmNone {
|
||||||
|
return "", fmt.Errorf("unknown checksum algorithm for header %q", checksumHeaderName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect raw per-part checksums
|
||||||
|
var combined []byte
|
||||||
|
for _, partNumber := range completedPartNumbers {
|
||||||
|
entries, ok := partEntries[partNumber]
|
||||||
|
if !ok || len(entries) == 0 {
|
||||||
|
return "", fmt.Errorf("part %d not found", partNumber)
|
||||||
|
}
|
||||||
|
if len(entries) > 1 {
|
||||||
|
sortEntriesByLatestChunk(entries)
|
||||||
|
}
|
||||||
|
entry := entries[0]
|
||||||
|
if entry.Extended == nil {
|
||||||
|
return "", fmt.Errorf("part %d missing checksum: upload initiated with %s but part was uploaded without a checksum", partNumber, checksumHeaderName)
|
||||||
|
}
|
||||||
|
// Validate the part's checksum algorithm matches the upload's expected algorithm
|
||||||
|
partAlgo, ok := entry.Extended[s3_constants.ExtChecksumAlgorithm]
|
||||||
|
if !ok || len(partAlgo) == 0 {
|
||||||
|
return "", fmt.Errorf("part %d missing checksum: upload initiated with %s but part was uploaded without a checksum", partNumber, checksumHeaderName)
|
||||||
|
}
|
||||||
|
if string(partAlgo) != checksumHeaderName {
|
||||||
|
return "", fmt.Errorf("part %d checksum algorithm mismatch: upload expects %s but part has %s", partNumber, checksumHeaderName, string(partAlgo))
|
||||||
|
}
|
||||||
|
partChecksumB64, ok := entry.Extended[s3_constants.ExtChecksumValue]
|
||||||
|
if !ok || len(partChecksumB64) == 0 {
|
||||||
|
return "", fmt.Errorf("part %d missing checksum value: upload initiated with %s but part has no checksum value", partNumber, checksumHeaderName)
|
||||||
|
}
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(string(partChecksumB64))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("part %d has invalid checksum encoding: %w", partNumber, err)
|
||||||
|
}
|
||||||
|
combined = append(combined, raw...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hash the concatenated raw checksums
|
||||||
|
h := getCheckSumWriter(algo)
|
||||||
|
if h == nil {
|
||||||
|
return "", fmt.Errorf("failed to create hash writer for %s", checksumHeaderName)
|
||||||
|
}
|
||||||
|
h.Write(combined)
|
||||||
|
compositeRaw := h.Sum(nil)
|
||||||
|
return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(compositeRaw), len(completedPartNumbers)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checksumAlgorithmFromHeaderName maps a canonical header name back to its algorithm.
|
||||||
|
func checksumAlgorithmFromHeaderName(headerName string) ChecksumAlgorithm {
|
||||||
|
for _, entry := range checksumHeaders {
|
||||||
|
if entry.name == headerName {
|
||||||
|
return entry.alg
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ChecksumAlgorithmNone
|
||||||
|
}
|
||||||
|
|
||||||
func getEtagFromEntry(entry *filer_pb.Entry) string {
|
func getEtagFromEntry(entry *filer_pb.Entry) string {
|
||||||
if entry.Extended != nil {
|
if entry.Extended != nil {
|
||||||
if etagBytes, ok := entry.Extended[s3_constants.ExtETagKey]; ok {
|
if etagBytes, ok := entry.Extended[s3_constants.ExtETagKey]; ok {
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ const (
|
|||||||
ExtLatestVersionIsDeleteMarker = "Seaweed-X-Amz-Latest-Version-Is-Delete-Marker"
|
ExtLatestVersionIsDeleteMarker = "Seaweed-X-Amz-Latest-Version-Is-Delete-Marker"
|
||||||
ExtMultipartObjectKey = "key"
|
ExtMultipartObjectKey = "key"
|
||||||
|
|
||||||
|
// S3 checksum storage keys (use x-seaweedfs- prefix to avoid leaking in generic header loop)
|
||||||
|
ExtChecksumAlgorithm = "x-seaweedfs-checksum-algorithm"
|
||||||
|
ExtChecksumValue = "x-seaweedfs-checksum-value"
|
||||||
|
|
||||||
// Bucket Policy
|
// Bucket Policy
|
||||||
ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
|
ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
|
||||||
|
|
||||||
|
|||||||
@@ -76,6 +76,16 @@ const (
|
|||||||
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
|
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
|
||||||
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
|
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
|
||||||
|
|
||||||
|
// S3 checksum headers
|
||||||
|
AmzChecksumAlgorithm = "X-Amz-Checksum-Algorithm"
|
||||||
|
AmzChecksumCRC32 = "X-Amz-Checksum-Crc32"
|
||||||
|
AmzChecksumCRC32C = "X-Amz-Checksum-Crc32c"
|
||||||
|
AmzChecksumCRC64NVME = "X-Amz-Checksum-Crc64nvme"
|
||||||
|
AmzChecksumSHA1 = "X-Amz-Checksum-Sha1"
|
||||||
|
AmzChecksumSHA256 = "X-Amz-Checksum-Sha256"
|
||||||
|
AmzTrailer = "X-Amz-Trailer"
|
||||||
|
AmzSdkChecksumAlgorithm = "X-Amz-Sdk-Checksum-Algorithm"
|
||||||
|
|
||||||
// S3 conditional headers
|
// S3 conditional headers
|
||||||
IfMatch = "If-Match"
|
IfMatch = "If-Match"
|
||||||
IfNoneMatch = "If-None-Match"
|
IfNoneMatch = "If-None-Match"
|
||||||
|
|||||||
@@ -2068,6 +2068,21 @@ func (s3a *S3ApiServer) setResponseHeaders(w http.ResponseWriter, r *http.Reques
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set checksum header if stored in metadata, but only when:
|
||||||
|
// 1. The request contains "x-amz-checksum-mode: ENABLED" (per AWS S3 spec)
|
||||||
|
// 2. The request is NOT a ranged GET (Range header absent)
|
||||||
|
// The stored checksum covers the full object; returning it for partial
|
||||||
|
// responses causes SDK checksum validation failures.
|
||||||
|
if r != nil && r.Header.Get("X-Amz-Checksum-Mode") == "ENABLED" && r.Header.Get("Range") == "" {
|
||||||
|
if entry.Extended != nil {
|
||||||
|
if algoName, ok := entry.Extended[s3_constants.ExtChecksumAlgorithm]; ok {
|
||||||
|
if checksumVal, ok := entry.Extended[s3_constants.ExtChecksumValue]; ok {
|
||||||
|
w.Header().Set(string(algoName), string(checksumVal))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Apply S3 passthrough headers from query parameters
|
// Apply S3 passthrough headers from query parameters
|
||||||
// AWS S3 supports overriding response headers via query parameters like:
|
// AWS S3 supports overriding response headers via query parameters like:
|
||||||
// ?response-cache-control=no-cache&response-content-type=application/json
|
// ?response-cache-control=no-cache&response-content-type=application/json
|
||||||
|
|||||||
@@ -180,6 +180,11 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
|||||||
w.Header().Set("x-amz-version-id", *response.VersionId)
|
w.Header().Set("x-amz-version-id", *response.VersionId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set composite checksum header if present
|
||||||
|
if response.ChecksumHeaderName != "" && response.ChecksumValue != "" {
|
||||||
|
w.Header().Set(response.ChecksumHeaderName, response.ChecksumValue)
|
||||||
|
}
|
||||||
|
|
||||||
stats_collect.RecordBucketActiveTime(bucket)
|
stats_collect.RecordBucketActiveTime(bucket)
|
||||||
stats_collect.S3UploadedObjectsCounter.WithLabelValues(bucket).Inc()
|
stats_collect.S3UploadedObjectsCounter.WithLabelValues(bucket).Inc()
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -71,6 +72,9 @@ type SSEResponseMetadata struct {
|
|||||||
SSEType string
|
SSEType string
|
||||||
KMSKeyID string
|
KMSKeyID string
|
||||||
BucketKeyEnabled bool
|
BucketKeyEnabled bool
|
||||||
|
// Checksum fields for S3 additional checksum support
|
||||||
|
ChecksumHeaderName string // e.g. "X-Amz-Checksum-Sha256"
|
||||||
|
ChecksumValue string // base64-encoded checksum value
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -358,6 +362,19 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
|||||||
plaintextHash := md5.New()
|
plaintextHash := md5.New()
|
||||||
dataReader = io.TeeReader(dataReader, plaintextHash)
|
dataReader = io.TeeReader(dataReader, plaintextHash)
|
||||||
|
|
||||||
|
// Detect and set up additional checksum computation (S3 checksum algorithm support)
|
||||||
|
checksumAlgo, checksumHeaderName, checksumErrCode := detectRequestedChecksumAlgorithm(r)
|
||||||
|
if checksumErrCode != s3err.ErrNone {
|
||||||
|
return "", checksumErrCode, SSEResponseMetadata{}
|
||||||
|
}
|
||||||
|
var checksumHash hash.Hash
|
||||||
|
if checksumAlgo != ChecksumAlgorithmNone {
|
||||||
|
checksumHash = getCheckSumWriter(checksumAlgo)
|
||||||
|
if checksumHash != nil {
|
||||||
|
dataReader = io.TeeReader(dataReader, checksumHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Handle all SSE encryption types in a unified manner
|
// Handle all SSE encryption types in a unified manner
|
||||||
sseResult, sseErrorCode := s3a.handleAllSSEEncryption(r, dataReader, partOffset)
|
sseResult, sseErrorCode := s3a.handleAllSSEEncryption(r, dataReader, partOffset)
|
||||||
if sseErrorCode != s3err.ErrNone {
|
if sseErrorCode != s3err.ErrNone {
|
||||||
@@ -634,6 +651,24 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
|||||||
// Store ETag in Extended attribute for future retrieval (e.g. multipart parts)
|
// Store ETag in Extended attribute for future retrieval (e.g. multipart parts)
|
||||||
entry.Extended[s3_constants.ExtETagKey] = []byte(etag)
|
entry.Extended[s3_constants.ExtETagKey] = []byte(etag)
|
||||||
|
|
||||||
|
// Store additional checksum if one was computed
|
||||||
|
checksumBase64 := ""
|
||||||
|
if checksumHash != nil && checksumHeaderName != "" {
|
||||||
|
checksumBase64 = base64.StdEncoding.EncodeToString(checksumHash.Sum(nil))
|
||||||
|
// Verify against client-provided checksum if present in request headers
|
||||||
|
// (non-chunked uploads send the value directly; chunked uploads validate in the reader)
|
||||||
|
if expectedChecksum := r.Header.Get(checksumHeaderName); expectedChecksum != "" {
|
||||||
|
if expectedChecksum != checksumBase64 {
|
||||||
|
glog.Warningf("putToFiler: checksum mismatch for %s: expected %s, got %s", checksumHeaderName, expectedChecksum, checksumBase64)
|
||||||
|
s3a.deleteOrphanedChunks(chunkResult.FileChunks)
|
||||||
|
return "", s3err.ErrBadDigest, SSEResponseMetadata{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(checksumHeaderName)
|
||||||
|
entry.Extended[s3_constants.ExtChecksumValue] = []byte(checksumBase64)
|
||||||
|
glog.V(3).Infof("putToFiler: stored checksum %s=%s for %s", checksumHeaderName, checksumBase64, filePath)
|
||||||
|
}
|
||||||
|
|
||||||
// Set object owner according to bucket ownership settings.
|
// Set object owner according to bucket ownership settings.
|
||||||
s3a.setObjectOwnerFromRequest(r, bucket, entry)
|
s3a.setObjectOwnerFromRequest(r, bucket, entry)
|
||||||
|
|
||||||
@@ -802,7 +837,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
|||||||
|
|
||||||
// Build SSE response metadata with encryption details
|
// Build SSE response metadata with encryption details
|
||||||
responseMetadata := SSEResponseMetadata{
|
responseMetadata := SSEResponseMetadata{
|
||||||
SSEType: sseType,
|
SSEType: sseType,
|
||||||
|
ChecksumHeaderName: checksumHeaderName,
|
||||||
|
ChecksumValue: checksumBase64,
|
||||||
}
|
}
|
||||||
|
|
||||||
// For SSE-KMS, include key ID and bucket-key-enabled flag from stored metadata
|
// For SSE-KMS, include key ID and bucket-key-enabled flag from stored metadata
|
||||||
@@ -816,6 +853,93 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
|||||||
return etag, s3err.ErrNone, responseMetadata
|
return etag, s3err.ErrNone, responseMetadata
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checksumAlgorithmMapping maps algorithm name strings to their enum and header name.
|
||||||
|
var checksumAlgorithmMapping = map[string]struct {
|
||||||
|
alg ChecksumAlgorithm
|
||||||
|
name string
|
||||||
|
}{
|
||||||
|
"CRC32": {ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||||
|
"CRC32C": {ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||||
|
"CRC64NVME": {ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||||
|
"SHA1": {ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||||
|
"SHA256": {ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||||
|
}
|
||||||
|
|
||||||
|
// trailerToChecksumAlgorithm maps trailer header names to their algorithm and canonical header name.
|
||||||
|
var trailerToChecksumAlgorithm = map[string]struct {
|
||||||
|
alg ChecksumAlgorithm
|
||||||
|
name string
|
||||||
|
}{
|
||||||
|
"x-amz-checksum-crc32": {ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||||
|
"x-amz-checksum-crc32c": {ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||||
|
"x-amz-checksum-crc64nvme": {ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||||
|
"x-amz-checksum-sha1": {ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||||
|
"x-amz-checksum-sha256": {ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||||
|
}
|
||||||
|
|
||||||
|
// checksumHeaders is the ordered list of individual checksum headers to check.
|
||||||
|
// Using a slice ensures deterministic selection order.
|
||||||
|
var checksumHeaders = []struct {
|
||||||
|
header string
|
||||||
|
alg ChecksumAlgorithm
|
||||||
|
name string
|
||||||
|
}{
|
||||||
|
{s3_constants.AmzChecksumCRC32, ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||||
|
{s3_constants.AmzChecksumCRC32C, ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||||
|
{s3_constants.AmzChecksumCRC64NVME, ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||||
|
{s3_constants.AmzChecksumSHA1, ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||||
|
{s3_constants.AmzChecksumSHA256, ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectRequestedChecksumAlgorithm detects the checksum algorithm requested by the client.
|
||||||
|
// It checks the x-amz-sdk-checksum-algorithm header, x-amz-checksum-algorithm header,
|
||||||
|
// x-amz-trailer header (including comma-separated values), and individual x-amz-checksum-*
|
||||||
|
// headers. Returns the algorithm enum, the canonical HTTP header name, and an error code
|
||||||
|
// if an unsupported algorithm is specified.
|
||||||
|
func detectRequestedChecksumAlgorithm(r *http.Request) (ChecksumAlgorithm, string, s3err.ErrorCode) {
|
||||||
|
// Check x-amz-sdk-checksum-algorithm (set by AWS SDKs)
|
||||||
|
if algo := r.Header.Get(s3_constants.AmzSdkChecksumAlgorithm); algo != "" {
|
||||||
|
if m, ok := checksumAlgorithmMapping[strings.ToUpper(algo)]; ok {
|
||||||
|
return m.alg, m.name, s3err.ErrNone
|
||||||
|
}
|
||||||
|
glog.Warningf("unsupported checksum algorithm in %s: %q", s3_constants.AmzSdkChecksumAlgorithm, algo)
|
||||||
|
return ChecksumAlgorithmNone, "", s3err.ErrInvalidRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check x-amz-checksum-algorithm header
|
||||||
|
if algo := r.Header.Get(s3_constants.AmzChecksumAlgorithm); algo != "" {
|
||||||
|
if m, ok := checksumAlgorithmMapping[strings.ToUpper(algo)]; ok {
|
||||||
|
return m.alg, m.name, s3err.ErrNone
|
||||||
|
}
|
||||||
|
glog.Warningf("unsupported checksum algorithm in %s: %q", s3_constants.AmzChecksumAlgorithm, algo)
|
||||||
|
return ChecksumAlgorithmNone, "", s3err.ErrInvalidRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check x-amz-trailer header (used by chunked uploads, may be comma-separated)
|
||||||
|
if trailer := r.Header.Get(s3_constants.AmzTrailer); trailer != "" {
|
||||||
|
for _, part := range strings.Split(trailer, ",") {
|
||||||
|
part = strings.TrimSpace(strings.ToLower(part))
|
||||||
|
if part == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m, ok := trailerToChecksumAlgorithm[part]; ok {
|
||||||
|
return m.alg, m.name, s3err.ErrNone
|
||||||
|
}
|
||||||
|
// Non-checksum trailers (e.g. x-amz-server-side-encryption) are fine — skip them
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check individual checksum headers (non-chunked uploads send the value directly)
|
||||||
|
// Uses ordered slice for deterministic selection
|
||||||
|
for _, entry := range checksumHeaders {
|
||||||
|
if r.Header.Get(entry.header) != "" {
|
||||||
|
return entry.alg, entry.name, s3err.ErrNone
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ChecksumAlgorithmNone, "", s3err.ErrNone
|
||||||
|
}
|
||||||
|
|
||||||
const defaultFileMode = uint32(0660)
|
const defaultFileMode = uint32(0660)
|
||||||
|
|
||||||
// resolveFileMode determines the file permission mode for an S3 upload.
|
// resolveFileMode determines the file permission mode for an S3 upload.
|
||||||
@@ -883,6 +1007,11 @@ func (s3a *S3ApiServer) setSSEResponseHeaders(w http.ResponseWriter, r *http.Req
|
|||||||
w.Header().Set(s3_constants.AmzServerSideEncryptionBucketKeyEnabled, "true")
|
w.Header().Set(s3_constants.AmzServerSideEncryptionBucketKeyEnabled, "true")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set checksum response header if a checksum was computed
|
||||||
|
if sseMetadata.ChecksumHeaderName != "" && sseMetadata.ChecksumValue != "" {
|
||||||
|
w.Header().Set(sseMetadata.ChecksumHeaderName, sseMetadata.ChecksumValue)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func filerErrorToS3Error(err error) s3err.ErrorCode {
|
func filerErrorToS3Error(err error) s3err.ErrorCode {
|
||||||
|
|||||||
Reference in New Issue
Block a user