gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

s3: support STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER for signed chunked uploads with checksums (#7623)

Browse Source 
* s3: support STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER for signed chunked uploads with checksums

When AWS SDK v2 clients upload with both chunked encoding and checksum
validation enabled, they use the x-amz-content-sha256 header value of
STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER instead of the simpler
STREAMING-AWS4-HMAC-SHA256-PAYLOAD.

This caused the chunked reader to not be properly activated, resulting
in chunk-signature metadata being stored as part of the file content.

Changes:
- Add streamingSignedPayloadTrailer constant for the new header value
- Update isRequestSignStreamingV4() to recognize this header
- Update newChunkedReader() to handle this streaming type
- Update calculateSeedSignature() to accept this header
- Add unit test for signed streaming upload with trailer

Fixes issue where Quarkus/AWS SDK v2 uploads with checksum validation
resulted in corrupted file content containing chunk-signature data.

* address review comments: add trailer signature to test, fix constant alignment

* test: separate canonical trailer text (\n) from on-wire format (\r\n)

* test: add negative test for invalid trailer signature

* refactor: check HTTP method first in streaming auth checks (fail-fast)

* test: handle crc32 Write error return for completeness

* refactor: extract createTrailerStreamingRequest helper to reduce test duplication

* fmt

* docs: clarify test comment about trailer signature validation status

* refactor: calculate chunk data length dynamically instead of hardcoding

* Update weed/s3api/chunked_reader_v4_test.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* fix: use current time for signatures instead of hardcoded past date

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Chris Lu
2025-12-04 14:51:37 -08:00
committed by GitHub
parent a5ab05ec03
commit 716f21fbd3
4 changed files with 171 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
weed/s3api/auth_signature_v4.go
View File

@@ -53,10 +53,11 @@ func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Ide
// Constants specific to this file
const (
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
unsignedPayload = "UNSIGNED-PAYLOAD"
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
unsignedPayload = "UNSIGNED-PAYLOAD"
// Limit for IAM/STS request body size to prevent DoS attacks
iamRequestBodyLimit = 10 * (1 << 20) // 10 MiB
)
@@ -214,14 +215,14 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
availableKeys = append(availableKeys, key)
}
iam.m.RUnlock()
glog.Warningf("InvalidAccessKeyId: attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
authInfo.AccessKey, len(availableKeys), iam.isAuthEnabled)
if glog.V(2) && len(availableKeys) > 0 {
glog.V(2).Infof("Available access keys: %v", availableKeys)
}
return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID
}
@@ -562,10 +563,10 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
iam.m.RLock()
availableKeyCount := len(iam.accessKeyIdent)
iam.m.RUnlock()
glog.Warningf("InvalidAccessKeyId (POST policy): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
credHeader.accessKey, availableKeyCount, iam.isAuthEnabled)
return s3err.ErrInvalidAccessKeyID
}

10
weed/s3api/chunked_reader_v4.go
View File

@@ -53,8 +53,8 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
// This check ensures we only proceed for streaming uploads.
switch authInfo.HashedPayload {
case streamingContentSHA256:
glog.V(3).Infof("streaming content sha256")
case streamingContentSHA256, streamingContentSHA256Trailer:
glog.V(3).Infof("streaming content sha256 (with trailer: %v)", authInfo.HashedPayload == streamingContentSHA256Trailer)
case streamingUnsignedPayload:
glog.V(3).Infof("streaming unsigned payload")
default:
@@ -87,9 +87,9 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea
var errCode s3err.ErrorCode
switch contentSha256Header {
// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
case streamingContentSHA256:
glog.V(3).Infof("streaming content sha256")
// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' or 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER'
case streamingContentSHA256, streamingContentSHA256Trailer:
glog.V(3).Infof("streaming content sha256 (with trailer: %v)", contentSha256Header == streamingContentSHA256Trailer)
credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req)
if errCode != s3err.ErrNone {
return nil, errCode

144
weed/s3api/chunked_reader_v4_test.go
View File

@@ -234,6 +234,150 @@ func TestSignedStreamingUpload(t *testing.T) {
assert.Equal(t, chunk1Data+chunk2Data, string(data))
}
// createTrailerStreamingRequest creates a streaming upload request with trailer for testing.
// If useValidTrailerSignature is true, uses a correctly calculated trailer signature;
// otherwise uses an intentionally wrong signature for negative testing.
func createTrailerStreamingRequest(t *testing.T, useValidTrailerSignature bool) (*http.Request, string) {
chunk1Data := "hello world\n"
chunk1DataLen := len(chunk1Data)
chunk1DataLenHex := fmt.Sprintf("%x", chunk1DataLen)
// Use current time for signatures
now := time.Now().UTC()
amzDate := now.Format(iso8601Format)
dateStamp := now.Format(yyyymmdd)
// Calculate seed signature
scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request"
// Build canonical request for seed signature
hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
canonicalHeaders := "content-encoding:aws-chunked\n" +
"host:s3.amazonaws.com\n" +
"x-amz-content-sha256:" + hashedPayload + "\n" +
"x-amz-date:" + amzDate + "\n" +
fmt.Sprintf("x-amz-decoded-content-length:%d\n", chunk1DataLen) +
"x-amz-trailer:x-amz-checksum-crc32\n"
signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-trailer"
canonicalRequest := "PUT\n" +
"/test-bucket/test-object\n" +
"\n" +
canonicalHeaders + "\n" +
signedHeaders + "\n" +
hashedPayload
canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest))
stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash
signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3")
seedSignature := getSignature(signingKey, stringToSign)
// Calculate chunk signatures
chunk1Hash := getSHA256Hash([]byte(chunk1Data))
chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash
chunk1Signature := getSignature(signingKey, chunk1StringToSign)
// Final chunk (0 bytes)
finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" +
chunk1Signature + "\n" + emptySHA256 + "\n" + emptySHA256
finalSignature := getSignature(signingKey, finalStringToSign)
// Calculate CRC32 checksum for trailer
crcWriter := crc32.NewIEEE()
_, crcErr := crcWriter.Write([]byte(chunk1Data))
assert.NoError(t, crcErr)
checksum := crcWriter.Sum(nil)
base64EncodedChecksum := base64.StdEncoding.EncodeToString(checksum)
// The on-wire trailer format uses \r\n (HTTP/aws-chunked convention)
trailerOnWire := "x-amz-checksum-crc32:" + base64EncodedChecksum + "\r\n"
// Calculate or use wrong trailer signature
var trailerSignature string
if useValidTrailerSignature {
// The canonical trailer content uses \n for signing (per AWS SigV4 spec)
trailerCanonical := "x-amz-checksum-crc32:" + base64EncodedChecksum + "\n"
trailerHash := getSHA256Hash([]byte(trailerCanonical))
trailerStringToSign := "AWS4-HMAC-SHA256-TRAILER\n" + amzDate + "\n" + scope + "\n" +
finalSignature + "\n" + trailerHash
trailerSignature = getSignature(signingKey, trailerStringToSign)
} else {
// Intentionally wrong signature for negative testing
trailerSignature = "0000000000000000000000000000000000000000000000000000000000000000"
}
// Build the chunked payload with trailer and trailer signature
payload := fmt.Sprintf("%s;chunk-signature=%s\r\n%s\r\n", chunk1DataLenHex, chunk1Signature, chunk1Data) +
fmt.Sprintf("0;chunk-signature=%s\r\n", finalSignature) +
trailerOnWire +
"x-amz-trailer-signature:" + trailerSignature + "\r\n" +
"\r\n"
// Create the request
req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object",
bytes.NewReader([]byte(payload)))
assert.NoError(t, err)
req.Header.Set("Host", "s3.amazonaws.com")
req.Header.Set("x-amz-date", amzDate)
req.Header.Set("x-amz-content-sha256", hashedPayload)
req.Header.Set("Content-Encoding", "aws-chunked")
req.Header.Set("x-amz-decoded-content-length", fmt.Sprintf("%d", chunk1DataLen))
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
defaultAccessKeyId, scope, signedHeaders, seedSignature)
req.Header.Set("Authorization", authHeader)
return req, chunk1Data
}
// TestSignedStreamingUploadWithTrailer tests streaming uploads with signed chunks and trailers
// This tests the STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER content-sha256 header value
// which is used by AWS SDK v2 when checksum validation is enabled
func TestSignedStreamingUploadWithTrailer(t *testing.T) {
iam := setupIam()
req, expectedData := createTrailerStreamingRequest(t, true)
// Test the chunked reader
reader, errCode := iam.newChunkedReader(req)
assert.Equal(t, s3err.ErrNone, errCode)
assert.NotNil(t, reader)
// Read and verify the payload
data, err := io.ReadAll(reader)
assert.NoError(t, err)
assert.Equal(t, expectedData, string(data))
}
// TestSignedStreamingUploadWithTrailerInvalidSignature tests behavior with invalid trailer signatures.
// This is a negative test case for trailer signature validation. It currently verifies that an invalid
// signature doesn't break content reading, and is prepared for when validation is implemented.
func TestSignedStreamingUploadWithTrailerInvalidSignature(t *testing.T) {
iam := setupIam()
req, expectedData := createTrailerStreamingRequest(t, false)
// Test the chunked reader - it should be created successfully
reader, errCode := iam.newChunkedReader(req)
assert.Equal(t, s3err.ErrNone, errCode)
assert.NotNil(t, reader)
// Read the payload - currently trailer signature validation may not be implemented,
// but this test documents the expected behavior and will catch regressions
// if trailer signature validation is added in the future
data, err := io.ReadAll(reader)
// Note: If trailer signature validation is implemented, this should fail with an error
// For now, we just verify the content is correctly extracted
if err != nil {
assert.Contains(t, err.Error(), "signature", "Error should indicate signature mismatch")
} else {
// If no error, content should still be correct (trailer sig validation not yet implemented)
assert.Equal(t, expectedData, string(data))
}
}
// TestSignedStreamingUploadInvalidSignature tests that invalid chunk signatures are rejected
// This is a negative test case to ensure signature validation is actually working
func TestSignedStreamingUploadInvalidSignature(t *testing.T) {

16
weed/s3api/s3api_auth.go
View File

@@ -48,14 +48,22 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
}
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
// Supports both with and without trailer variants:
// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD (original)
// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (with trailing checksums)
func isRequestSignStreamingV4(r *http.Request) bool {
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
r.Method == http.MethodPut
if r.Method != http.MethodPut {
return false
}
contentSha256 := r.Header.Get("x-amz-content-sha256")
return contentSha256 == streamingContentSHA256 || contentSha256 == streamingContentSHA256Trailer
}
func isRequestUnsignedStreaming(r *http.Request) bool {
return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload &&
r.Method == http.MethodPut
if r.Method != http.MethodPut {
return false
}
return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload
}
// Authorization type.