gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove ~50k lines of unreachable dead code (#8913)

Browse Source 
* chore: remove unreachable dead code across the codebase

Remove ~50,000 lines of unreachable code identified by static analysis.

Major removals:
- weed/filer/redis_lua: entire unused Redis Lua filer store implementation
- weed/wdclient/net2, resource_pool: unused connection/resource pool packages
- weed/plugin/worker/lifecycle: unused lifecycle plugin worker
- weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy,
  multipart IAM, key rotation, and various SSE helper functions
- weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions
- weed/mq/offset: unused SQL storage and migration code
- weed/worker: unused registry, task, and monitoring functions
- weed/query: unused SQL engine, parquet scanner, and type functions
- weed/shell: unused EC proportional rebalance functions
- weed/storage/erasure_coding/distribution: unused distribution analysis functions
- Individual unreachable functions removed from 150+ files across admin,
  credential, filer, iam, kms, mount, mq, operation, pb, s3api, server,
  shell, storage, topology, and util packages

* fix(s3): reset shared memory store in IAM test to prevent flaky failure

TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because
the MemoryStore credential backend is a singleton registered via init().
Earlier tests that create anonymous identities pollute the shared store,
causing LookupAnonymous() to unexpectedly return true.

Fix by calling Reset() on the memory store before the test runs.

* style: run gofmt on changed files

* fix: restore KMS functions used by integration tests

* fix(plugin): prevent panic on send to closed worker session channel

The Plugin.sendToWorker method could panic with "send on closed channel"
when a worker disconnected while a message was being sent. The race was
between streamSession.close() closing the outgoing channel and sendToWorker
writing to it concurrently.

Add a done channel to streamSession that is closed before the outgoing
channel, and check it in sendToWorker's select to safely detect closed
sessions without panicking.
This commit is contained in:
Chris Lu
2026-04-03 16:04:27 -07:00
committed by GitHub
parent 8fad85aed7
commit 995dfc4d5d
264 changed files with 62 additions and 46027 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
weed/s3api/auth_credentials.go
View File

@@ -144,6 +144,10 @@ func (c *Credential) isCredentialExpired() bool {
}
// NewIdentityAccessManagement creates a new IAM manager
func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, filerClient, "")
}
// SetFilerClient updates the filer client and its associated credential store
func (iam *IdentityAccessManagement) SetFilerClient(filerClient *wdclient.FilerClient) {
iam.m.Lock()
@@ -196,10 +200,6 @@ func parseExternalUrlToHost(externalUrl string) (string, error) {
return net.JoinHostPort(host, port), nil
}
func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, filerClient, "")
}
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient *wdclient.FilerClient, explicitStore string) *IdentityAccessManagement {
var externalHost string
if option.ExternalUrl != "" {

1393
weed/s3api/auth_credentials_test.go
View File

File diff suppressed because it is too large Load Diff

6
weed/s3api/bucket_metadata.go
View File

@@ -224,12 +224,6 @@ func (r *BucketRegistry) removeMetadataCache(bucket string) {
delete(r.metadataCache, bucket)
}
func (r *BucketRegistry) markNotFound(bucket string) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()
r.notFound[bucket] = struct{}{}
}
func (r *BucketRegistry) unMarkNotFound(bucket string) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()

267
weed/s3api/filer_multipart_test.go
View File

@@ -1,267 +0,0 @@
package s3api
import (
"encoding/hex"
"net/http"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
)
func TestInitiateMultipartUploadResult(t *testing.T) {
expected := `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
response := &InitiateMultipartUploadResult{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
},
}
encoded := string(s3err.EncodeXMLResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}
func TestListPartsResult(t *testing.T) {
expected := `<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Part><ETag>&#34;12345678&#34;</ETag><LastModified>1970-01-01T00:00:00Z</LastModified><PartNumber>1</PartNumber><Size>123</Size></Part></ListPartsResult>`
response := &ListPartsResult{
Part: []*s3.Part{
{
PartNumber: aws.Int64(int64(1)),
LastModified: aws.Time(time.Unix(0, 0).UTC()),
Size: aws.Int64(int64(123)),
ETag: aws.String("\"12345678\""),
},
},
}
encoded := string(s3err.EncodeXMLResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}
func TestCompleteMultipartResultIncludesVersionId(t *testing.T) {
r := &http.Request{Host: "localhost", Header: make(http.Header)}
input := &s3.CompleteMultipartUploadInput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
}
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("version-123"),
},
}
result := completeMultipartResult(r, input, "\"etag-value\"", entry)
if assert.NotNil(t, result.VersionId) {
assert.Equal(t, "version-123", *result.VersionId)
}
}
func TestCompleteMultipartResultOmitsNullVersionId(t *testing.T) {
r := &http.Request{Host: "localhost", Header: make(http.Header)}
input := &s3.CompleteMultipartUploadInput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
}
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("null"),
},
}
result := completeMultipartResult(r, input, "\"etag-value\"", entry)
assert.Nil(t, result.VersionId)
}
func Test_parsePartNumber(t *testing.T) {
tests := []struct {
name string
fileName string
partNum int
}{
{
"first",
"0001_uuid.part",
1,
},
{
"second",
"0002.part",
2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
partNumber, _ := parsePartNumber(tt.fileName)
assert.Equalf(t, tt.partNum, partNumber, "parsePartNumber(%v)", tt.fileName)
})
}
}
func TestGetEntryNameAndDir(t *testing.T) {
s3a := &S3ApiServer{
option: &S3ApiServerOption{
BucketsPath: "/buckets",
},
}
tests := []struct {
name string
bucket string
key string
expectedName string
expectedDirEnd string // We check the suffix since dir includes BucketsPath
}{
{
name: "simple file at root",
bucket: "test-bucket",
key: "/file.txt",
expectedName: "file.txt",
expectedDirEnd: "/buckets/test-bucket",
},
{
name: "file in subdirectory",
bucket: "test-bucket",
key: "/folder/file.txt",
expectedName: "file.txt",
expectedDirEnd: "/buckets/test-bucket/folder",
},
{
name: "file in nested subdirectory",
bucket: "test-bucket",
key: "/folder/subfolder/file.txt",
expectedName: "file.txt",
expectedDirEnd: "/buckets/test-bucket/folder/subfolder",
},
{
name: "key without leading slash",
bucket: "test-bucket",
key: "folder/file.txt",
expectedName: "file.txt",
expectedDirEnd: "/buckets/test-bucket/folder",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
input := &s3.CompleteMultipartUploadInput{
Bucket: aws.String(tt.bucket),
Key: aws.String(tt.key),
}
entryName, dirName := s3a.getEntryNameAndDir(input)
assert.Equal(t, tt.expectedName, entryName, "entry name mismatch")
assert.Equal(t, tt.expectedDirEnd, dirName, "directory mismatch")
})
}
}
func TestValidateCompletePartETag(t *testing.T) {
t.Run("matches_composite_etag_from_extended", func(t *testing.T) {
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("ea58527f14c6ae0dd53089966e44941b-2"),
},
Attributes: &filer_pb.FuseAttributes{},
}
match, invalid, part, stored := validateCompletePartETag(`"ea58527f14c6ae0dd53089966e44941b-2"`, entry)
assert.True(t, match)
assert.False(t, invalid)
assert.Equal(t, "ea58527f14c6ae0dd53089966e44941b-2", part)
assert.Equal(t, "ea58527f14c6ae0dd53089966e44941b-2", stored)
})
t.Run("matches_md5_from_attributes", func(t *testing.T) {
md5Bytes, err := hex.DecodeString("324b2665939fde5b8678d3a8b5c46970")
assert.NoError(t, err)
entry := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{
Md5: md5Bytes,
},
}
match, invalid, part, stored := validateCompletePartETag("324b2665939fde5b8678d3a8b5c46970", entry)
assert.True(t, match)
assert.False(t, invalid)
assert.Equal(t, "324b2665939fde5b8678d3a8b5c46970", part)
assert.Equal(t, "324b2665939fde5b8678d3a8b5c46970", stored)
})
t.Run("detects_mismatch", func(t *testing.T) {
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("67fdd2e302502ff9f9b606bc036e6892-2"),
},
Attributes: &filer_pb.FuseAttributes{},
}
match, invalid, _, _ := validateCompletePartETag("686f7d71bacdcd539dd4e17a0d7f1e5f-2", entry)
assert.False(t, match)
assert.False(t, invalid)
})
t.Run("flags_empty_client_etag_as_invalid", func(t *testing.T) {
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("67fdd2e302502ff9f9b606bc036e6892-2"),
},
Attributes: &filer_pb.FuseAttributes{},
}
match, invalid, _, _ := validateCompletePartETag(`""`, entry)
assert.False(t, match)
assert.True(t, invalid)
})
}
func TestCompleteMultipartUploadRejectsOutOfOrderParts(t *testing.T) {
s3a := NewS3ApiServerForTest()
input := &s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucket"),
Key: aws.String("object"),
UploadId: aws.String("upload"),
}
parts := &CompleteMultipartUpload{
Parts: []CompletedPart{
{PartNumber: 2, ETag: "\"etag-2\""},
{PartNumber: 1, ETag: "\"etag-1\""},
},
}
result, errCode := s3a.completeMultipartUpload(&http.Request{Header: make(http.Header)}, input, parts)
assert.Nil(t, result)
assert.Equal(t, s3err.ErrInvalidPartOrder, errCode)
}
func TestCompleteMultipartUploadAllowsDuplicatePartNumbers(t *testing.T) {
s3a := NewS3ApiServerForTest()
input := &s3.CompleteMultipartUploadInput{
Bucket: aws.String("bucket"),
Key: aws.String("object"),
UploadId: aws.String("upload"),
}
parts := &CompleteMultipartUpload{
Parts: []CompletedPart{
{PartNumber: 1, ETag: "\"etag-older\""},
{PartNumber: 1, ETag: "\"etag-newer\""},
},
}
result, errCode := s3a.completeMultipartUpload(&http.Request{Header: make(http.Header)}, input, parts)
assert.Nil(t, result)
assert.Equal(t, s3err.ErrNoSuchUpload, errCode)
}

16
weed/s3api/iam_optional_test.go
View File

@@ -3,9 +3,22 @@ package s3api
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/stretchr/testify/assert"
)
// resetMemoryStore resets the shared in-memory credential store so that tests
// that rely on an empty store are not polluted by earlier tests.
func resetMemoryStore() {
for _, store := range credential.Stores {
if store.GetName() == credential.StoreTypeMemory {
if resettable, ok := store.(interface{ Reset() }); ok {
resettable.Reset()
}
}
}
}
func TestLoadIAMManagerWithNoConfig(t *testing.T) {
// Verify that IAM can be initialized without any config
option := &S3ApiServerOption{
@@ -17,6 +30,9 @@ func TestLoadIAMManagerWithNoConfig(t *testing.T) {
}
func TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey(t *testing.T) {
// Reset the shared memory store to avoid state leaking from other tests.
resetMemoryStore()
// Initialize IAM with empty config — no anonymous identity is configured,
// so LookupAnonymous should return not-found.
option := &S3ApiServerOption{

19
weed/s3api/iceberg/commit_helpers.go
View File

@@ -6,8 +6,6 @@ import (
"errors"
"fmt"
"net/http"
"path"
"strconv"
"strings"
"github.com/apache/iceberg-go/table"
@@ -25,10 +23,6 @@ type icebergRequestError struct {
message string
}
func (e *icebergRequestError) Error() string {
return e.message
}
type createOnCommitInput struct {
bucketARN string
markerBucket string
@@ -88,19 +82,6 @@ func isS3TablesAlreadyExists(err error) bool {
(tableErr.Type == s3tables.ErrCodeTableAlreadyExists || tableErr.Type == s3tables.ErrCodeNamespaceAlreadyExists || strings.Contains(strings.ToLower(tableErr.Message), "already exists"))
}
func parseMetadataVersionFromLocation(metadataLocation string) int {
base := path.Base(metadataLocation)
if !strings.HasPrefix(base, "v") || !strings.HasSuffix(base, ".metadata.json") {
return 0
}
rawVersion := strings.TrimPrefix(strings.TrimSuffix(base, ".metadata.json"), "v")
version, err := strconv.Atoi(rawVersion)
if err != nil || version <= 0 {
return 0
}
return version
}
func (s *Server) finalizeCreateOnCommit(ctx context.Context, input createOnCommitInput) (*CommitTableResponse, *icebergRequestError) {
builder, err := table.MetadataBuilderFromBase(input.baseMetadata, input.baseMetadataLoc)
if err != nil {

76
weed/s3api/iceberg/iceberg_stage_create_helpers_test.go
View File

@@ -1,76 +0,0 @@
package iceberg
import (
"strings"
"testing"
"github.com/apache/iceberg-go/table"
"github.com/google/uuid"
)
func TestHasAssertCreateRequirement(t *testing.T) {
requirements := table.Requirements{table.AssertCreate()}
if !hasAssertCreateRequirement(requirements) {
t.Fatalf("hasAssertCreateRequirement() = false, want true")
}
requirements = table.Requirements{table.AssertDefaultSortOrderID(0)}
if hasAssertCreateRequirement(requirements) {
t.Fatalf("hasAssertCreateRequirement() = true, want false")
}
}
func TestParseMetadataVersionFromLocation(t *testing.T) {
testCases := []struct {
location string
version int
}{
{location: "s3://b/ns/t/metadata/v1.metadata.json", version: 1},
{location: "s3://b/ns/t/metadata/v25.metadata.json", version: 25},
{location: "v1.metadata.json", version: 1},
{location: "s3://b/ns/t/metadata/v0.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/v-1.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/vABC.metadata.json", version: 0},
{location: "s3://b/ns/t/metadata/current.json", version: 0},
{location: "", version: 0},
}
for _, tc := range testCases {
t.Run(tc.location, func(t *testing.T) {
if got := parseMetadataVersionFromLocation(tc.location); got != tc.version {
t.Errorf("parseMetadataVersionFromLocation(%q) = %d, want %d", tc.location, got, tc.version)
}
})
}
}
func TestStageCreateMarkerNamespaceKey(t *testing.T) {
key := stageCreateMarkerNamespaceKey([]string{"a", "b"})
if key == "a\x1fb" {
t.Fatalf("stageCreateMarkerNamespaceKey() returned unescaped namespace key %q", key)
}
if !strings.Contains(key, "%1F") {
t.Fatalf("stageCreateMarkerNamespaceKey() = %q, want escaped unit separator", key)
}
}
func TestStageCreateMarkerDir(t *testing.T) {
dir := stageCreateMarkerDir("warehouse", []string{"ns"}, "orders")
if !strings.Contains(dir, stageCreateMarkerDirName) {
t.Fatalf("stageCreateMarkerDir() = %q, want marker dir segment %q", dir, stageCreateMarkerDirName)
}
if !strings.HasSuffix(dir, "/orders") {
t.Fatalf("stageCreateMarkerDir() = %q, want suffix /orders", dir)
}
}
func TestStageCreateStagedTablePath(t *testing.T) {
tableUUID := uuid.MustParse("11111111-2222-3333-4444-555555555555")
stagedPath := stageCreateStagedTablePath([]string{"ns"}, "orders", tableUUID)
if !strings.Contains(stagedPath, stageCreateMarkerDirName) {
t.Fatalf("stageCreateStagedTablePath() = %q, want marker dir segment %q", stagedPath, stageCreateMarkerDirName)
}
if !strings.HasSuffix(stagedPath, "/"+tableUUID.String()) {
t.Fatalf("stageCreateStagedTablePath() = %q, want UUID suffix %q", stagedPath, tableUUID.String())
}
}

38
weed/s3api/object_lock_utils.go
View File

@@ -2,8 +2,6 @@ package s3api
import (
"context"
"encoding/xml"
"fmt"
"strconv"
"time"
@@ -35,21 +33,6 @@ func StoreVersioningInExtended(entry *filer_pb.Entry, enabled bool) error {
return nil
}
// LoadVersioningFromExtended loads versioning configuration from entry extended attributes
func LoadVersioningFromExtended(entry *filer_pb.Entry) (bool, bool) {
if entry == nil || entry.Extended == nil {
return false, false // not found, default to suspended
}
// Check for S3 API compatible key
if versioningBytes, exists := entry.Extended[s3_constants.ExtVersioningKey]; exists {
enabled := string(versioningBytes) == s3_constants.VersioningEnabled
return enabled, true
}
return false, false // not found
}
// GetVersioningStatus returns the versioning status as a string: "", "Enabled", or "Suspended"
// Empty string means versioning was never enabled
func GetVersioningStatus(entry *filer_pb.Entry) string {
@@ -90,15 +73,6 @@ func CreateObjectLockConfiguration(enabled bool, mode string, days int, years in
return config
}
// ObjectLockConfigurationToXML converts ObjectLockConfiguration to XML bytes
func ObjectLockConfigurationToXML(config *ObjectLockConfiguration) ([]byte, error) {
if config == nil {
return nil, fmt.Errorf("object lock configuration is nil")
}
return xml.Marshal(config)
}
// StoreObjectLockConfigurationInExtended stores Object Lock configuration in entry extended attributes
func StoreObjectLockConfigurationInExtended(entry *filer_pb.Entry, config *ObjectLockConfiguration) error {
if entry.Extended == nil {
@@ -379,18 +353,6 @@ func validateDefaultRetention(retention *DefaultRetention) error {
return nil
}
// ====================================================================
// SHARED OBJECT LOCK CHECKING FUNCTIONS
// ====================================================================
// These functions delegate to s3_objectlock package to avoid code duplication.
// They are kept here for backward compatibility with existing callers.
// EntryHasActiveLock checks if an entry has an active retention or legal hold
// Delegates to s3_objectlock.EntryHasActiveLock
func EntryHasActiveLock(entry *filer_pb.Entry, currentTime time.Time) bool {
return s3_objectlock.EntryHasActiveLock(entry, currentTime)
}
// HasObjectsWithActiveLocks checks if any objects in the bucket have active retention or legal hold
// Delegates to s3_objectlock.HasObjectsWithActiveLocks
func HasObjectsWithActiveLocks(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketPath string) (bool, error) {

321
weed/s3api/policy/post-policy.go
View File

@@ -1,321 +0,0 @@
package policy
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// expirationDateFormat date format for expiration key in json policy.
const expirationDateFormat = "2006-01-02T15:04:05.999Z"
// policyCondition explanation:
// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
//
// Example:
//
// policyCondition {
// matchType: "$eq",
// key: "$Content-Type",
// value: "image/png",
// }
type policyCondition struct {
matchType string
condition string
value string
}
// PostPolicy - Provides strict static type conversion and validation
// for Amazon S3's POST policy JSON string.
type PostPolicy struct {
// Expiration date and time of the POST policy.
expiration time.Time
// Collection of different policy conditions.
conditions []policyCondition
// ContentLengthRange minimum and maximum allowable size for the
// uploaded content.
contentLengthRange struct {
min int64
max int64
}
// Post form data.
formData map[string]string
}
// NewPostPolicy - Instantiate new post policy.
func NewPostPolicy() *PostPolicy {
p := &PostPolicy{}
p.conditions = make([]policyCondition, 0)
p.formData = make(map[string]string)
return p
}
// SetExpires - Sets expiration time for the new policy.
func (p *PostPolicy) SetExpires(t time.Time) error {
if t.IsZero() {
return errInvalidArgument("No expiry time set.")
}
p.expiration = t
return nil
}
// SetKey - Sets an object name for the policy based upload.
func (p *PostPolicy) SetKey(key string) error {
if strings.TrimSpace(key) == "" || key == "" {
return errInvalidArgument("Object name is empty.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$key",
value: key,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["key"] = key
return nil
}
// SetKeyStartsWith - Sets an object name that an policy based upload
// can start with.
func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error {
if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" {
return errInvalidArgument("Object prefix is empty.")
}
policyCond := policyCondition{
matchType: "starts-with",
condition: "$key",
value: keyStartsWith,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["key"] = keyStartsWith
return nil
}
// SetBucket - Sets bucket at which objects will be uploaded to.
func (p *PostPolicy) SetBucket(bucketName string) error {
if strings.TrimSpace(bucketName) == "" || bucketName == "" {
return errInvalidArgument("Bucket name is empty.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$bucket",
value: bucketName,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["bucket"] = bucketName
return nil
}
// SetCondition - Sets condition for credentials, date and algorithm
func (p *PostPolicy) SetCondition(matchType, condition, value string) error {
if strings.TrimSpace(value) == "" || value == "" {
return errInvalidArgument("No value specified for condition")
}
policyCond := policyCondition{
matchType: matchType,
condition: "$" + condition,
value: value,
}
if condition == "X-Amz-Credential" || condition == "X-Amz-Date" || condition == "X-Amz-Algorithm" {
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[condition] = value
return nil
}
return errInvalidArgument("Invalid condition in policy")
}
// SetContentType - Sets content-type of the object for this policy
// based upload.
func (p *PostPolicy) SetContentType(contentType string) error {
if strings.TrimSpace(contentType) == "" || contentType == "" {
return errInvalidArgument("No content type specified.")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$Content-Type",
value: contentType,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["Content-Type"] = contentType
return nil
}
// SetContentLengthRange - Set new min and max content length
// condition for all incoming uploads.
func (p *PostPolicy) SetContentLengthRange(min, max int64) error {
if min > max {
return errInvalidArgument("Minimum limit is larger than maximum limit.")
}
if min < 0 {
return errInvalidArgument("Minimum limit cannot be negative.")
}
if max < 0 {
return errInvalidArgument("Maximum limit cannot be negative.")
}
p.contentLengthRange.min = min
p.contentLengthRange.max = max
return nil
}
// SetSuccessActionRedirect - Sets the redirect success url of the object for this policy
// based upload.
func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error {
if strings.TrimSpace(redirect) == "" || redirect == "" {
return errInvalidArgument("Redirect is empty")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$success_action_redirect",
value: redirect,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["success_action_redirect"] = redirect
return nil
}
// SetSuccessStatusAction - Sets the status success code of the object for this policy
// based upload.
func (p *PostPolicy) SetSuccessStatusAction(status string) error {
if strings.TrimSpace(status) == "" || status == "" {
return errInvalidArgument("Status is empty")
}
policyCond := policyCondition{
matchType: "eq",
condition: "$success_action_status",
value: status,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData["success_action_status"] = status
return nil
}
// SetUserMetadata - Set user metadata as a key/value couple.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserMetadata(key string, value string) error {
if strings.TrimSpace(key) == "" || key == "" {
return errInvalidArgument("Key is empty")
}
if strings.TrimSpace(value) == "" || value == "" {
return errInvalidArgument("Value is empty")
}
headerName := fmt.Sprintf("x-amz-meta-%s", key)
policyCond := policyCondition{
matchType: "eq",
condition: fmt.Sprintf("$%s", headerName),
value: value,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[headerName] = value
return nil
}
// SetUserData - Set user data as a key/value couple.
// Can be retrieved through a HEAD request or an event.
func (p *PostPolicy) SetUserData(key string, value string) error {
if key == "" {
return errInvalidArgument("Key is empty")
}
if value == "" {
return errInvalidArgument("Value is empty")
}
headerName := fmt.Sprintf("x-amz-%s", key)
policyCond := policyCondition{
matchType: "eq",
condition: fmt.Sprintf("$%s", headerName),
value: value,
}
if err := p.addNewPolicy(policyCond); err != nil {
return err
}
p.formData[headerName] = value
return nil
}
// addNewPolicy - internal helper to validate adding new policies.
func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error {
if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" {
return errInvalidArgument("Policy fields are empty.")
}
p.conditions = append(p.conditions, policyCond)
return nil
}
// String function for printing policy in json formatted string.
func (p PostPolicy) String() string {
return string(p.marshalJSON())
}
// marshalJSON - Provides Marshaled JSON in bytes.
func (p PostPolicy) marshalJSON() []byte {
expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"`
var conditionsStr string
conditions := []string{}
for _, po := range p.conditions {
conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value))
}
if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 {
conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]",
p.contentLengthRange.min, p.contentLengthRange.max))
}
if len(conditions) > 0 {
conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]"
}
retStr := "{"
retStr = retStr + expirationStr + ","
retStr = retStr + conditionsStr
retStr = retStr + "}"
return []byte(retStr)
}
// base64 - Produces base64 of PostPolicy's Marshaled json.
func (p PostPolicy) base64() string {
return base64.StdEncoding.EncodeToString(p.marshalJSON())
}
// errInvalidArgument - Invalid argument response.
func errInvalidArgument(message string) error {
return s3err.RESTErrorResponse{
StatusCode: http.StatusBadRequest,
Code: "InvalidArgument",
Message: message,
RequestID: "client",
}
}

106
weed/s3api/policy/postpolicyform_test.go
View File

@@ -1,106 +0,0 @@
package policy
/*
* MinIO Cloud Storage, (C) 2016 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import (
"encoding/base64"
"fmt"
"net/http"
"testing"
"time"
)
// Test Post Policy parsing and checking conditions
func TestPostPolicyForm(t *testing.T) {
pp := NewPostPolicy()
pp.SetBucket("testbucket")
pp.SetContentType("image/jpeg")
pp.SetUserMetadata("uuid", "14365123651274")
pp.SetKeyStartsWith("user/user1/filename")
pp.SetContentLengthRange(1048579, 10485760)
pp.SetSuccessStatusAction("201")
type testCase struct {
Bucket string
Key string
XAmzDate string
XAmzAlgorithm string
XAmzCredential string
XAmzMetaUUID string
ContentType string
SuccessActionStatus string
Policy string
Expired bool
expectedErr error
}
testCases := []testCase{
// Everything is fine with this test
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil},
// Expired policy document
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")},
// Different AMZ date
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Key which doesn't start with user/user1/filename
{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect bucket name.
{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect key name
{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect date
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect ContentType
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
// Incorrect Metadata
{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")},
}
// Validate all the test cases.
for i, tt := range testCases {
formValues := make(http.Header)
formValues.Set("Bucket", tt.Bucket)
formValues.Set("Key", tt.Key)
formValues.Set("Content-Type", tt.ContentType)
formValues.Set("X-Amz-Date", tt.XAmzDate)
formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID)
formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm)
formValues.Set("X-Amz-Credential", tt.XAmzCredential)
if tt.Expired {
// Expired already.
pp.SetExpires(time.Now().UTC().AddDate(0, 0, -10))
} else {
// Expires in 10 days.
pp.SetExpires(time.Now().UTC().AddDate(0, 0, 10))
}
formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String())))
formValues.Set("Success_action_status", tt.SuccessActionStatus)
policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String())))
if err != nil {
t.Fatal(err)
}
postPolicyForm, err := ParsePostPolicyForm(string(policyBytes))
if err != nil {
t.Fatal(err)
}
err = CheckPostPolicy(formValues, postPolicyForm)
if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() {
t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error())
}
}
}

47
weed/s3api/policy_engine/conditions.go
View File

@@ -125,22 +125,6 @@ func (c *NormalizedValueCache) evictLeastRecentlyUsed() {
delete(c.cache, tail.key)
}
// Clear clears all cached values
func (c *NormalizedValueCache) Clear() {
c.mu.Lock()
defer c.mu.Unlock()
c.cache = make(map[string]*LRUNode)
c.head.next = c.tail
c.tail.prev = c.head
}
// GetStats returns cache statistics
func (c *NormalizedValueCache) GetStats() (size int, maxSize int) {
c.mu.RLock()
defer c.mu.RUnlock()
return len(c.cache), c.maxSize
}
// Global cache instance with size limit
var normalizedValueCache = NewNormalizedValueCache(1000)
@@ -769,34 +753,3 @@ func EvaluateConditions(conditions PolicyConditions, contextValues map[string][]
return true
}
// EvaluateConditionsLegacy evaluates conditions using the old interface{} format for backward compatibility
// objectEntry is the object's metadata from entry.Extended (can be nil)
func EvaluateConditionsLegacy(conditions map[string]interface{}, contextValues map[string][]string, objectEntry map[string][]byte) bool {
if len(conditions) == 0 {
return true // No conditions means always true
}
for operator, conditionMap := range conditions {
conditionEvaluator, err := GetConditionEvaluator(operator)
if err != nil {
glog.Warningf("Unsupported condition operator: %s", operator)
continue
}
conditionMapTyped, ok := conditionMap.(map[string]interface{})
if !ok {
glog.Warningf("Invalid condition format for operator: %s", operator)
continue
}
for key, value := range conditionMapTyped {
contextVals := getConditionContextValue(key, contextValues, objectEntry)
if !conditionEvaluator.Evaluate(value, contextVals) {
return false // If any condition fails, the whole condition block fails
}
}
}
return true
}

86
weed/s3api/policy_engine/engine.go
View File

@@ -610,92 +610,6 @@ func BuildActionName(action string) string {
return fmt.Sprintf("s3:%s", action)
}
// IsReadAction checks if an action is a read action
func IsReadAction(action string) bool {
readActions := []string{
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectAcl",
"s3:GetObjectVersionAcl",
"s3:GetObjectTagging",
"s3:GetObjectVersionTagging",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
"s3:GetBucketVersioning",
"s3:GetBucketAcl",
"s3:GetBucketCors",
"s3:GetBucketPolicy",
"s3:GetBucketTagging",
"s3:GetBucketNotification",
"s3:GetBucketObjectLockConfiguration",
"s3:GetObjectRetention",
"s3:GetObjectLegalHold",
}
for _, readAction := range readActions {
if action == readAction {
return true
}
}
return false
}
// IsWriteAction checks if an action is a write action
func IsWriteAction(action string) bool {
writeActions := []string{
"s3:PutObject",
"s3:PutObjectAcl",
"s3:PutObjectTagging",
"s3:DeleteObject",
"s3:DeleteObjectVersion",
"s3:DeleteObjectTagging",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
"s3:PutBucketAcl",
"s3:PutBucketCors",
"s3:PutBucketPolicy",
"s3:PutBucketTagging",
"s3:PutBucketNotification",
"s3:PutBucketVersioning",
"s3:DeleteBucketPolicy",
"s3:DeleteBucketTagging",
"s3:DeleteBucketCors",
"s3:PutBucketObjectLockConfiguration",
"s3:PutObjectRetention",
"s3:PutObjectLegalHold",
"s3:BypassGovernanceRetention",
}
for _, writeAction := range writeActions {
if action == writeAction {
return true
}
}
return false
}
// GetBucketNameFromArn extracts bucket name from ARN
func GetBucketNameFromArn(arn string) string {
if strings.HasPrefix(arn, "arn:aws:s3:::") {
parts := strings.SplitN(arn[13:], "/", 2)
return parts[0]
}
return ""
}
// GetObjectNameFromArn extracts object name from ARN
func GetObjectNameFromArn(arn string) string {
if strings.HasPrefix(arn, "arn:aws:s3:::") {
parts := strings.SplitN(arn[13:], "/", 2)
if len(parts) > 1 {
return parts[1]
}
}
return ""
}
// GetPolicyStatements returns all policy statements for a bucket
func (engine *PolicyEngine) GetPolicyStatements(bucketName string) []PolicyStatement {
engine.mutex.RLock()

77
weed/s3api/policy_engine/engine_test.go
View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
)
@@ -226,47 +225,6 @@ func TestConditionEvaluators(t *testing.T) {
}
}
func TestConvertIdentityToPolicy(t *testing.T) {
identityActions := []string{
"Read:bucket1/*",
"Write:bucket1/*",
"Admin:bucket2",
}
policy, err := ConvertIdentityToPolicy(identityActions)
if err != nil {
t.Fatalf("Failed to convert identity to policy: %v", err)
}
if policy.Version != "2012-10-17" {
t.Errorf("Expected version 2012-10-17, got %s", policy.Version)
}
if len(policy.Statement) != 3 {
t.Errorf("Expected 3 statements, got %d", len(policy.Statement))
}
// Check first statement (Read)
stmt := policy.Statement[0]
if stmt.Effect != PolicyEffectAllow {
t.Errorf("Expected Allow effect, got %s", stmt.Effect)
}
actions := normalizeToStringSlice(stmt.Action)
// Read action now includes: GetObject, GetObjectVersion, ListBucket, ListBucketVersions,
// GetObjectAcl, GetObjectVersionAcl, GetObjectTagging, GetObjectVersionTagging,
// GetBucketLocation, GetBucketVersioning, GetBucketAcl, GetBucketCors, GetBucketTagging, GetBucketNotification
if len(actions) != 14 {
t.Errorf("Expected 14 read actions, got %d: %v", len(actions), actions)
}
resources := normalizeToStringSlice(stmt.Resource)
// Read action now includes both bucket ARN (for ListBucket*) and object ARN (for GetObject*)
if len(resources) != 2 {
t.Errorf("Expected 2 resources (bucket and bucket/*), got %d: %v", len(resources), resources)
}
}
func TestPolicyValidation(t *testing.T) {
tests := []struct {
name string
@@ -794,41 +752,6 @@ func TestCompilePolicy(t *testing.T) {
}
}
// TestNewPolicyBackedIAMWithLegacy tests the constructor overload
func TestNewPolicyBackedIAMWithLegacy(t *testing.T) {
// Mock legacy IAM
mockLegacyIAM := &MockLegacyIAM{}
// Test the new constructor
policyBackedIAM := NewPolicyBackedIAMWithLegacy(mockLegacyIAM)
// Verify that the legacy IAM is set
if policyBackedIAM.legacyIAM != mockLegacyIAM {
t.Errorf("Expected legacy IAM to be set, but it wasn't")
}
// Verify that the policy engine is initialized
if policyBackedIAM.policyEngine == nil {
t.Errorf("Expected policy engine to be initialized, but it wasn't")
}
// Compare with the traditional approach
traditionalIAM := NewPolicyBackedIAM()
traditionalIAM.SetLegacyIAM(mockLegacyIAM)
// Both should behave the same
if policyBackedIAM.legacyIAM != traditionalIAM.legacyIAM {
t.Errorf("Expected both approaches to result in the same legacy IAM")
}
}
// MockLegacyIAM implements the LegacyIAM interface for testing
type MockLegacyIAM struct{}
func (m *MockLegacyIAM) authRequest(r *http.Request, action Action) (Identity, s3err.ErrorCode) {
return nil, s3err.ErrNone
}
// TestExistingObjectTagCondition tests s3:ExistingObjectTag/<tag-key> condition support
func TestExistingObjectTagCondition(t *testing.T) {
engine := NewPolicyEngine()

642
weed/s3api/policy_engine/integration.go
View File

@@ -1,642 +0,0 @@
package policy_engine
import (
"fmt"
"net/http"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// Action represents an S3 action - this should match the type in auth_credentials.go
type Action string
// Identity represents a user identity - this should match the type in auth_credentials.go
type Identity interface {
CanDo(action Action, bucket string, objectKey string) bool
}
// PolicyBackedIAM provides policy-based access control with fallback to legacy IAM
type PolicyBackedIAM struct {
policyEngine *PolicyEngine
legacyIAM LegacyIAM // Interface to delegate to existing IAM system
}
// LegacyIAM interface for delegating to existing IAM implementation
type LegacyIAM interface {
authRequest(r *http.Request, action Action) (Identity, s3err.ErrorCode)
}
// NewPolicyBackedIAM creates a new policy-backed IAM system
func NewPolicyBackedIAM() *PolicyBackedIAM {
return &PolicyBackedIAM{
policyEngine: NewPolicyEngine(),
legacyIAM: nil, // Will be set when integrated with existing IAM
}
}
// NewPolicyBackedIAMWithLegacy creates a new policy-backed IAM system with legacy IAM set
func NewPolicyBackedIAMWithLegacy(legacyIAM LegacyIAM) *PolicyBackedIAM {
return &PolicyBackedIAM{
policyEngine: NewPolicyEngine(),
legacyIAM: legacyIAM,
}
}
// SetLegacyIAM sets the legacy IAM system for fallback
func (p *PolicyBackedIAM) SetLegacyIAM(legacyIAM LegacyIAM) {
p.legacyIAM = legacyIAM
}
// SetBucketPolicy sets the policy for a bucket
func (p *PolicyBackedIAM) SetBucketPolicy(bucketName string, policyJSON string) error {
return p.policyEngine.SetBucketPolicy(bucketName, policyJSON)
}
// GetBucketPolicy gets the policy for a bucket
func (p *PolicyBackedIAM) GetBucketPolicy(bucketName string) (*PolicyDocument, error) {
return p.policyEngine.GetBucketPolicy(bucketName)
}
// DeleteBucketPolicy deletes the policy for a bucket
func (p *PolicyBackedIAM) DeleteBucketPolicy(bucketName string) error {
return p.policyEngine.DeleteBucketPolicy(bucketName)
}
// CanDo checks if a principal can perform an action on a resource
func (p *PolicyBackedIAM) CanDo(action, bucketName, objectName, principal string, r *http.Request) bool {
// If there's a bucket policy, evaluate it
if p.policyEngine.HasPolicyForBucket(bucketName) {
result := p.policyEngine.EvaluatePolicyForRequest(bucketName, objectName, action, principal, r)
switch result {
case PolicyResultAllow:
return true
case PolicyResultDeny:
return false
case PolicyResultIndeterminate:
// Fall through to legacy system
}
}
// No bucket policy or indeterminate result, use legacy conversion
return p.evaluateLegacyAction(action, bucketName, objectName, principal)
}
// evaluateLegacyAction evaluates actions using legacy identity-based rules
func (p *PolicyBackedIAM) evaluateLegacyAction(action, bucketName, objectName, principal string) bool {
// If we have a legacy IAM system to delegate to, use it
if p.legacyIAM != nil {
// Create a dummy request for legacy evaluation
// In real implementation, this would use the actual request
r := &http.Request{
Header: make(http.Header),
}
// Convert the action string to Action type
legacyAction := Action(action)
// Use legacy IAM to check permission
identity, errCode := p.legacyIAM.authRequest(r, legacyAction)
if errCode != s3err.ErrNone {
return false
}
// If we have an identity, check if it can perform the action
if identity != nil {
return identity.CanDo(legacyAction, bucketName, objectName)
}
}
// No legacy IAM available, convert to policy and evaluate
return p.evaluateUsingPolicyConversion(action, bucketName, objectName, principal)
}
// evaluateUsingPolicyConversion converts legacy action to policy and evaluates
func (p *PolicyBackedIAM) evaluateUsingPolicyConversion(action, bucketName, objectName, principal string) bool {
// For now, use a conservative approach for legacy actions
// In a real implementation, this would integrate with the existing identity system
glog.V(2).Infof("Legacy action evaluation for %s on %s/%s by %s", action, bucketName, objectName, principal)
// Return false to maintain security until proper legacy integration is implemented
// This ensures no unintended access is granted
return false
}
// extractBucketAndPrefix extracts bucket name and prefix from a resource pattern.
// Examples:
//
// "bucket" -> bucket="bucket", prefix=""
// "bucket/*" -> bucket="bucket", prefix=""
// "bucket/prefix/*" -> bucket="bucket", prefix="prefix"
// "bucket/a/b/c/*" -> bucket="bucket", prefix="a/b/c"
func extractBucketAndPrefix(pattern string) (string, string) {
// Validate input
pattern = strings.TrimSpace(pattern)
if pattern == "" || pattern == "/" {
return "", ""
}
// Remove trailing /* if present
pattern = strings.TrimSuffix(pattern, "/*")
// Remove a single trailing slash to avoid empty path segments
if strings.HasSuffix(pattern, "/") {
pattern = pattern[:len(pattern)-1]
}
if pattern == "" {
return "", ""
}
// Split on the first /
parts := strings.SplitN(pattern, "/", 2)
bucket := strings.TrimSpace(parts[0])
if bucket == "" {
return "", ""
}
if len(parts) == 1 {
// No slash, entire pattern is bucket
return bucket, ""
}
// Has slash, first part is bucket, rest is prefix
prefix := strings.Trim(parts[1], "/")
return bucket, prefix
}
// buildObjectResourceArn generates ARNs for object-level access.
// It properly handles both bucket-level (all objects) and prefix-level access.
// Returns empty slice if bucket is invalid to prevent generating malformed ARNs.
func buildObjectResourceArn(resourcePattern string) []string {
bucket, prefix := extractBucketAndPrefix(resourcePattern)
// If bucket is empty, the pattern is invalid; avoid generating malformed ARNs
if bucket == "" {
return []string{}
}
if prefix != "" {
// Prefix-based access: restrict to objects under this prefix
return []string{fmt.Sprintf("arn:aws:s3:::%s/%s/*", bucket, prefix)}
}
// Bucket-level access: all objects in bucket
return []string{fmt.Sprintf("arn:aws:s3:::%s/*", bucket)}
}
// ConvertIdentityToPolicy converts a legacy identity action to an AWS policy
func ConvertIdentityToPolicy(identityActions []string) (*PolicyDocument, error) {
statements := make([]PolicyStatement, 0)
for _, action := range identityActions {
stmt, err := convertSingleAction(action)
if err != nil {
glog.Warningf("Failed to convert action %s: %v", action, err)
continue
}
if stmt != nil {
statements = append(statements, *stmt)
}
}
if len(statements) == 0 {
return nil, fmt.Errorf("no valid statements generated")
}
return &PolicyDocument{
Version: PolicyVersion2012_10_17,
Statement: statements,
}, nil
}
// convertSingleAction converts a single legacy action to a policy statement.
// action format: "ActionType:ResourcePattern" (e.g., "Write:bucket/prefix/*")
func convertSingleAction(action string) (*PolicyStatement, error) {
parts := strings.Split(action, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid action format: %s", action)
}
actionType := parts[0]
resourcePattern := parts[1]
var s3Actions []string
var resources []string
switch actionType {
case "Read":
// Read includes both object-level (GetObject, GetObjectAcl, GetObjectTagging, GetObjectVersions)
// and bucket-level operations (ListBucket, GetBucketLocation, GetBucketVersioning, GetBucketCors, etc.)
s3Actions = []string{
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectAcl",
"s3:GetObjectVersionAcl",
"s3:GetObjectTagging",
"s3:GetObjectVersionTagging",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
"s3:GetBucketVersioning",
"s3:GetBucketAcl",
"s3:GetBucketCors",
"s3:GetBucketTagging",
"s3:GetBucketNotification",
}
bucket, _ := extractBucketAndPrefix(resourcePattern)
objectResources := buildObjectResourceArn(resourcePattern)
// Include both bucket ARN (for ListBucket* and Get*Bucket operations) and object ARNs (for GetObject* operations)
if bucket != "" {
resources = append([]string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}, objectResources...)
} else {
resources = objectResources
}
case "Write":
// Write includes object-level writes (PutObject, DeleteObject, PutObjectAcl, DeleteObjectVersion, DeleteObjectTagging, PutObjectTagging)
// and bucket-level writes (PutBucketVersioning, PutBucketCors, DeleteBucketCors, PutBucketAcl, PutBucketTagging, DeleteBucketTagging, PutBucketNotification)
// and multipart upload operations (AbortMultipartUpload, ListMultipartUploads, ListParts).
// ListMultipartUploads and ListParts are included because they are part of the multipart upload workflow
// and require Write permissions to be meaningful (no point listing uploads if you can't abort/complete them).
s3Actions = []string{
"s3:PutObject",
"s3:PutObjectAcl",
"s3:PutObjectTagging",
"s3:DeleteObject",
"s3:DeleteObjectVersion",
"s3:DeleteObjectTagging",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
"s3:PutBucketAcl",
"s3:PutBucketCors",
"s3:PutBucketTagging",
"s3:PutBucketNotification",
"s3:PutBucketVersioning",
"s3:DeleteBucketTagging",
"s3:DeleteBucketCors",
}
bucket, _ := extractBucketAndPrefix(resourcePattern)
objectResources := buildObjectResourceArn(resourcePattern)
// Include bucket ARN so bucket-level write operations (e.g., PutBucketVersioning, PutBucketCors)
// have the correct resource, while still allowing object-level writes.
if bucket != "" {
resources = append([]string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}, objectResources...)
} else {
resources = objectResources
}
case "Admin":
s3Actions = []string{"s3:*"}
bucket, prefix := extractBucketAndPrefix(resourcePattern)
if bucket == "" {
// Invalid pattern, return error
return nil, fmt.Errorf("Admin action requires a valid bucket name")
}
if prefix != "" {
// Subpath admin access: restrict to objects under this prefix
resources = []string{
fmt.Sprintf("arn:aws:s3:::%s", bucket),
fmt.Sprintf("arn:aws:s3:::%s/%s/*", bucket, prefix),
}
} else {
// Bucket-level admin access: full bucket permissions
resources = []string{
fmt.Sprintf("arn:aws:s3:::%s", bucket),
fmt.Sprintf("arn:aws:s3:::%s/*", bucket),
}
}
case "List":
// List includes bucket listing operations and also ListAllMyBuckets
s3Actions = []string{"s3:ListBucket", "s3:ListBucketVersions", "s3:ListAllMyBuckets"}
// ListBucket actions only require bucket ARN, not object-level ARNs
bucket, _ := extractBucketAndPrefix(resourcePattern)
if bucket != "" {
resources = []string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}
} else {
// Invalid pattern, return empty resources to fail validation
resources = []string{}
}
case "Tagging":
// Tagging includes both object-level and bucket-level tagging operations
s3Actions = []string{
"s3:GetObjectTagging",
"s3:PutObjectTagging",
"s3:DeleteObjectTagging",
"s3:GetBucketTagging",
"s3:PutBucketTagging",
"s3:DeleteBucketTagging",
}
bucket, _ := extractBucketAndPrefix(resourcePattern)
objectResources := buildObjectResourceArn(resourcePattern)
// Include bucket ARN so bucket-level tagging operations have the correct resource
if bucket != "" {
resources = append([]string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}, objectResources...)
} else {
resources = objectResources
}
case "BypassGovernanceRetention":
s3Actions = []string{"s3:BypassGovernanceRetention"}
resources = buildObjectResourceArn(resourcePattern)
case "GetObjectRetention":
s3Actions = []string{"s3:GetObjectRetention"}
resources = buildObjectResourceArn(resourcePattern)
case "PutObjectRetention":
s3Actions = []string{"s3:PutObjectRetention"}
resources = buildObjectResourceArn(resourcePattern)
case "GetObjectLegalHold":
s3Actions = []string{"s3:GetObjectLegalHold"}
resources = buildObjectResourceArn(resourcePattern)
case "PutObjectLegalHold":
s3Actions = []string{"s3:PutObjectLegalHold"}
resources = buildObjectResourceArn(resourcePattern)
case "GetBucketObjectLockConfiguration":
s3Actions = []string{"s3:GetBucketObjectLockConfiguration"}
bucket, _ := extractBucketAndPrefix(resourcePattern)
if bucket != "" {
resources = []string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}
} else {
// Invalid pattern, return empty resources to fail validation
resources = []string{}
}
case "PutBucketObjectLockConfiguration":
s3Actions = []string{"s3:PutBucketObjectLockConfiguration"}
bucket, _ := extractBucketAndPrefix(resourcePattern)
if bucket != "" {
resources = []string{fmt.Sprintf("arn:aws:s3:::%s", bucket)}
} else {
// Invalid pattern, return empty resources to fail validation
resources = []string{}
}
default:
return nil, fmt.Errorf("unknown action type: %s", actionType)
}
return &PolicyStatement{
Effect: PolicyEffectAllow,
Action: NewStringOrStringSlice(s3Actions...),
Resource: NewStringOrStringSlicePtr(resources...),
}, nil
}
// GetActionMappings returns the mapping of legacy actions to S3 actions
func GetActionMappings() map[string][]string {
return map[string][]string{
"Read": {
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectAcl",
"s3:GetObjectVersionAcl",
"s3:GetObjectTagging",
"s3:GetObjectVersionTagging",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
"s3:GetBucketVersioning",
"s3:GetBucketAcl",
"s3:GetBucketCors",
"s3:GetBucketTagging",
"s3:GetBucketNotification",
},
"Write": {
"s3:PutObject",
"s3:PutObjectAcl",
"s3:PutObjectTagging",
"s3:DeleteObject",
"s3:DeleteObjectVersion",
"s3:DeleteObjectTagging",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
"s3:PutBucketAcl",
"s3:PutBucketCors",
"s3:PutBucketTagging",
"s3:PutBucketNotification",
"s3:PutBucketVersioning",
"s3:DeleteBucketTagging",
"s3:DeleteBucketCors",
},
"Admin": {
"s3:*",
},
"List": {
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:ListAllMyBuckets",
},
"Tagging": {
"s3:GetObjectTagging",
"s3:PutObjectTagging",
"s3:DeleteObjectTagging",
"s3:GetBucketTagging",
"s3:PutBucketTagging",
"s3:DeleteBucketTagging",
},
"BypassGovernanceRetention": {
"s3:BypassGovernanceRetention",
},
"GetObjectRetention": {
"s3:GetObjectRetention",
},
"PutObjectRetention": {
"s3:PutObjectRetention",
},
"GetObjectLegalHold": {
"s3:GetObjectLegalHold",
},
"PutObjectLegalHold": {
"s3:PutObjectLegalHold",
},
"GetBucketObjectLockConfiguration": {
"s3:GetBucketObjectLockConfiguration",
},
"PutBucketObjectLockConfiguration": {
"s3:PutBucketObjectLockConfiguration",
},
}
}
// ValidateActionMapping validates that a legacy action can be mapped to S3 actions
func ValidateActionMapping(action string) error {
mappings := GetActionMappings()
parts := strings.Split(action, ":")
if len(parts) != 2 {
return fmt.Errorf("invalid action format: %s, expected format: 'ActionType:Resource'", action)
}
actionType := parts[0]
resource := parts[1]
if _, exists := mappings[actionType]; !exists {
return fmt.Errorf("unknown action type: %s", actionType)
}
if resource == "" {
return fmt.Errorf("resource cannot be empty")
}
return nil
}
// ConvertLegacyActions converts an array of legacy actions to S3 actions
func ConvertLegacyActions(legacyActions []string) ([]string, error) {
mappings := GetActionMappings()
s3Actions := make([]string, 0)
for _, legacyAction := range legacyActions {
if err := ValidateActionMapping(legacyAction); err != nil {
return nil, err
}
parts := strings.Split(legacyAction, ":")
actionType := parts[0]
if actionType == "Admin" {
// Admin gives all permissions, so we can just return s3:*
return []string{"s3:*"}, nil
}
if mapped, exists := mappings[actionType]; exists {
s3Actions = append(s3Actions, mapped...)
}
}
// Remove duplicates
uniqueActions := make([]string, 0)
seen := make(map[string]bool)
for _, action := range s3Actions {
if !seen[action] {
uniqueActions = append(uniqueActions, action)
seen[action] = true
}
}
return uniqueActions, nil
}
// GetResourcesFromLegacyAction extracts resources from a legacy action.
// It delegates to convertSingleAction to ensure consistent resource ARN generation
// across the codebase and avoid duplicating action-type-specific logic.
func GetResourcesFromLegacyAction(legacyAction string) ([]string, error) {
stmt, err := convertSingleAction(legacyAction)
if err != nil {
return nil, err
}
return stmt.Resource.Strings(), nil
}
// CreatePolicyFromLegacyIdentity creates a policy document from legacy identity actions
func CreatePolicyFromLegacyIdentity(identityName string, actions []string) (*PolicyDocument, error) {
statements := make([]PolicyStatement, 0)
// Group actions by resource pattern
resourceActions := make(map[string][]string)
for _, action := range actions {
// Validate action format before processing
if err := ValidateActionMapping(action); err != nil {
glog.Warningf("Skipping invalid action %q for identity %q: %v", action, identityName, err)
continue
}
parts := strings.Split(action, ":")
if len(parts) != 2 {
continue
}
resourcePattern := parts[1]
actionType := parts[0]
if _, exists := resourceActions[resourcePattern]; !exists {
resourceActions[resourcePattern] = make([]string, 0)
}
resourceActions[resourcePattern] = append(resourceActions[resourcePattern], actionType)
}
// Create statements for each resource pattern
for resourcePattern, actionTypes := range resourceActions {
s3Actions := make([]string, 0)
resourceSet := make(map[string]struct{})
// Collect S3 actions and aggregate resource ARNs from all action types.
// Different action types have different resource ARN requirements:
// - List: bucket-level ARNs only
// - Read/Write/Tagging: object-level ARNs
// - Admin: full bucket access
// We must merge all required ARNs for the combined policy statement.
for _, actionType := range actionTypes {
if actionType == "Admin" {
s3Actions = []string{"s3:*"}
// Admin action determines the resources, so we can break after processing it.
res, err := GetResourcesFromLegacyAction(fmt.Sprintf("Admin:%s", resourcePattern))
if err != nil {
glog.Warningf("Failed to get resources for Admin action on %s: %v", resourcePattern, err)
resourceSet = nil // Invalidate to skip this statement
break
}
for _, r := range res {
resourceSet[r] = struct{}{}
}
break
}
if mapped, exists := GetActionMappings()[actionType]; exists {
s3Actions = append(s3Actions, mapped...)
res, err := GetResourcesFromLegacyAction(fmt.Sprintf("%s:%s", actionType, resourcePattern))
if err != nil {
glog.Warningf("Failed to get resources for %s action on %s: %v", actionType, resourcePattern, err)
resourceSet = nil // Invalidate to skip this statement
break
}
for _, r := range res {
resourceSet[r] = struct{}{}
}
}
}
if resourceSet == nil || len(s3Actions) == 0 {
continue
}
resources := make([]string, 0, len(resourceSet))
for r := range resourceSet {
resources = append(resources, r)
}
statement := PolicyStatement{
Sid: fmt.Sprintf("%s-%s", identityName, strings.ReplaceAll(resourcePattern, "/", "-")),
Effect: PolicyEffectAllow,
Action: NewStringOrStringSlice(s3Actions...),
Resource: NewStringOrStringSlicePtr(resources...),
}
statements = append(statements, statement)
}
if len(statements) == 0 {
return nil, fmt.Errorf("no valid statements generated for identity %s", identityName)
}
return &PolicyDocument{
Version: PolicyVersion2012_10_17,
Statement: statements,
}, nil
}
// HasPolicyForBucket checks if a bucket has a policy
func (p *PolicyBackedIAM) HasPolicyForBucket(bucketName string) bool {
return p.policyEngine.HasPolicyForBucket(bucketName)
}
// GetPolicyEngine returns the underlying policy engine
func (p *PolicyBackedIAM) GetPolicyEngine() *PolicyEngine {
return p.policyEngine
}

373
weed/s3api/policy_engine/integration_test.go
View File

@@ -1,373 +0,0 @@
package policy_engine
import (
"testing"
"github.com/stretchr/testify/assert"
)
// TestConvertSingleActionDeleteObject tests support for s3:DeleteObject action (Issue #7864)
func TestConvertSingleActionDeleteObject(t *testing.T) {
// Test that Write action includes DeleteObject S3 action
stmt, err := convertSingleAction("Write:bucket")
assert.NoError(t, err)
assert.NotNil(t, stmt)
// Check that s3:DeleteObject is included in the actions
actions := stmt.Action.Strings()
assert.Contains(t, actions, "s3:DeleteObject", "Write action should include s3:DeleteObject")
assert.Contains(t, actions, "s3:PutObject", "Write action should include s3:PutObject")
}
// TestConvertSingleActionSubpath tests subpath handling for legacy actions (Issue #7864)
func TestConvertSingleActionSubpath(t *testing.T) {
testCases := []struct {
name string
action string
expectedActions []string
expectedResources []string
description string
}{
{
name: "Write_on_bucket",
action: "Write:mybucket",
expectedActions: []string{"s3:PutObject", "s3:DeleteObject", "s3:PutObjectAcl", "s3:DeleteObjectVersion", "s3:PutObjectTagging", "s3:DeleteObjectTagging", "s3:AbortMultipartUpload", "s3:ListMultipartUploads", "s3:ListParts", "s3:PutBucketAcl", "s3:PutBucketCors", "s3:PutBucketTagging", "s3:PutBucketNotification", "s3:PutBucketVersioning", "s3:DeleteBucketTagging", "s3:DeleteBucketCors"},
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"},
description: "Write permission on bucket should include bucket and object ARNs",
},
{
name: "Write_on_bucket_with_wildcard",
action: "Write:mybucket/*",
expectedActions: []string{"s3:PutObject", "s3:DeleteObject", "s3:PutObjectAcl", "s3:DeleteObjectVersion", "s3:PutObjectTagging", "s3:DeleteObjectTagging", "s3:AbortMultipartUpload", "s3:ListMultipartUploads", "s3:ListParts", "s3:PutBucketAcl", "s3:PutBucketCors", "s3:PutBucketTagging", "s3:PutBucketNotification", "s3:PutBucketVersioning", "s3:DeleteBucketTagging", "s3:DeleteBucketCors"},
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"},
description: "Write permission with /* should include bucket and object ARNs",
},
{
name: "Write_on_subpath",
action: "Write:mybucket/sub_path/*",
expectedActions: []string{"s3:PutObject", "s3:DeleteObject", "s3:PutObjectAcl", "s3:DeleteObjectVersion", "s3:PutObjectTagging", "s3:DeleteObjectTagging", "s3:AbortMultipartUpload", "s3:ListMultipartUploads", "s3:ListParts", "s3:PutBucketAcl", "s3:PutBucketCors", "s3:PutBucketTagging", "s3:PutBucketNotification", "s3:PutBucketVersioning", "s3:DeleteBucketTagging", "s3:DeleteBucketCors"},
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/sub_path/*"},
description: "Write permission on subpath should include bucket and subpath objects ARNs",
},
{
name: "Read_on_subpath",
action: "Read:mybucket/documents/*",
expectedActions: []string{"s3:GetObject", "s3:GetObjectVersion", "s3:ListBucket", "s3:ListBucketVersions", "s3:GetObjectAcl", "s3:GetObjectVersionAcl", "s3:GetObjectTagging", "s3:GetObjectVersionTagging", "s3:GetBucketLocation", "s3:GetBucketVersioning", "s3:GetBucketAcl", "s3:GetBucketCors", "s3:GetBucketTagging", "s3:GetBucketNotification"},
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/documents/*"},
description: "Read permission on subpath should include bucket ARN and subpath objects",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
stmt, err := convertSingleAction(tc.action)
assert.NoError(t, err, tc.description)
assert.NotNil(t, stmt)
// Check actions
actions := stmt.Action.Strings()
for _, expectedAction := range tc.expectedActions {
assert.Contains(t, actions, expectedAction,
"Action %s should be included for %s", expectedAction, tc.action)
}
// Check resources - verify all expected resources are present
resources := stmt.Resource.Strings()
assert.ElementsMatch(t, resources, tc.expectedResources,
"Resources should match exactly for %s. Got %v, expected %v", tc.action, resources, tc.expectedResources)
})
}
}
// TestConvertSingleActionSubpathDeleteAllowed tests that DeleteObject works on subpaths
func TestConvertSingleActionSubpathDeleteAllowed(t *testing.T) {
// This test specifically addresses Issue #7864 part 1:
// "when a user is granted permission to a subpath, eg s3.configure -user someuser
// -actions Write -buckets some_bucket/sub_path/* -apply
// the user will only be able to put, but not delete object under somebucket/sub_path"
stmt, err := convertSingleAction("Write:some_bucket/sub_path/*")
assert.NoError(t, err)
// The fix: s3:DeleteObject should be in the allowed actions
actions := stmt.Action.Strings()
assert.Contains(t, actions, "s3:DeleteObject",
"Write permission on subpath should allow deletion of objects in that path")
// The resource should be restricted to the subpath
resources := stmt.Resource.Strings()
assert.Contains(t, resources, "arn:aws:s3:::some_bucket/sub_path/*",
"Delete permission should apply to objects under the subpath")
}
// TestConvertSingleActionNestedPaths tests deeply nested paths
func TestConvertSingleActionNestedPaths(t *testing.T) {
testCases := []struct {
action string
expectedResources []string
}{
{
action: "Write:bucket/a/b/c/*",
expectedResources: []string{"arn:aws:s3:::bucket", "arn:aws:s3:::bucket/a/b/c/*"},
},
{
action: "Read:bucket/data/documents/2024/*",
expectedResources: []string{"arn:aws:s3:::bucket", "arn:aws:s3:::bucket/data/documents/2024/*"},
},
}
for _, tc := range testCases {
stmt, err := convertSingleAction(tc.action)
assert.NoError(t, err)
resources := stmt.Resource.Strings()
assert.ElementsMatch(t, resources, tc.expectedResources)
}
}
// TestGetResourcesFromLegacyAction tests that GetResourcesFromLegacyAction generates
// action-appropriate resources consistent with convertSingleAction
func TestGetResourcesFromLegacyAction(t *testing.T) {
testCases := []struct {
name string
action string
expectedResources []string
description string
}{
// List actions - bucket-only (no object ARNs)
{
name: "List_on_bucket",
action: "List:mybucket",
expectedResources: []string{"arn:aws:s3:::mybucket"},
description: "List action should only have bucket ARN",
},
{
name: "List_on_bucket_with_wildcard",
action: "List:mybucket/*",
expectedResources: []string{"arn:aws:s3:::mybucket"},
description: "List action should only have bucket ARN regardless of wildcard",
},
// Read actions - bucket and object-level ARNs (includes List* and Get* operations)
{
name: "Read_on_bucket",
action: "Read:mybucket",
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"},
description: "Read action should have both bucket and object ARNs",
},
{
name: "Read_on_subpath",
action: "Read:mybucket/documents/*",
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/documents/*"},
description: "Read action on subpath should have bucket ARN and object ARN for subpath",
},
// Write actions - bucket and object ARNs (includes bucket-level operations)
{
name: "Write_on_subpath",
action: "Write:mybucket/sub_path/*",
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/sub_path/*"},
description: "Write action should have bucket and object ARNs",
},
// Admin actions - both bucket and object ARNs
{
name: "Admin_on_bucket",
action: "Admin:mybucket",
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"},
description: "Admin action should have both bucket and object ARNs",
},
{
name: "Admin_on_subpath",
action: "Admin:mybucket/admin/section/*",
expectedResources: []string{"arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/admin/section/*"},
description: "Admin action on subpath should restrict to subpath, preventing privilege escalation",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
resources, err := GetResourcesFromLegacyAction(tc.action)
assert.NoError(t, err, tc.description)
assert.ElementsMatch(t, resources, tc.expectedResources,
"Resources should match expected. Got %v, expected %v", resources, tc.expectedResources)
// Also verify consistency with convertSingleAction where applicable
stmt, err := convertSingleAction(tc.action)
assert.NoError(t, err)
stmtResources := stmt.Resource.Strings()
assert.ElementsMatch(t, resources, stmtResources,
"GetResourcesFromLegacyAction should match convertSingleAction resources for %s", tc.action)
})
}
}
// TestExtractBucketAndPrefixEdgeCases validates edge case handling in extractBucketAndPrefix
func TestExtractBucketAndPrefixEdgeCases(t *testing.T) {
testCases := []struct {
name string
pattern string
expectedBucket string
expectedPrefix string
description string
}{
{
name: "Empty string",
pattern: "",
expectedBucket: "",
expectedPrefix: "",
description: "Empty pattern should return empty strings",
},
{
name: "Whitespace only",
pattern: " ",
expectedBucket: "",
expectedPrefix: "",
description: "Whitespace-only pattern should return empty strings",
},
{
name: "Slash only",
pattern: "/",
expectedBucket: "",
expectedPrefix: "",
description: "Slash-only pattern should return empty strings",
},
{
name: "Double slash prefix",
pattern: "bucket//prefix/*",
expectedBucket: "bucket",
expectedPrefix: "prefix",
description: "Double slash should be normalized (trailing slashes removed)",
},
{
name: "Normal bucket",
pattern: "mybucket",
expectedBucket: "mybucket",
expectedPrefix: "",
description: "Bucket-only pattern should work correctly",
},
{
name: "Bucket with prefix",
pattern: "mybucket/myprefix/*",
expectedBucket: "mybucket",
expectedPrefix: "myprefix",
description: "Bucket with prefix should be parsed correctly",
},
{
name: "Nested prefix",
pattern: "mybucket/a/b/c/*",
expectedBucket: "mybucket",
expectedPrefix: "a/b/c",
description: "Nested prefix should be preserved",
},
{
name: "Bucket with trailing slash",
pattern: "mybucket/",
expectedBucket: "mybucket",
expectedPrefix: "",
description: "Trailing slash on bucket should be normalized",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
bucket, prefix := extractBucketAndPrefix(tc.pattern)
assert.Equal(t, tc.expectedBucket, bucket, tc.description)
assert.Equal(t, tc.expectedPrefix, prefix, tc.description)
})
}
}
// TestCreatePolicyFromLegacyIdentityMultipleActions validates correct resource ARN aggregation
// when multiple action types target the same resource pattern
func TestCreatePolicyFromLegacyIdentityMultipleActions(t *testing.T) {
testCases := []struct {
name string
identityName string
actions []string
expectedStatements int
expectedActionsInStmt1 []string
expectedResourcesInStmt1 []string
description string
}{
{
name: "List_and_Write_on_subpath",
identityName: "data-manager",
actions: []string{"List:mybucket/data/*", "Write:mybucket/data/*"},
expectedStatements: 1,
expectedActionsInStmt1: []string{
"s3:ListBucket", "s3:ListBucketVersions", "s3:ListAllMyBuckets",
"s3:PutObject", "s3:DeleteObject", "s3:PutObjectAcl", "s3:DeleteObjectVersion",
"s3:PutObjectTagging", "s3:DeleteObjectTagging", "s3:AbortMultipartUpload",
"s3:ListMultipartUploads", "s3:ListParts", "s3:PutBucketAcl", "s3:PutBucketCors",
"s3:PutBucketTagging", "s3:PutBucketNotification", "s3:PutBucketVersioning",
"s3:DeleteBucketTagging", "s3:DeleteBucketCors",
},
expectedResourcesInStmt1: []string{
"arn:aws:s3:::mybucket", // From List and Write actions
"arn:aws:s3:::mybucket/data/*", // From Write action
},
description: "List + Write on same subpath should aggregate all actions and both bucket and object ARNs",
},
{
name: "Read_and_Tagging_on_bucket",
identityName: "tag-reader",
actions: []string{"Read:mybucket", "Tagging:mybucket"},
expectedStatements: 1,
expectedActionsInStmt1: []string{
"s3:GetObject", "s3:GetObjectVersion",
"s3:ListBucket", "s3:ListBucketVersions",
"s3:GetObjectAcl", "s3:GetObjectVersionAcl",
"s3:GetObjectTagging", "s3:GetObjectVersionTagging",
"s3:PutObjectTagging", "s3:DeleteObjectTagging",
"s3:GetBucketLocation", "s3:GetBucketVersioning",
"s3:GetBucketAcl", "s3:GetBucketCors", "s3:GetBucketTagging",
"s3:GetBucketNotification", "s3:PutBucketTagging", "s3:DeleteBucketTagging",
},
expectedResourcesInStmt1: []string{
"arn:aws:s3:::mybucket",
"arn:aws:s3:::mybucket/*",
},
description: "Read + Tagging on same bucket should aggregate all bucket and object-level actions and ARNs",
},
{
name: "Admin_with_other_actions",
identityName: "admin-user",
actions: []string{"Admin:mybucket/admin/*", "Write:mybucket/admin/*"},
expectedStatements: 1,
expectedActionsInStmt1: []string{"s3:*"},
expectedResourcesInStmt1: []string{
"arn:aws:s3:::mybucket",
"arn:aws:s3:::mybucket/admin/*",
},
description: "Admin action should dominate and set s3:*, other actions still processed for resources",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
policy, err := CreatePolicyFromLegacyIdentity(tc.identityName, tc.actions)
assert.NoError(t, err, tc.description)
assert.NotNil(t, policy)
// Check statement count
assert.Equal(t, tc.expectedStatements, len(policy.Statement),
"Expected %d statement(s), got %d", tc.expectedStatements, len(policy.Statement))
if tc.expectedStatements > 0 {
stmt := policy.Statement[0]
// Check actions
actualActions := stmt.Action.Strings()
for _, expectedAction := range tc.expectedActionsInStmt1 {
assert.Contains(t, actualActions, expectedAction,
"Action %s should be included in statement", expectedAction)
}
// Check resources - all expected resources should be present
actualResources := stmt.Resource.Strings()
assert.ElementsMatch(t, tc.expectedResourcesInStmt1, actualResources,
"Statement should aggregate all required resource ARNs. Got %v, expected %v",
actualResources, tc.expectedResourcesInStmt1)
}
})
}
}

5
weed/s3api/policy_engine/types.go
View File

@@ -490,11 +490,6 @@ func GetBucketFromResource(resource string) string {
return ""
}
// IsObjectResource checks if resource refers to objects
func IsObjectResource(resource string) bool {
return strings.Contains(resource, "/")
}
// MatchesAction checks if an action matches any of the compiled action matchers.
// It also implicitly grants multipart upload actions if s3:PutObject is allowed,
// since multipart upload is an implementation detail of putting objects.

67
weed/s3api/s3_bucket_encryption.go
View File

@@ -288,70 +288,3 @@ func (s3a *S3ApiServer) GetDefaultEncryptionHeaders(bucket string) map[string]st
return headers
}
// IsDefaultEncryptionEnabled checks if default encryption is enabled for a configuration
func IsDefaultEncryptionEnabled(config *s3_pb.EncryptionConfiguration) bool {
return config != nil && config.SseAlgorithm != ""
}
// GetDefaultEncryptionHeaders generates default encryption headers from configuration
func GetDefaultEncryptionHeaders(config *s3_pb.EncryptionConfiguration) map[string]string {
if config == nil || config.SseAlgorithm == "" {
return nil
}
headers := make(map[string]string)
headers[s3_constants.AmzServerSideEncryption] = config.SseAlgorithm
if config.SseAlgorithm == "aws:kms" && config.KmsKeyId != "" {
headers[s3_constants.AmzServerSideEncryptionAwsKmsKeyId] = config.KmsKeyId
}
return headers
}
// encryptionConfigFromXMLBytes parses XML bytes to encryption configuration
func encryptionConfigFromXMLBytes(xmlBytes []byte) (*s3_pb.EncryptionConfiguration, error) {
var xmlConfig ServerSideEncryptionConfiguration
if err := xml.Unmarshal(xmlBytes, &xmlConfig); err != nil {
return nil, err
}
// Validate namespace - should be empty or the standard AWS namespace
if xmlConfig.XMLName.Space != "" && xmlConfig.XMLName.Space != "http://s3.amazonaws.com/doc/2006-03-01/" {
return nil, fmt.Errorf("invalid XML namespace: %s", xmlConfig.XMLName.Space)
}
// Validate the configuration
if len(xmlConfig.Rules) == 0 {
return nil, fmt.Errorf("encryption configuration must have at least one rule")
}
rule := xmlConfig.Rules[0]
if rule.ApplyServerSideEncryptionByDefault.SSEAlgorithm == "" {
return nil, fmt.Errorf("encryption algorithm is required")
}
// Validate algorithm
validAlgorithms := map[string]bool{
"AES256": true,
"aws:kms": true,
}
if !validAlgorithms[rule.ApplyServerSideEncryptionByDefault.SSEAlgorithm] {
return nil, fmt.Errorf("unsupported encryption algorithm: %s", rule.ApplyServerSideEncryptionByDefault.SSEAlgorithm)
}
config := encryptionConfigFromXML(&xmlConfig)
return config, nil
}
// encryptionConfigToXMLBytes converts encryption configuration to XML bytes
func encryptionConfigToXMLBytes(config *s3_pb.EncryptionConfiguration) ([]byte, error) {
if config == nil {
return nil, fmt.Errorf("encryption configuration is nil")
}
xmlConfig := encryptionConfigToXML(config)
return xml.Marshal(xmlConfig)
}

120
weed/s3api/s3_iam_middleware.go
View File

@@ -13,7 +13,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/iam/providers"
"github.com/seaweedfs/seaweedfs/weed/iam/sts"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
@@ -381,52 +380,6 @@ func buildS3ResourceArn(bucket string, objectKey string) string {
return "arn:aws:s3:::" + bucket + "/" + objectKey
}
// mapLegacyActionToIAM provides fallback mapping for legacy actions
// This ensures backward compatibility while the system transitions to granular actions
func mapLegacyActionToIAM(legacyAction Action) string {
switch legacyAction {
case s3_constants.ACTION_READ:
return "s3:GetObject" // Fallback for unmapped read operations
case s3_constants.ACTION_WRITE:
return "s3:PutObject" // Fallback for unmapped write operations
case s3_constants.ACTION_LIST:
return "s3:ListBucket" // Fallback for unmapped list operations
case s3_constants.ACTION_TAGGING:
return "s3:GetObjectTagging" // Fallback for unmapped tagging operations
case s3_constants.ACTION_READ_ACP:
return "s3:GetObjectAcl" // Fallback for unmapped ACL read operations
case s3_constants.ACTION_WRITE_ACP:
return "s3:PutObjectAcl" // Fallback for unmapped ACL write operations
case s3_constants.ACTION_DELETE_BUCKET:
return "s3:DeleteBucket" // Fallback for unmapped bucket delete operations
case s3_constants.ACTION_ADMIN:
return "s3:*" // Fallback for unmapped admin operations
// Handle granular multipart actions (already correctly mapped)
case s3_constants.S3_ACTION_CREATE_MULTIPART:
return s3_constants.S3_ACTION_CREATE_MULTIPART
case s3_constants.S3_ACTION_UPLOAD_PART:
return s3_constants.S3_ACTION_UPLOAD_PART
case s3_constants.S3_ACTION_COMPLETE_MULTIPART:
return s3_constants.S3_ACTION_COMPLETE_MULTIPART
case s3_constants.S3_ACTION_ABORT_MULTIPART:
return s3_constants.S3_ACTION_ABORT_MULTIPART
case s3_constants.S3_ACTION_LIST_MULTIPART_UPLOADS:
return s3_constants.S3_ACTION_LIST_MULTIPART_UPLOADS
case s3_constants.S3_ACTION_LIST_PARTS:
return s3_constants.S3_ACTION_LIST_PARTS
default:
// If it's already a properly formatted S3 action, return as-is
actionStr := string(legacyAction)
if strings.HasPrefix(actionStr, "s3:") {
return actionStr
}
// Fallback: convert to S3 action format
return "s3:" + actionStr
}
}
// extractRequestContext extracts request context for policy conditions
func extractRequestContext(r *http.Request) map[string]interface{} {
context := make(map[string]interface{})
@@ -553,79 +506,6 @@ type EnhancedS3ApiServer struct {
iamIntegration IAMIntegration
}
// NewEnhancedS3ApiServer creates an S3 API server with IAM integration
func NewEnhancedS3ApiServer(baseServer *S3ApiServer, iamManager *integration.IAMManager) *EnhancedS3ApiServer {
// Set the IAM integration on the base server
baseServer.SetIAMIntegration(iamManager)
return &EnhancedS3ApiServer{
S3ApiServer: baseServer,
iamIntegration: NewS3IAMIntegration(iamManager, "localhost:8888"),
}
}
// AuthenticateJWTRequest handles JWT authentication for S3 requests
func (enhanced *EnhancedS3ApiServer) AuthenticateJWTRequest(r *http.Request) (*Identity, s3err.ErrorCode) {
ctx := r.Context()
// Use our IAM integration for JWT authentication
iamIdentity, errCode := enhanced.iamIntegration.AuthenticateJWT(ctx, r)
if errCode != s3err.ErrNone {
return nil, errCode
}
// Convert IAMIdentity to the existing Identity structure
identity := &Identity{
Name: iamIdentity.Name,
Account: iamIdentity.Account,
// Note: Actions will be determined by policy evaluation
Actions: []Action{}, // Empty - authorization handled by policy engine
PolicyNames: iamIdentity.PolicyNames,
}
// Store session token for later authorization
r.Header.Set("X-SeaweedFS-Session-Token", iamIdentity.SessionToken)
r.Header.Set("X-SeaweedFS-Principal", iamIdentity.Principal)
return identity, s3err.ErrNone
}
// AuthorizeRequest handles authorization for S3 requests using policy engine
func (enhanced *EnhancedS3ApiServer) AuthorizeRequest(r *http.Request, identity *Identity, action Action) s3err.ErrorCode {
ctx := r.Context()
// Get session info from request headers (set during authentication)
sessionToken := r.Header.Get("X-SeaweedFS-Session-Token")
principal := r.Header.Get("X-SeaweedFS-Principal")
if sessionToken == "" || principal == "" {
glog.V(3).Info("No session information available for authorization")
return s3err.ErrAccessDenied
}
// Extract bucket and object from request
bucket, object := s3_constants.GetBucketAndObject(r)
prefix := s3_constants.GetPrefix(r)
// For List operations, use prefix for permission checking if available
if action == s3_constants.ACTION_LIST && object == "" && prefix != "" {
object = prefix
} else if (object == "/" || object == "") && prefix != "" {
object = prefix
}
// Create IAM identity for authorization
iamIdentity := &IAMIdentity{
Name: identity.Name,
Principal: principal,
SessionToken: sessionToken,
Account: identity.Account,
}
// Use our IAM integration for authorization
return enhanced.iamIntegration.AuthorizeAction(ctx, iamIdentity, action, bucket, object, r)
}
// OIDCIdentity represents an identity validated through OIDC
type OIDCIdentity struct {
UserID string

584
weed/s3api/s3_iam_simple_test.go
View File

@@ -1,584 +0,0 @@
package s3api
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/iam/policy"
"github.com/seaweedfs/seaweedfs/weed/iam/sts"
"github.com/seaweedfs/seaweedfs/weed/iam/utils"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newTestS3IAMManagerWithDefaultEffect(t *testing.T, defaultEffect string) *integration.IAMManager {
t.Helper()
iamManager := integration.NewIAMManager()
config := &integration.IAMConfig{
STS: &sts.STSConfig{
TokenDuration: sts.FlexibleDuration{Duration: time.Hour},
MaxSessionLength: sts.FlexibleDuration{Duration: time.Hour * 12},
Issuer: "test-sts",
SigningKey: []byte("test-signing-key-32-characters-long"),
},
Policy: &policy.PolicyEngineConfig{
DefaultEffect: defaultEffect,
StoreType: "memory",
},
Roles: &integration.RoleStoreConfig{
StoreType: "memory",
},
}
err := iamManager.Initialize(config, func() string {
return "localhost:8888"
})
require.NoError(t, err)
return iamManager
}
func newTestS3IAMManager(t *testing.T) *integration.IAMManager {
t.Helper()
return newTestS3IAMManagerWithDefaultEffect(t, "Deny")
}
// TestS3IAMMiddleware tests the basic S3 IAM middleware functionality
func TestS3IAMMiddleware(t *testing.T) {
iamManager := newTestS3IAMManager(t)
// Create S3 IAM integration
s3IAMIntegration := NewS3IAMIntegration(iamManager, "localhost:8888")
// Test that integration is created successfully
assert.NotNil(t, s3IAMIntegration)
assert.True(t, s3IAMIntegration.enabled)
}
func TestS3IAMMiddlewareStaticV4ManagedPolicies(t *testing.T) {
ctx := context.Background()
iamManager := newTestS3IAMManager(t)
allowPolicy := &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Action: policy.StringList{"s3:PutObject", "s3:ListBucket"},
Resource: policy.StringList{"arn:aws:s3:::cli-allowed-bucket", "arn:aws:s3:::cli-allowed-bucket/*"},
},
},
}
require.NoError(t, iamManager.CreatePolicy(ctx, "localhost:8888", "cli-bucket-access-policy", allowPolicy))
s3IAMIntegration := NewS3IAMIntegration(iamManager, "localhost:8888")
identity := &IAMIdentity{
Name: "cli-test-user",
Principal: "arn:aws:iam::000000000000:user/cli-test-user",
PolicyNames: []string{"cli-bucket-access-policy"},
}
putReq := httptest.NewRequest(http.MethodPut, "http://example.com/cli-allowed-bucket/test-file.txt", http.NoBody)
putErrCode := s3IAMIntegration.AuthorizeAction(ctx, identity, s3_constants.ACTION_WRITE, "cli-allowed-bucket", "test-file.txt", putReq)
assert.Equal(t, s3err.ErrNone, putErrCode)
listReq := httptest.NewRequest(http.MethodGet, "http://example.com/cli-allowed-bucket/", http.NoBody)
listErrCode := s3IAMIntegration.AuthorizeAction(ctx, identity, s3_constants.ACTION_LIST, "cli-allowed-bucket", "", listReq)
assert.Equal(t, s3err.ErrNone, listErrCode)
}
func TestS3IAMMiddlewareAttachedPoliciesRestrictDefaultAllow(t *testing.T) {
ctx := context.Background()
iamManager := newTestS3IAMManagerWithDefaultEffect(t, "Allow")
allowPolicy := &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Action: policy.StringList{"s3:PutObject", "s3:ListBucket"},
Resource: policy.StringList{"arn:aws:s3:::cli-allowed-bucket", "arn:aws:s3:::cli-allowed-bucket/*"},
},
},
}
require.NoError(t, iamManager.CreatePolicy(ctx, "localhost:8888", "cli-bucket-access-policy", allowPolicy))
s3IAMIntegration := NewS3IAMIntegration(iamManager, "localhost:8888")
identity := &IAMIdentity{
Name: "cli-test-user",
Principal: "arn:aws:iam::000000000000:user/cli-test-user",
PolicyNames: []string{"cli-bucket-access-policy"},
}
allowedReq := httptest.NewRequest(http.MethodPut, "http://example.com/cli-allowed-bucket/test-file.txt", http.NoBody)
allowedErrCode := s3IAMIntegration.AuthorizeAction(ctx, identity, s3_constants.ACTION_WRITE, "cli-allowed-bucket", "test-file.txt", allowedReq)
assert.Equal(t, s3err.ErrNone, allowedErrCode)
forbiddenReq := httptest.NewRequest(http.MethodPut, "http://example.com/cli-forbidden-bucket/forbidden-file.txt", http.NoBody)
forbiddenErrCode := s3IAMIntegration.AuthorizeAction(ctx, identity, s3_constants.ACTION_WRITE, "cli-forbidden-bucket", "forbidden-file.txt", forbiddenReq)
assert.Equal(t, s3err.ErrAccessDenied, forbiddenErrCode)
forbiddenListReq := httptest.NewRequest(http.MethodGet, "http://example.com/cli-forbidden-bucket/", http.NoBody)
forbiddenListErrCode := s3IAMIntegration.AuthorizeAction(ctx, identity, s3_constants.ACTION_LIST, "cli-forbidden-bucket", "", forbiddenListReq)
assert.Equal(t, s3err.ErrAccessDenied, forbiddenListErrCode)
}
// TestS3IAMMiddlewareJWTAuth tests JWT authentication
func TestS3IAMMiddlewareJWTAuth(t *testing.T) {
// Skip for now since it requires full setup
t.Skip("JWT authentication test requires full IAM setup")
// Create IAM integration
s3iam := NewS3IAMIntegration(nil, "localhost:8888") // Disabled integration
// Create test request with JWT token
req := httptest.NewRequest("GET", "/test-bucket/test-object", http.NoBody)
req.Header.Set("Authorization", "Bearer test-token")
// Test authentication (should return not implemented when disabled)
ctx := context.Background()
identity, errCode := s3iam.AuthenticateJWT(ctx, req)
assert.Nil(t, identity)
assert.NotEqual(t, errCode, 0) // Should return an error
}
// TestBuildS3ResourceArn tests resource ARN building
func TestBuildS3ResourceArn(t *testing.T) {
tests := []struct {
name string
bucket string
object string
expected string
}{
{
name: "empty bucket and object",
bucket: "",
object: "",
expected: "arn:aws:s3:::*",
},
{
name: "bucket only",
bucket: "test-bucket",
object: "",
expected: "arn:aws:s3:::test-bucket",
},
{
name: "bucket and object",
bucket: "test-bucket",
object: "test-object.txt",
expected: "arn:aws:s3:::test-bucket/test-object.txt",
},
{
name: "bucket and object with leading slash",
bucket: "test-bucket",
object: "/test-object.txt",
expected: "arn:aws:s3:::test-bucket/test-object.txt",
},
{
name: "bucket and nested object",
bucket: "test-bucket",
object: "folder/subfolder/test-object.txt",
expected: "arn:aws:s3:::test-bucket/folder/subfolder/test-object.txt",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := buildS3ResourceArn(tt.bucket, tt.object)
assert.Equal(t, tt.expected, result)
})
}
}
// TestDetermineGranularS3Action tests granular S3 action determination from HTTP requests
func TestDetermineGranularS3Action(t *testing.T) {
tests := []struct {
name string
method string
bucket string
objectKey string
queryParams map[string]string
fallbackAction Action
expected string
description string
}{
// Object-level operations
{
name: "get_object",
method: "GET",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_READ,
expected: "s3:GetObject",
description: "Basic object retrieval",
},
{
name: "get_object_acl",
method: "GET",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{"acl": ""},
fallbackAction: s3_constants.ACTION_READ_ACP,
expected: "s3:GetObjectAcl",
description: "Object ACL retrieval",
},
{
name: "get_object_tagging",
method: "GET",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{"tagging": ""},
fallbackAction: s3_constants.ACTION_TAGGING,
expected: "s3:GetObjectTagging",
description: "Object tagging retrieval",
},
{
name: "put_object",
method: "PUT",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:PutObject",
description: "Basic object upload",
},
{
name: "put_object_acl",
method: "PUT",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{"acl": ""},
fallbackAction: s3_constants.ACTION_WRITE_ACP,
expected: "s3:PutObjectAcl",
description: "Object ACL modification",
},
{
name: "delete_object",
method: "DELETE",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_WRITE, // DELETE object uses WRITE fallback
expected: "s3:DeleteObject",
description: "Object deletion - correctly mapped to DeleteObject (not PutObject)",
},
{
name: "delete_object_tagging",
method: "DELETE",
bucket: "test-bucket",
objectKey: "test-object.txt",
queryParams: map[string]string{"tagging": ""},
fallbackAction: s3_constants.ACTION_TAGGING,
expected: "s3:DeleteObjectTagging",
description: "Object tag deletion",
},
// Multipart upload operations
{
name: "create_multipart_upload",
method: "POST",
bucket: "test-bucket",
objectKey: "large-file.txt",
queryParams: map[string]string{"uploads": ""},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:CreateMultipartUpload",
description: "Multipart upload initiation",
},
{
name: "upload_part",
method: "PUT",
bucket: "test-bucket",
objectKey: "large-file.txt",
queryParams: map[string]string{"uploadId": "12345", "partNumber": "1"},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:UploadPart",
description: "Multipart part upload",
},
{
name: "complete_multipart_upload",
method: "POST",
bucket: "test-bucket",
objectKey: "large-file.txt",
queryParams: map[string]string{"uploadId": "12345"},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:CompleteMultipartUpload",
description: "Multipart upload completion",
},
{
name: "abort_multipart_upload",
method: "DELETE",
bucket: "test-bucket",
objectKey: "large-file.txt",
queryParams: map[string]string{"uploadId": "12345"},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:AbortMultipartUpload",
description: "Multipart upload abort",
},
// Bucket-level operations
{
name: "list_bucket",
method: "GET",
bucket: "test-bucket",
objectKey: "",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_LIST,
expected: "s3:ListBucket",
description: "Bucket listing",
},
{
name: "get_bucket_acl",
method: "GET",
bucket: "test-bucket",
objectKey: "",
queryParams: map[string]string{"acl": ""},
fallbackAction: s3_constants.ACTION_READ_ACP,
expected: "s3:GetBucketAcl",
description: "Bucket ACL retrieval",
},
{
name: "put_bucket_policy",
method: "PUT",
bucket: "test-bucket",
objectKey: "",
queryParams: map[string]string{"policy": ""},
fallbackAction: s3_constants.ACTION_WRITE,
expected: "s3:PutBucketPolicy",
description: "Bucket policy modification",
},
{
name: "delete_bucket",
method: "DELETE",
bucket: "test-bucket",
objectKey: "",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_DELETE_BUCKET,
expected: "s3:DeleteBucket",
description: "Bucket deletion",
},
{
name: "list_multipart_uploads",
method: "GET",
bucket: "test-bucket",
objectKey: "",
queryParams: map[string]string{"uploads": ""},
fallbackAction: s3_constants.ACTION_LIST,
expected: "s3:ListBucketMultipartUploads",
description: "List multipart uploads in bucket",
},
// Fallback scenarios
{
name: "legacy_read_fallback",
method: "GET",
bucket: "",
objectKey: "",
queryParams: map[string]string{},
fallbackAction: s3_constants.ACTION_READ,
expected: "s3:GetObject",
description: "Legacy read action fallback",
},
{
name: "already_granular_action",
method: "GET",
bucket: "",
objectKey: "",
queryParams: map[string]string{},
fallbackAction: "s3:GetBucketLocation", // Already granular
expected: "s3:GetBucketLocation",
description: "Already granular action passed through",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create HTTP request with query parameters
req := &http.Request{
Method: tt.method,
URL: &url.URL{Path: "/" + tt.bucket + "/" + tt.objectKey},
}
// Add query parameters
query := req.URL.Query()
for key, value := range tt.queryParams {
query.Set(key, value)
}
req.URL.RawQuery = query.Encode()
// Test the action determination
result := ResolveS3Action(req, string(tt.fallbackAction), tt.bucket, tt.objectKey)
assert.Equal(t, tt.expected, result,
"Test %s failed: %s. Expected %s but got %s",
tt.name, tt.description, tt.expected, result)
})
}
}
// TestMapLegacyActionToIAM tests the legacy action fallback mapping
func TestMapLegacyActionToIAM(t *testing.T) {
tests := []struct {
name string
legacyAction Action
expected string
}{
{
name: "read_action_fallback",
legacyAction: s3_constants.ACTION_READ,
expected: "s3:GetObject",
},
{
name: "write_action_fallback",
legacyAction: s3_constants.ACTION_WRITE,
expected: "s3:PutObject",
},
{
name: "admin_action_fallback",
legacyAction: s3_constants.ACTION_ADMIN,
expected: "s3:*",
},
{
name: "granular_multipart_action",
legacyAction: s3_constants.S3_ACTION_CREATE_MULTIPART,
expected: s3_constants.S3_ACTION_CREATE_MULTIPART,
},
{
name: "unknown_action_with_s3_prefix",
legacyAction: "s3:CustomAction",
expected: "s3:CustomAction",
},
{
name: "unknown_action_without_prefix",
legacyAction: "CustomAction",
expected: "s3:CustomAction",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := mapLegacyActionToIAM(tt.legacyAction)
assert.Equal(t, tt.expected, result)
})
}
}
// TestExtractSourceIP tests source IP extraction from requests
func TestExtractSourceIP(t *testing.T) {
tests := []struct {
name string
setupReq func() *http.Request
expectedIP string
}{
{
name: "X-Forwarded-For header",
setupReq: func() *http.Request {
req := httptest.NewRequest("GET", "/test", http.NoBody)
req.Header.Set("X-Forwarded-For", "192.168.1.100, 10.0.0.1")
// Set RemoteAddr to private IP to simulate trusted proxy
req.RemoteAddr = "127.0.0.1:12345"
return req
},
expectedIP: "192.168.1.100",
},
{
name: "X-Real-IP header",
setupReq: func() *http.Request {
req := httptest.NewRequest("GET", "/test", http.NoBody)
req.Header.Set("X-Real-IP", "192.168.1.200")
// Set RemoteAddr to private IP to simulate trusted proxy
req.RemoteAddr = "127.0.0.1:12345"
return req
},
expectedIP: "192.168.1.200",
},
{
name: "RemoteAddr fallback",
setupReq: func() *http.Request {
req := httptest.NewRequest("GET", "/test", http.NoBody)
req.RemoteAddr = "192.168.1.300:12345"
return req
},
expectedIP: "192.168.1.300",
},
{
name: "Untrusted proxy - public RemoteAddr ignores X-Forwarded-For",
setupReq: func() *http.Request {
req := httptest.NewRequest("GET", "/test", http.NoBody)
req.Header.Set("X-Forwarded-For", "192.168.1.100")
// Public IP - headers should NOT be trusted
req.RemoteAddr = "8.8.8.8:12345"
return req
},
expectedIP: "8.8.8.8", // Should use RemoteAddr, not X-Forwarded-For
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := tt.setupReq()
result := extractSourceIP(req)
assert.Equal(t, tt.expectedIP, result)
})
}
}
// TestExtractRoleNameFromPrincipal tests role name extraction
func TestExtractRoleNameFromPrincipal(t *testing.T) {
tests := []struct {
name string
principal string
expected string
}{
{
name: "valid assumed role ARN",
principal: "arn:aws:sts::assumed-role/S3ReadOnlyRole/session-123",
expected: "S3ReadOnlyRole",
},
{
name: "invalid format",
principal: "invalid-principal",
expected: "", // Returns empty string to signal invalid format
},
{
name: "missing session name",
principal: "arn:aws:sts::assumed-role/TestRole",
expected: "TestRole", // Extracts role name even without session name
},
{
name: "empty principal",
principal: "",
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := utils.ExtractRoleNameFromPrincipal(tt.principal)
assert.Equal(t, tt.expected, result)
})
}
}
// TestIAMIdentityIsAdmin tests the IsAdmin method
func TestIAMIdentityIsAdmin(t *testing.T) {
identity := &IAMIdentity{
Name: "test-identity",
Principal: "arn:aws:sts::assumed-role/TestRole/session",
SessionToken: "test-token",
}
// In our implementation, IsAdmin always returns false since admin status
// is determined by policies, not identity
result := identity.IsAdmin()
assert.False(t, result)
}

420
weed/s3api/s3_multipart_iam.go
View File

@@ -1,420 +0,0 @@
package s3api
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// S3MultipartIAMManager handles IAM integration for multipart upload operations
type S3MultipartIAMManager struct {
s3iam *S3IAMIntegration
}
// NewS3MultipartIAMManager creates a new multipart IAM manager
func NewS3MultipartIAMManager(s3iam *S3IAMIntegration) *S3MultipartIAMManager {
return &S3MultipartIAMManager{
s3iam: s3iam,
}
}
// MultipartUploadRequest represents a multipart upload request
type MultipartUploadRequest struct {
Bucket string `json:"bucket"` // S3 bucket name
ObjectKey string `json:"object_key"` // S3 object key
UploadID string `json:"upload_id"` // Multipart upload ID
PartNumber int `json:"part_number"` // Part number for upload part
Operation string `json:"operation"` // Multipart operation type
SessionToken string `json:"session_token"` // JWT session token
Headers map[string]string `json:"headers"` // Request headers
ContentSize int64 `json:"content_size"` // Content size for validation
}
// MultipartUploadPolicy represents security policies for multipart uploads
type MultipartUploadPolicy struct {
MaxPartSize int64 `json:"max_part_size"` // Maximum part size (5GB AWS limit)
MinPartSize int64 `json:"min_part_size"` // Minimum part size (5MB AWS limit, except last part)
MaxParts int `json:"max_parts"` // Maximum number of parts (10,000 AWS limit)
MaxUploadDuration time.Duration `json:"max_upload_duration"` // Maximum time to complete multipart upload
AllowedContentTypes []string `json:"allowed_content_types"` // Allowed content types
RequiredHeaders []string `json:"required_headers"` // Required headers for validation
IPWhitelist []string `json:"ip_whitelist"` // Allowed IP addresses/ranges
}
// MultipartOperation represents different multipart upload operations
type MultipartOperation string
const (
MultipartOpInitiate MultipartOperation = "initiate"
MultipartOpUploadPart MultipartOperation = "upload_part"
MultipartOpComplete MultipartOperation = "complete"
MultipartOpAbort MultipartOperation = "abort"
MultipartOpList MultipartOperation = "list"
MultipartOpListParts MultipartOperation = "list_parts"
)
// ValidateMultipartOperationWithIAM validates multipart operations using IAM policies
func (iam *IdentityAccessManagement) ValidateMultipartOperationWithIAM(r *http.Request, identity *Identity, operation MultipartOperation) s3err.ErrorCode {
if iam.iamIntegration == nil {
// Fall back to standard validation
return s3err.ErrNone
}
// Extract bucket and object from request
bucket, object := s3_constants.GetBucketAndObject(r)
// Determine the S3 action based on multipart operation
action := determineMultipartS3Action(operation)
// Extract session token from request
sessionToken := extractSessionTokenFromRequest(r)
if sessionToken == "" {
// No session token - use standard auth
return s3err.ErrNone
}
// Retrieve the actual principal ARN from the request header
// This header is set during initial authentication and contains the correct assumed role ARN
principalArn := r.Header.Get("X-SeaweedFS-Principal")
if principalArn == "" {
glog.V(2).Info("IAM authorization for multipart operation failed: missing principal ARN in request header")
return s3err.ErrAccessDenied
}
// Create IAM identity for authorization
iamIdentity := &IAMIdentity{
Name: identity.Name,
Principal: principalArn,
SessionToken: sessionToken,
Account: identity.Account,
}
// Authorize using IAM
ctx := r.Context()
errCode := iam.iamIntegration.AuthorizeAction(ctx, iamIdentity, action, bucket, object, r)
if errCode != s3err.ErrNone {
glog.V(3).Infof("IAM authorization failed for multipart operation: principal=%s operation=%s action=%s bucket=%s object=%s",
iamIdentity.Principal, operation, action, bucket, object)
return errCode
}
glog.V(3).Infof("IAM authorization succeeded for multipart operation: principal=%s operation=%s action=%s bucket=%s object=%s",
iamIdentity.Principal, operation, action, bucket, object)
return s3err.ErrNone
}
// ValidateMultipartRequestWithPolicy validates multipart request against security policy
func (policy *MultipartUploadPolicy) ValidateMultipartRequestWithPolicy(req *MultipartUploadRequest) error {
if req == nil {
return fmt.Errorf("multipart request cannot be nil")
}
// Validate part size for upload part operations
if req.Operation == string(MultipartOpUploadPart) {
if req.ContentSize > policy.MaxPartSize {
return fmt.Errorf("part size %d exceeds maximum allowed %d", req.ContentSize, policy.MaxPartSize)
}
// Minimum part size validation (except for last part)
// Note: Last part validation would require knowing if this is the final part
if req.ContentSize < policy.MinPartSize && req.ContentSize > 0 {
glog.V(2).Infof("Part size %d is below minimum %d - assuming last part", req.ContentSize, policy.MinPartSize)
}
// Validate part number
if req.PartNumber < 1 || req.PartNumber > policy.MaxParts {
return fmt.Errorf("part number %d is invalid (must be 1-%d)", req.PartNumber, policy.MaxParts)
}
}
// Validate required headers first
if req.Headers != nil {
for _, requiredHeader := range policy.RequiredHeaders {
if _, exists := req.Headers[requiredHeader]; !exists {
// Check lowercase version
if _, exists := req.Headers[strings.ToLower(requiredHeader)]; !exists {
return fmt.Errorf("required header %s is missing", requiredHeader)
}
}
}
}
// Validate content type if specified
if len(policy.AllowedContentTypes) > 0 && req.Headers != nil {
contentType := req.Headers["Content-Type"]
if contentType == "" {
contentType = req.Headers["content-type"]
}
allowed := false
for _, allowedType := range policy.AllowedContentTypes {
if contentType == allowedType {
allowed = true
break
}
}
if !allowed {
return fmt.Errorf("content type %s is not allowed", contentType)
}
}
return nil
}
// Enhanced multipart handlers with IAM integration
// NewMultipartUploadWithIAM handles initiate multipart upload with IAM validation
func (s3a *S3ApiServer) NewMultipartUploadWithIAM(w http.ResponseWriter, r *http.Request) {
// Validate IAM permissions first
if s3a.iam.iamIntegration != nil {
if identity, errCode := s3a.iam.authRequest(r, s3_constants.ACTION_WRITE); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
} else {
// Additional multipart-specific IAM validation
if errCode := s3a.iam.ValidateMultipartOperationWithIAM(r, identity, MultipartOpInitiate); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
}
}
// Delegate to existing handler
s3a.NewMultipartUploadHandler(w, r)
}
// CompleteMultipartUploadWithIAM handles complete multipart upload with IAM validation
func (s3a *S3ApiServer) CompleteMultipartUploadWithIAM(w http.ResponseWriter, r *http.Request) {
// Validate IAM permissions first
if s3a.iam.iamIntegration != nil {
if identity, errCode := s3a.iam.authRequest(r, s3_constants.ACTION_WRITE); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
} else {
// Additional multipart-specific IAM validation
if errCode := s3a.iam.ValidateMultipartOperationWithIAM(r, identity, MultipartOpComplete); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
}
}
// Delegate to existing handler
s3a.CompleteMultipartUploadHandler(w, r)
}
// AbortMultipartUploadWithIAM handles abort multipart upload with IAM validation
func (s3a *S3ApiServer) AbortMultipartUploadWithIAM(w http.ResponseWriter, r *http.Request) {
// Validate IAM permissions first
if s3a.iam.iamIntegration != nil {
if identity, errCode := s3a.iam.authRequest(r, s3_constants.ACTION_WRITE); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
} else {
// Additional multipart-specific IAM validation
if errCode := s3a.iam.ValidateMultipartOperationWithIAM(r, identity, MultipartOpAbort); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
}
}
// Delegate to existing handler
s3a.AbortMultipartUploadHandler(w, r)
}
// ListMultipartUploadsWithIAM handles list multipart uploads with IAM validation
func (s3a *S3ApiServer) ListMultipartUploadsWithIAM(w http.ResponseWriter, r *http.Request) {
// Validate IAM permissions first
if s3a.iam.iamIntegration != nil {
if identity, errCode := s3a.iam.authRequest(r, s3_constants.ACTION_LIST); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
} else {
// Additional multipart-specific IAM validation
if errCode := s3a.iam.ValidateMultipartOperationWithIAM(r, identity, MultipartOpList); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
}
}
// Delegate to existing handler
s3a.ListMultipartUploadsHandler(w, r)
}
// UploadPartWithIAM handles upload part with IAM validation
func (s3a *S3ApiServer) UploadPartWithIAM(w http.ResponseWriter, r *http.Request) {
// Validate IAM permissions first
if s3a.iam.iamIntegration != nil {
if identity, errCode := s3a.iam.authRequest(r, s3_constants.ACTION_WRITE); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
} else {
// Additional multipart-specific IAM validation
if errCode := s3a.iam.ValidateMultipartOperationWithIAM(r, identity, MultipartOpUploadPart); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Validate part size and other policies
if err := s3a.validateUploadPartRequest(r); err != nil {
glog.Errorf("Upload part validation failed: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
}
}
// Delegate to existing object PUT handler (which handles upload part)
s3a.PutObjectHandler(w, r)
}
// Helper functions
// determineMultipartS3Action maps multipart operations to granular S3 actions
// This enables fine-grained IAM policies for multipart upload operations
func determineMultipartS3Action(operation MultipartOperation) Action {
switch operation {
case MultipartOpInitiate:
return s3_constants.S3_ACTION_CREATE_MULTIPART
case MultipartOpUploadPart:
return s3_constants.S3_ACTION_UPLOAD_PART
case MultipartOpComplete:
return s3_constants.S3_ACTION_COMPLETE_MULTIPART
case MultipartOpAbort:
return s3_constants.S3_ACTION_ABORT_MULTIPART
case MultipartOpList:
return s3_constants.S3_ACTION_LIST_MULTIPART_UPLOADS
case MultipartOpListParts:
return s3_constants.S3_ACTION_LIST_PARTS
default:
// Fail closed for unmapped operations to prevent unintended access
glog.Errorf("unmapped multipart operation: %s", operation)
return "s3:InternalErrorUnknownMultipartAction" // Non-existent action ensures denial
}
}
// extractSessionTokenFromRequest extracts session token from various request sources
func extractSessionTokenFromRequest(r *http.Request) string {
// Check Authorization header for Bearer token
if authHeader := r.Header.Get("Authorization"); authHeader != "" {
if strings.HasPrefix(authHeader, "Bearer ") {
return strings.TrimPrefix(authHeader, "Bearer ")
}
}
// Check X-Amz-Security-Token header
if token := r.Header.Get("X-Amz-Security-Token"); token != "" {
return token
}
// Check query parameters for presigned URL tokens
if token := r.URL.Query().Get("X-Amz-Security-Token"); token != "" {
return token
}
return ""
}
// validateUploadPartRequest validates upload part request against policies
func (s3a *S3ApiServer) validateUploadPartRequest(r *http.Request) error {
// Get default multipart policy
policy := DefaultMultipartUploadPolicy()
// Extract part number from query
partNumberStr := r.URL.Query().Get("partNumber")
if partNumberStr == "" {
return fmt.Errorf("missing partNumber parameter")
}
partNumber, err := strconv.Atoi(partNumberStr)
if err != nil {
return fmt.Errorf("invalid partNumber: %v", err)
}
// Get content length
contentLength := r.ContentLength
if contentLength < 0 {
contentLength = 0
}
// Create multipart request for validation
bucket, object := s3_constants.GetBucketAndObject(r)
multipartReq := &MultipartUploadRequest{
Bucket: bucket,
ObjectKey: object,
PartNumber: partNumber,
Operation: string(MultipartOpUploadPart),
ContentSize: contentLength,
Headers: make(map[string]string),
}
// Copy relevant headers
for key, values := range r.Header {
if len(values) > 0 {
multipartReq.Headers[key] = values[0]
}
}
// Validate against policy
return policy.ValidateMultipartRequestWithPolicy(multipartReq)
}
// DefaultMultipartUploadPolicy returns a default multipart upload security policy
func DefaultMultipartUploadPolicy() *MultipartUploadPolicy {
return &MultipartUploadPolicy{
MaxPartSize: 5 * 1024 * 1024 * 1024, // 5GB AWS limit
MinPartSize: 5 * 1024 * 1024, // 5MB AWS minimum (except last part)
MaxParts: 10000, // AWS limit
MaxUploadDuration: 7 * 24 * time.Hour, // 7 days to complete upload
AllowedContentTypes: []string{}, // Empty means all types allowed
RequiredHeaders: []string{}, // No required headers by default
IPWhitelist: []string{}, // Empty means no IP restrictions
}
}
// MultipartUploadSession represents an ongoing multipart upload session
type MultipartUploadSession struct {
UploadID string `json:"upload_id"`
Bucket string `json:"bucket"`
ObjectKey string `json:"object_key"`
Initiator string `json:"initiator"` // User who initiated the upload
Owner string `json:"owner"` // Object owner
CreatedAt time.Time `json:"created_at"` // When upload was initiated
Parts []MultipartUploadPart `json:"parts"` // Uploaded parts
Metadata map[string]string `json:"metadata"` // Object metadata
Policy *MultipartUploadPolicy `json:"policy"` // Applied security policy
SessionToken string `json:"session_token"` // IAM session token
}
// MultipartUploadPart represents an uploaded part
type MultipartUploadPart struct {
PartNumber int `json:"part_number"`
Size int64 `json:"size"`
ETag string `json:"etag"`
LastModified time.Time `json:"last_modified"`
Checksum string `json:"checksum"` // Optional integrity checksum
}
// GetMultipartUploadSessions retrieves active multipart upload sessions for a bucket
func (s3a *S3ApiServer) GetMultipartUploadSessions(bucket string) ([]*MultipartUploadSession, error) {
// This would typically query the filer for active multipart uploads
// For now, return empty list as this is a placeholder for the full implementation
return []*MultipartUploadSession{}, nil
}
// CleanupExpiredMultipartUploads removes expired multipart upload sessions
func (s3a *S3ApiServer) CleanupExpiredMultipartUploads(maxAge time.Duration) error {
// This would typically scan for and remove expired multipart uploads
// Implementation would depend on how multipart sessions are stored in the filer
glog.V(2).Infof("Cleanup expired multipart uploads older than %v", maxAge)
return nil
}

614
weed/s3api/s3_multipart_iam_test.go
View File

@@ -1,614 +0,0 @@
package s3api
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/golang-jwt/jwt/v5"
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/iam/ldap"
"github.com/seaweedfs/seaweedfs/weed/iam/oidc"
"github.com/seaweedfs/seaweedfs/weed/iam/policy"
"github.com/seaweedfs/seaweedfs/weed/iam/sts"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// createTestJWTMultipart creates a test JWT token with the specified issuer, subject and signing key
func createTestJWTMultipart(t *testing.T, issuer, subject, signingKey string) string {
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"iss": issuer,
"sub": subject,
"aud": "test-client-id",
"exp": time.Now().Add(time.Hour).Unix(),
"iat": time.Now().Unix(),
// Add claims that trust policy validation expects
"idp": "test-oidc", // Identity provider claim for trust policy matching
})
tokenString, err := token.SignedString([]byte(signingKey))
require.NoError(t, err)
return tokenString
}
// TestMultipartIAMValidation tests IAM validation for multipart operations
func TestMultipartIAMValidation(t *testing.T) {
// Set up IAM system
iamManager := setupTestIAMManagerForMultipart(t)
s3iam := NewS3IAMIntegration(iamManager, "localhost:8888")
s3iam.enabled = true
// Create IAM with integration
iam := &IdentityAccessManagement{
isAuthEnabled: true,
}
iam.SetIAMIntegration(s3iam)
// Set up roles
ctx := context.Background()
setupTestRolesForMultipart(ctx, iamManager)
// Create a valid JWT token for testing
validJWTToken := createTestJWTMultipart(t, "https://test-issuer.com", "test-user-123", "test-signing-key")
// Get session token
response, err := iamManager.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
RoleArn: "arn:aws:iam::role/S3WriteRole",
WebIdentityToken: validJWTToken,
RoleSessionName: "multipart-test-session",
})
require.NoError(t, err)
sessionToken := response.Credentials.SessionToken
tests := []struct {
name string
operation MultipartOperation
method string
path string
sessionToken string
expectedResult s3err.ErrorCode
}{
{
name: "Initiate multipart upload",
operation: MultipartOpInitiate,
method: "POST",
path: "/test-bucket/test-file.txt?uploads",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "Upload part",
operation: MultipartOpUploadPart,
method: "PUT",
path: "/test-bucket/test-file.txt?partNumber=1&uploadId=test-upload-id",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "Complete multipart upload",
operation: MultipartOpComplete,
method: "POST",
path: "/test-bucket/test-file.txt?uploadId=test-upload-id",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "Abort multipart upload",
operation: MultipartOpAbort,
method: "DELETE",
path: "/test-bucket/test-file.txt?uploadId=test-upload-id",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "List multipart uploads",
operation: MultipartOpList,
method: "GET",
path: "/test-bucket?uploads",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "Upload part without session token",
operation: MultipartOpUploadPart,
method: "PUT",
path: "/test-bucket/test-file.txt?partNumber=1&uploadId=test-upload-id",
sessionToken: "",
expectedResult: s3err.ErrNone, // Falls back to standard auth
},
{
name: "Upload part with invalid session token",
operation: MultipartOpUploadPart,
method: "PUT",
path: "/test-bucket/test-file.txt?partNumber=1&uploadId=test-upload-id",
sessionToken: "invalid-token",
expectedResult: s3err.ErrAccessDenied,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create request for multipart operation
req := createMultipartRequest(t, tt.method, tt.path, tt.sessionToken)
// Create identity for testing
identity := &Identity{
Name: "test-user",
Account: &AccountAdmin,
}
// Test validation
result := iam.ValidateMultipartOperationWithIAM(req, identity, tt.operation)
assert.Equal(t, tt.expectedResult, result, "Multipart IAM validation result should match expected")
})
}
}
// TestMultipartUploadPolicy tests multipart upload security policies
func TestMultipartUploadPolicy(t *testing.T) {
policy := &MultipartUploadPolicy{
MaxPartSize: 10 * 1024 * 1024, // 10MB for testing
MinPartSize: 5 * 1024 * 1024, // 5MB minimum
MaxParts: 100, // 100 parts max for testing
AllowedContentTypes: []string{"application/json", "text/plain"},
RequiredHeaders: []string{"Content-Type"},
}
tests := []struct {
name string
request *MultipartUploadRequest
expectedError string
}{
{
name: "Valid upload part request",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 1,
Operation: string(MultipartOpUploadPart),
ContentSize: 8 * 1024 * 1024, // 8MB
Headers: map[string]string{
"Content-Type": "application/json",
},
},
expectedError: "",
},
{
name: "Part size too large",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 1,
Operation: string(MultipartOpUploadPart),
ContentSize: 15 * 1024 * 1024, // 15MB exceeds limit
Headers: map[string]string{
"Content-Type": "application/json",
},
},
expectedError: "part size",
},
{
name: "Invalid part number (too high)",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 150, // Exceeds max parts
Operation: string(MultipartOpUploadPart),
ContentSize: 8 * 1024 * 1024,
Headers: map[string]string{
"Content-Type": "application/json",
},
},
expectedError: "part number",
},
{
name: "Invalid part number (too low)",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 0, // Must be >= 1
Operation: string(MultipartOpUploadPart),
ContentSize: 8 * 1024 * 1024,
Headers: map[string]string{
"Content-Type": "application/json",
},
},
expectedError: "part number",
},
{
name: "Content type not allowed",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 1,
Operation: string(MultipartOpUploadPart),
ContentSize: 8 * 1024 * 1024,
Headers: map[string]string{
"Content-Type": "video/mp4", // Not in allowed list
},
},
expectedError: "content type video/mp4 is not allowed",
},
{
name: "Missing required header",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
PartNumber: 1,
Operation: string(MultipartOpUploadPart),
ContentSize: 8 * 1024 * 1024,
Headers: map[string]string{}, // Missing Content-Type
},
expectedError: "required header Content-Type is missing",
},
{
name: "Non-upload operation (should not validate size)",
request: &MultipartUploadRequest{
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Operation: string(MultipartOpInitiate),
Headers: map[string]string{
"Content-Type": "application/json",
},
},
expectedError: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := policy.ValidateMultipartRequestWithPolicy(tt.request)
if tt.expectedError == "" {
assert.NoError(t, err, "Policy validation should succeed")
} else {
assert.Error(t, err, "Policy validation should fail")
assert.Contains(t, err.Error(), tt.expectedError, "Error message should contain expected text")
}
})
}
}
// TestMultipartS3ActionMapping tests the mapping of multipart operations to S3 actions
func TestMultipartS3ActionMapping(t *testing.T) {
tests := []struct {
operation MultipartOperation
expectedAction Action
}{
{MultipartOpInitiate, s3_constants.S3_ACTION_CREATE_MULTIPART},
{MultipartOpUploadPart, s3_constants.S3_ACTION_UPLOAD_PART},
{MultipartOpComplete, s3_constants.S3_ACTION_COMPLETE_MULTIPART},
{MultipartOpAbort, s3_constants.S3_ACTION_ABORT_MULTIPART},
{MultipartOpList, s3_constants.S3_ACTION_LIST_MULTIPART_UPLOADS},
{MultipartOpListParts, s3_constants.S3_ACTION_LIST_PARTS},
{MultipartOperation("unknown"), "s3:InternalErrorUnknownMultipartAction"}, // Fail-closed for security
}
for _, tt := range tests {
t.Run(string(tt.operation), func(t *testing.T) {
action := determineMultipartS3Action(tt.operation)
assert.Equal(t, tt.expectedAction, action, "S3 action mapping should match expected")
})
}
}
// TestSessionTokenExtraction tests session token extraction from various sources
func TestSessionTokenExtraction(t *testing.T) {
tests := []struct {
name string
setupRequest func() *http.Request
expectedToken string
}{
{
name: "Bearer token in Authorization header",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt", nil)
req.Header.Set("Authorization", "Bearer test-session-token-123")
return req
},
expectedToken: "test-session-token-123",
},
{
name: "X-Amz-Security-Token header",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt", nil)
req.Header.Set("X-Amz-Security-Token", "security-token-456")
return req
},
expectedToken: "security-token-456",
},
{
name: "X-Amz-Security-Token query parameter",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt?X-Amz-Security-Token=query-token-789", nil)
return req
},
expectedToken: "query-token-789",
},
{
name: "No token present",
setupRequest: func() *http.Request {
return httptest.NewRequest("PUT", "/test-bucket/test-file.txt", nil)
},
expectedToken: "",
},
{
name: "Authorization header without Bearer",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt", nil)
req.Header.Set("Authorization", "AWS access_key:signature")
return req
},
expectedToken: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := tt.setupRequest()
token := extractSessionTokenFromRequest(req)
assert.Equal(t, tt.expectedToken, token, "Extracted token should match expected")
})
}
}
// TestUploadPartValidation tests upload part request validation
func TestUploadPartValidation(t *testing.T) {
s3Server := &S3ApiServer{}
tests := []struct {
name string
setupRequest func() *http.Request
expectedError string
}{
{
name: "Valid upload part request",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt?partNumber=1&uploadId=test-123", nil)
req.Header.Set("Content-Type", "application/octet-stream")
req.ContentLength = 6 * 1024 * 1024 // 6MB
return req
},
expectedError: "",
},
{
name: "Missing partNumber parameter",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt?uploadId=test-123", nil)
req.Header.Set("Content-Type", "application/octet-stream")
req.ContentLength = 6 * 1024 * 1024
return req
},
expectedError: "missing partNumber parameter",
},
{
name: "Invalid partNumber format",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt?partNumber=abc&uploadId=test-123", nil)
req.Header.Set("Content-Type", "application/octet-stream")
req.ContentLength = 6 * 1024 * 1024
return req
},
expectedError: "invalid partNumber",
},
{
name: "Part size too large",
setupRequest: func() *http.Request {
req := httptest.NewRequest("PUT", "/test-bucket/test-file.txt?partNumber=1&uploadId=test-123", nil)
req.Header.Set("Content-Type", "application/octet-stream")
req.ContentLength = 6 * 1024 * 1024 * 1024 // 6GB exceeds 5GB limit
return req
},
expectedError: "part size",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := tt.setupRequest()
err := s3Server.validateUploadPartRequest(req)
if tt.expectedError == "" {
assert.NoError(t, err, "Upload part validation should succeed")
} else {
assert.Error(t, err, "Upload part validation should fail")
assert.Contains(t, err.Error(), tt.expectedError, "Error message should contain expected text")
}
})
}
}
// TestDefaultMultipartUploadPolicy tests the default policy configuration
func TestDefaultMultipartUploadPolicy(t *testing.T) {
policy := DefaultMultipartUploadPolicy()
assert.Equal(t, int64(5*1024*1024*1024), policy.MaxPartSize, "Max part size should be 5GB")
assert.Equal(t, int64(5*1024*1024), policy.MinPartSize, "Min part size should be 5MB")
assert.Equal(t, 10000, policy.MaxParts, "Max parts should be 10,000")
assert.Equal(t, 7*24*time.Hour, policy.MaxUploadDuration, "Max upload duration should be 7 days")
assert.Empty(t, policy.AllowedContentTypes, "Should allow all content types by default")
assert.Empty(t, policy.RequiredHeaders, "Should have no required headers by default")
assert.Empty(t, policy.IPWhitelist, "Should have no IP restrictions by default")
}
// TestMultipartUploadSession tests multipart upload session structure
func TestMultipartUploadSession(t *testing.T) {
session := &MultipartUploadSession{
UploadID: "test-upload-123",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Initiator: "arn:aws:iam::user/testuser",
Owner: "arn:aws:iam::user/testuser",
CreatedAt: time.Now(),
Parts: []MultipartUploadPart{
{
PartNumber: 1,
Size: 5 * 1024 * 1024,
ETag: "abc123",
LastModified: time.Now(),
Checksum: "sha256:def456",
},
},
Metadata: map[string]string{
"Content-Type": "application/octet-stream",
"x-amz-meta-custom": "value",
},
Policy: DefaultMultipartUploadPolicy(),
SessionToken: "session-token-789",
}
assert.NotEmpty(t, session.UploadID, "Upload ID should not be empty")
assert.NotEmpty(t, session.Bucket, "Bucket should not be empty")
assert.NotEmpty(t, session.ObjectKey, "Object key should not be empty")
assert.Len(t, session.Parts, 1, "Should have one part")
assert.Equal(t, 1, session.Parts[0].PartNumber, "Part number should be 1")
assert.NotNil(t, session.Policy, "Policy should not be nil")
}
// Helper functions for tests
func setupTestIAMManagerForMultipart(t *testing.T) *integration.IAMManager {
// Create IAM manager
manager := integration.NewIAMManager()
// Initialize with test configuration
config := &integration.IAMConfig{
STS: &sts.STSConfig{
TokenDuration: sts.FlexibleDuration{Duration: time.Hour},
MaxSessionLength: sts.FlexibleDuration{Duration: time.Hour * 12},
Issuer: "test-sts",
SigningKey: []byte("test-signing-key-32-characters-long"),
},
Policy: &policy.PolicyEngineConfig{
DefaultEffect: "Deny",
StoreType: "memory",
},
Roles: &integration.RoleStoreConfig{
StoreType: "memory",
},
}
err := manager.Initialize(config, func() string {
return "localhost:8888" // Mock filer address for testing
})
require.NoError(t, err)
// Set up test identity providers
setupTestProvidersForMultipart(t, manager)
return manager
}
func setupTestProvidersForMultipart(t *testing.T, manager *integration.IAMManager) {
// Set up OIDC provider
oidcProvider := oidc.NewMockOIDCProvider("test-oidc")
oidcConfig := &oidc.OIDCConfig{
Issuer: "https://test-issuer.com",
ClientID: "test-client-id",
}
err := oidcProvider.Initialize(oidcConfig)
require.NoError(t, err)
oidcProvider.SetupDefaultTestData()
// Set up LDAP provider
ldapProvider := ldap.NewMockLDAPProvider("test-ldap")
err = ldapProvider.Initialize(nil) // Mock doesn't need real config
require.NoError(t, err)
ldapProvider.SetupDefaultTestData()
// Register providers
err = manager.RegisterIdentityProvider(oidcProvider)
require.NoError(t, err)
err = manager.RegisterIdentityProvider(ldapProvider)
require.NoError(t, err)
}
func setupTestRolesForMultipart(ctx context.Context, manager *integration.IAMManager) {
// Create write policy for multipart operations
writePolicy := &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "AllowS3MultipartOperations",
Effect: "Allow",
Action: []string{
"s3:PutObject",
"s3:GetObject",
"s3:ListBucket",
"s3:DeleteObject",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
manager.CreatePolicy(ctx, "", "S3WritePolicy", writePolicy)
// Create write role
manager.CreateRole(ctx, "", "S3WriteRole", &integration.RoleDefinition{
RoleName: "S3WriteRole",
TrustPolicy: &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Principal: map[string]interface{}{
"Federated": "test-oidc",
},
Action: []string{"sts:AssumeRoleWithWebIdentity"},
},
},
},
AttachedPolicies: []string{"S3WritePolicy"},
})
// Create a role for multipart users
manager.CreateRole(ctx, "", "MultipartUser", &integration.RoleDefinition{
RoleName: "MultipartUser",
TrustPolicy: &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Principal: map[string]interface{}{
"Federated": "test-oidc",
},
Action: []string{"sts:AssumeRoleWithWebIdentity"},
},
},
},
AttachedPolicies: []string{"S3WritePolicy"},
})
}
func createMultipartRequest(t *testing.T, method, path, sessionToken string) *http.Request {
req := httptest.NewRequest(method, path, nil)
// Add session token if provided
if sessionToken != "" {
req.Header.Set("Authorization", "Bearer "+sessionToken)
// Set the principal ARN header that matches the assumed role from the test setup
// This corresponds to the role "arn:aws:iam::role/S3WriteRole" with session name "multipart-test-session"
req.Header.Set("X-SeaweedFS-Principal", "arn:aws:sts::assumed-role/S3WriteRole/multipart-test-session")
}
// Add common headers
req.Header.Set("Content-Type", "application/octet-stream")
return req
}

618
weed/s3api/s3_policy_templates.go
View File

@@ -1,618 +0,0 @@
package s3api
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/iam/policy"
)
// S3PolicyTemplates provides pre-built IAM policy templates for common S3 use cases
type S3PolicyTemplates struct{}
// NewS3PolicyTemplates creates a new policy templates provider
func NewS3PolicyTemplates() *S3PolicyTemplates {
return &S3PolicyTemplates{}
}
// GetS3ReadOnlyPolicy returns a policy that allows read-only access to all S3 resources
func (t *S3PolicyTemplates) GetS3ReadOnlyPolicy() *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "S3ReadOnlyAccess",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
"s3:GetBucketVersioning",
"s3:ListAllMyBuckets",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
}
// GetS3WriteOnlyPolicy returns a policy that allows write-only access to all S3 resources
func (t *S3PolicyTemplates) GetS3WriteOnlyPolicy() *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "S3WriteOnlyAccess",
Effect: "Allow",
Action: []string{
"s3:PutObject",
"s3:PutObjectAcl",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
}
// GetS3AdminPolicy returns a policy that allows full admin access to all S3 resources
func (t *S3PolicyTemplates) GetS3AdminPolicy() *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "S3FullAccess",
Effect: "Allow",
Action: []string{
"s3:*",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
}
// GetBucketSpecificReadPolicy returns a policy for read-only access to a specific bucket
func (t *S3PolicyTemplates) GetBucketSpecificReadPolicy(bucketName string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "BucketSpecificReadAccess",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:GetObjectVersion",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
"arn:aws:s3:::" + bucketName + "/*",
},
},
},
}
}
// GetBucketSpecificWritePolicy returns a policy for write-only access to a specific bucket
func (t *S3PolicyTemplates) GetBucketSpecificWritePolicy(bucketName string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "BucketSpecificWriteAccess",
Effect: "Allow",
Action: []string{
"s3:PutObject",
"s3:PutObjectAcl",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
"arn:aws:s3:::" + bucketName + "/*",
},
},
},
}
}
// GetPathBasedAccessPolicy returns a policy that restricts access to a specific path within a bucket
func (t *S3PolicyTemplates) GetPathBasedAccessPolicy(bucketName, pathPrefix string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "ListBucketPermission",
Effect: "Allow",
Action: []string{
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
},
Condition: map[string]map[string]interface{}{
"StringLike": map[string]interface{}{
"s3:prefix": []string{pathPrefix + "/*"},
},
},
},
{
Sid: "PathBasedObjectAccess",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
},
Resource: []string{
"arn:aws:s3:::" + bucketName + "/" + pathPrefix + "/*",
},
},
},
}
}
// GetIPRestrictedPolicy returns a policy that restricts access based on source IP
func (t *S3PolicyTemplates) GetIPRestrictedPolicy(allowedCIDRs []string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "IPRestrictedS3Access",
Effect: "Allow",
Action: []string{
"s3:*",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
Condition: map[string]map[string]interface{}{
"IpAddress": map[string]interface{}{
"aws:SourceIp": allowedCIDRs,
},
},
},
},
}
}
// GetTimeBasedAccessPolicy returns a policy that allows access only during specific hours
func (t *S3PolicyTemplates) GetTimeBasedAccessPolicy(startHour, endHour int) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "TimeBasedS3Access",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
Condition: map[string]map[string]interface{}{
"DateGreaterThan": map[string]interface{}{
"aws:CurrentTime": time.Now().Format("2006-01-02") + "T" +
formatHour(startHour) + ":00:00Z",
},
"DateLessThan": map[string]interface{}{
"aws:CurrentTime": time.Now().Format("2006-01-02") + "T" +
formatHour(endHour) + ":00:00Z",
},
},
},
},
}
}
// GetMultipartUploadPolicy returns a policy specifically for multipart upload operations
func (t *S3PolicyTemplates) GetMultipartUploadPolicy(bucketName string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "MultipartUploadOperations",
Effect: "Allow",
Action: []string{
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
},
Resource: []string{
"arn:aws:s3:::" + bucketName + "/*",
},
},
{
Sid: "ListBucketForMultipart",
Effect: "Allow",
Action: []string{
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
},
},
},
}
}
// GetPresignedURLPolicy returns a policy for generating and using presigned URLs
func (t *S3PolicyTemplates) GetPresignedURLPolicy(bucketName string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "PresignedURLAccess",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:PutObject",
},
Resource: []string{
"arn:aws:s3:::" + bucketName + "/*",
},
Condition: map[string]map[string]interface{}{
"StringEquals": map[string]interface{}{
"s3:x-amz-signature-version": "AWS4-HMAC-SHA256",
},
},
},
},
}
}
// GetTemporaryAccessPolicy returns a policy for temporary access with expiration
func (t *S3PolicyTemplates) GetTemporaryAccessPolicy(bucketName string, expirationHours int) *policy.PolicyDocument {
expirationTime := time.Now().Add(time.Duration(expirationHours) * time.Hour)
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "TemporaryS3Access",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
"arn:aws:s3:::" + bucketName + "/*",
},
Condition: map[string]map[string]interface{}{
"DateLessThan": map[string]interface{}{
"aws:CurrentTime": expirationTime.UTC().Format("2006-01-02T15:04:05Z"),
},
},
},
},
}
}
// GetContentTypeRestrictedPolicy returns a policy that restricts uploads to specific content types
func (t *S3PolicyTemplates) GetContentTypeRestrictedPolicy(bucketName string, allowedContentTypes []string) *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "ContentTypeRestrictedUpload",
Effect: "Allow",
Action: []string{
"s3:PutObject",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
},
Resource: []string{
"arn:aws:s3:::" + bucketName + "/*",
},
Condition: map[string]map[string]interface{}{
"StringEquals": map[string]interface{}{
"s3:content-type": allowedContentTypes,
},
},
},
{
Sid: "ReadAccess",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:ListBucket",
},
Resource: []string{
"arn:aws:s3:::" + bucketName,
"arn:aws:s3:::" + bucketName + "/*",
},
},
},
}
}
// GetDenyDeletePolicy returns a policy that allows all operations except delete
func (t *S3PolicyTemplates) GetDenyDeletePolicy() *policy.PolicyDocument {
return &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "AllowAllExceptDelete",
Effect: "Allow",
Action: []string{
"s3:GetObject",
"s3:GetObjectVersion",
"s3:PutObject",
"s3:PutObjectAcl",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:CreateMultipartUpload",
"s3:UploadPart",
"s3:CompleteMultipartUpload",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploads",
"s3:ListParts",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
{
Sid: "DenyDeleteOperations",
Effect: "Deny",
Action: []string{
"s3:DeleteObject",
"s3:DeleteObjectVersion",
"s3:DeleteBucket",
},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
}
// Helper function to format hour with leading zero
func formatHour(hour int) string {
if hour < 10 {
return "0" + string(rune('0'+hour))
}
return string(rune('0'+hour/10)) + string(rune('0'+hour%10))
}
// PolicyTemplateDefinition represents metadata about a policy template
type PolicyTemplateDefinition struct {
Name string `json:"name"`
Description string `json:"description"`
Category string `json:"category"`
UseCase string `json:"use_case"`
Parameters []PolicyTemplateParam `json:"parameters,omitempty"`
Policy *policy.PolicyDocument `json:"policy"`
}
// PolicyTemplateParam represents a parameter for customizing policy templates
type PolicyTemplateParam struct {
Name string `json:"name"`
Type string `json:"type"`
Description string `json:"description"`
Required bool `json:"required"`
DefaultValue string `json:"default_value,omitempty"`
Example string `json:"example,omitempty"`
}
// GetAllPolicyTemplates returns all available policy templates with metadata
func (t *S3PolicyTemplates) GetAllPolicyTemplates() []PolicyTemplateDefinition {
return []PolicyTemplateDefinition{
{
Name: "S3ReadOnlyAccess",
Description: "Provides read-only access to all S3 buckets and objects",
Category: "Basic Access",
UseCase: "Data consumers, backup services, monitoring applications",
Policy: t.GetS3ReadOnlyPolicy(),
},
{
Name: "S3WriteOnlyAccess",
Description: "Provides write-only access to all S3 buckets and objects",
Category: "Basic Access",
UseCase: "Data ingestion services, backup applications",
Policy: t.GetS3WriteOnlyPolicy(),
},
{
Name: "S3AdminAccess",
Description: "Provides full administrative access to all S3 resources",
Category: "Administrative",
UseCase: "S3 administrators, service accounts with full control",
Policy: t.GetS3AdminPolicy(),
},
{
Name: "BucketSpecificRead",
Description: "Provides read-only access to a specific bucket",
Category: "Bucket-Specific",
UseCase: "Applications that need access to specific data sets",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket to grant access to",
Required: true,
Example: "my-data-bucket",
},
},
Policy: t.GetBucketSpecificReadPolicy("${bucketName}"),
},
{
Name: "BucketSpecificWrite",
Description: "Provides write-only access to a specific bucket",
Category: "Bucket-Specific",
UseCase: "Upload services, data ingestion for specific datasets",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket to grant access to",
Required: true,
Example: "my-upload-bucket",
},
},
Policy: t.GetBucketSpecificWritePolicy("${bucketName}"),
},
{
Name: "PathBasedAccess",
Description: "Restricts access to a specific path/prefix within a bucket",
Category: "Path-Restricted",
UseCase: "Multi-tenant applications, user-specific directories",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket",
Required: true,
Example: "shared-bucket",
},
{
Name: "pathPrefix",
Type: "string",
Description: "Path prefix to restrict access to",
Required: true,
Example: "user123/documents",
},
},
Policy: t.GetPathBasedAccessPolicy("${bucketName}", "${pathPrefix}"),
},
{
Name: "IPRestrictedAccess",
Description: "Allows access only from specific IP addresses or ranges",
Category: "Security",
UseCase: "Corporate networks, office-based access, VPN restrictions",
Parameters: []PolicyTemplateParam{
{
Name: "allowedCIDRs",
Type: "array",
Description: "List of allowed IP addresses or CIDR ranges",
Required: true,
Example: "[\"192.168.1.0/24\", \"10.0.0.0/8\"]",
},
},
Policy: t.GetIPRestrictedPolicy([]string{"${allowedCIDRs}"}),
},
{
Name: "MultipartUploadOnly",
Description: "Allows only multipart upload operations on a specific bucket",
Category: "Upload-Specific",
UseCase: "Large file upload services, streaming applications",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket for multipart uploads",
Required: true,
Example: "large-files-bucket",
},
},
Policy: t.GetMultipartUploadPolicy("${bucketName}"),
},
{
Name: "PresignedURLAccess",
Description: "Policy for generating and using presigned URLs",
Category: "Presigned URLs",
UseCase: "Frontend applications, temporary file sharing",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket for presigned URL access",
Required: true,
Example: "shared-files-bucket",
},
},
Policy: t.GetPresignedURLPolicy("${bucketName}"),
},
{
Name: "ContentTypeRestricted",
Description: "Restricts uploads to specific content types",
Category: "Content Control",
UseCase: "Image galleries, document repositories, media libraries",
Parameters: []PolicyTemplateParam{
{
Name: "bucketName",
Type: "string",
Description: "Name of the S3 bucket",
Required: true,
Example: "media-bucket",
},
{
Name: "allowedContentTypes",
Type: "array",
Description: "List of allowed MIME content types",
Required: true,
Example: "[\"image/jpeg\", \"image/png\", \"video/mp4\"]",
},
},
Policy: t.GetContentTypeRestrictedPolicy("${bucketName}", []string{"${allowedContentTypes}"}),
},
{
Name: "DenyDeleteAccess",
Description: "Allows all operations except delete (immutable storage)",
Category: "Data Protection",
UseCase: "Compliance storage, audit logs, backup retention",
Policy: t.GetDenyDeletePolicy(),
},
}
}
// GetPolicyTemplateByName returns a specific policy template by name
func (t *S3PolicyTemplates) GetPolicyTemplateByName(name string) *PolicyTemplateDefinition {
templates := t.GetAllPolicyTemplates()
for _, template := range templates {
if template.Name == name {
return &template
}
}
return nil
}
// GetPolicyTemplatesByCategory returns all policy templates in a specific category
func (t *S3PolicyTemplates) GetPolicyTemplatesByCategory(category string) []PolicyTemplateDefinition {
var result []PolicyTemplateDefinition
templates := t.GetAllPolicyTemplates()
for _, template := range templates {
if template.Category == category {
result = append(result, template)
}
}
return result
}

504
weed/s3api/s3_policy_templates_test.go
View File

@@ -1,504 +0,0 @@
package s3api
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestS3PolicyTemplates(t *testing.T) {
templates := NewS3PolicyTemplates()
t.Run("S3ReadOnlyPolicy", func(t *testing.T) {
policy := templates.GetS3ReadOnlyPolicy()
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "S3ReadOnlyAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:GetObject")
assert.Contains(t, stmt.Action, "s3:ListBucket")
assert.NotContains(t, stmt.Action, "s3:PutObject")
assert.NotContains(t, stmt.Action, "s3:DeleteObject")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*/*")
})
t.Run("S3WriteOnlyPolicy", func(t *testing.T) {
policy := templates.GetS3WriteOnlyPolicy()
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "S3WriteOnlyAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:PutObject")
assert.Contains(t, stmt.Action, "s3:CreateMultipartUpload")
assert.NotContains(t, stmt.Action, "s3:GetObject")
assert.NotContains(t, stmt.Action, "s3:DeleteObject")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*/*")
})
t.Run("S3AdminPolicy", func(t *testing.T) {
policy := templates.GetS3AdminPolicy()
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "S3FullAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:*")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*")
assert.Contains(t, stmt.Resource, "arn:aws:s3:::*/*")
})
}
func TestBucketSpecificPolicies(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "test-bucket"
t.Run("BucketSpecificReadPolicy", func(t *testing.T) {
policy := templates.GetBucketSpecificReadPolicy(bucketName)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "BucketSpecificReadAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:GetObject")
assert.Contains(t, stmt.Action, "s3:ListBucket")
assert.NotContains(t, stmt.Action, "s3:PutObject")
expectedBucketArn := "arn:aws:s3:::" + bucketName
expectedObjectArn := "arn:aws:s3:::" + bucketName + "/*"
assert.Contains(t, stmt.Resource, expectedBucketArn)
assert.Contains(t, stmt.Resource, expectedObjectArn)
})
t.Run("BucketSpecificWritePolicy", func(t *testing.T) {
policy := templates.GetBucketSpecificWritePolicy(bucketName)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "BucketSpecificWriteAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:PutObject")
assert.Contains(t, stmt.Action, "s3:CreateMultipartUpload")
assert.NotContains(t, stmt.Action, "s3:GetObject")
expectedBucketArn := "arn:aws:s3:::" + bucketName
expectedObjectArn := "arn:aws:s3:::" + bucketName + "/*"
assert.Contains(t, stmt.Resource, expectedBucketArn)
assert.Contains(t, stmt.Resource, expectedObjectArn)
})
}
func TestPathBasedAccessPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "shared-bucket"
pathPrefix := "user123/documents"
policy := templates.GetPathBasedAccessPolicy(bucketName, pathPrefix)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 2)
// First statement: List bucket with prefix condition
listStmt := policy.Statement[0]
assert.Equal(t, "Allow", listStmt.Effect)
assert.Equal(t, "ListBucketPermission", listStmt.Sid)
assert.Contains(t, listStmt.Action, "s3:ListBucket")
assert.Contains(t, listStmt.Resource, "arn:aws:s3:::"+bucketName)
assert.NotNil(t, listStmt.Condition)
// Second statement: Object operations on path
objectStmt := policy.Statement[1]
assert.Equal(t, "Allow", objectStmt.Effect)
assert.Equal(t, "PathBasedObjectAccess", objectStmt.Sid)
assert.Contains(t, objectStmt.Action, "s3:GetObject")
assert.Contains(t, objectStmt.Action, "s3:PutObject")
assert.Contains(t, objectStmt.Action, "s3:DeleteObject")
expectedObjectArn := "arn:aws:s3:::" + bucketName + "/" + pathPrefix + "/*"
assert.Contains(t, objectStmt.Resource, expectedObjectArn)
}
func TestIPRestrictedPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
allowedCIDRs := []string{"192.168.1.0/24", "10.0.0.0/8"}
policy := templates.GetIPRestrictedPolicy(allowedCIDRs)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "IPRestrictedS3Access", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:*")
assert.NotNil(t, stmt.Condition)
// Check IP condition structure
condition := stmt.Condition
ipAddress, exists := condition["IpAddress"]
assert.True(t, exists)
sourceIp, exists := ipAddress["aws:SourceIp"]
assert.True(t, exists)
assert.Equal(t, allowedCIDRs, sourceIp)
}
func TestTimeBasedAccessPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
startHour := 9 // 9 AM
endHour := 17 // 5 PM
policy := templates.GetTimeBasedAccessPolicy(startHour, endHour)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "TimeBasedS3Access", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:GetObject")
assert.Contains(t, stmt.Action, "s3:PutObject")
assert.Contains(t, stmt.Action, "s3:ListBucket")
assert.NotNil(t, stmt.Condition)
// Check time condition structure
condition := stmt.Condition
_, hasGreater := condition["DateGreaterThan"]
_, hasLess := condition["DateLessThan"]
assert.True(t, hasGreater)
assert.True(t, hasLess)
}
func TestMultipartUploadPolicyTemplate(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "large-files"
policy := templates.GetMultipartUploadPolicy(bucketName)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 2)
// First statement: Multipart operations
multipartStmt := policy.Statement[0]
assert.Equal(t, "Allow", multipartStmt.Effect)
assert.Equal(t, "MultipartUploadOperations", multipartStmt.Sid)
assert.Contains(t, multipartStmt.Action, "s3:CreateMultipartUpload")
assert.Contains(t, multipartStmt.Action, "s3:UploadPart")
assert.Contains(t, multipartStmt.Action, "s3:CompleteMultipartUpload")
assert.Contains(t, multipartStmt.Action, "s3:AbortMultipartUpload")
assert.Contains(t, multipartStmt.Action, "s3:ListMultipartUploads")
assert.Contains(t, multipartStmt.Action, "s3:ListParts")
expectedObjectArn := "arn:aws:s3:::" + bucketName + "/*"
assert.Contains(t, multipartStmt.Resource, expectedObjectArn)
// Second statement: List bucket
listStmt := policy.Statement[1]
assert.Equal(t, "Allow", listStmt.Effect)
assert.Equal(t, "ListBucketForMultipart", listStmt.Sid)
assert.Contains(t, listStmt.Action, "s3:ListBucket")
expectedBucketArn := "arn:aws:s3:::" + bucketName
assert.Contains(t, listStmt.Resource, expectedBucketArn)
}
func TestPresignedURLPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "shared-files"
policy := templates.GetPresignedURLPolicy(bucketName)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "PresignedURLAccess", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:GetObject")
assert.Contains(t, stmt.Action, "s3:PutObject")
assert.NotNil(t, stmt.Condition)
expectedObjectArn := "arn:aws:s3:::" + bucketName + "/*"
assert.Contains(t, stmt.Resource, expectedObjectArn)
// Check signature version condition
condition := stmt.Condition
stringEquals, exists := condition["StringEquals"]
assert.True(t, exists)
signatureVersion, exists := stringEquals["s3:x-amz-signature-version"]
assert.True(t, exists)
assert.Equal(t, "AWS4-HMAC-SHA256", signatureVersion)
}
func TestTemporaryAccessPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "temp-bucket"
expirationHours := 24
policy := templates.GetTemporaryAccessPolicy(bucketName, expirationHours)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 1)
stmt := policy.Statement[0]
assert.Equal(t, "Allow", stmt.Effect)
assert.Equal(t, "TemporaryS3Access", stmt.Sid)
assert.Contains(t, stmt.Action, "s3:GetObject")
assert.Contains(t, stmt.Action, "s3:PutObject")
assert.Contains(t, stmt.Action, "s3:ListBucket")
assert.NotNil(t, stmt.Condition)
// Check expiration condition
condition := stmt.Condition
dateLessThan, exists := condition["DateLessThan"]
assert.True(t, exists)
currentTime, exists := dateLessThan["aws:CurrentTime"]
assert.True(t, exists)
assert.IsType(t, "", currentTime) // Should be a string timestamp
}
func TestContentTypeRestrictedPolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
bucketName := "media-bucket"
allowedTypes := []string{"image/jpeg", "image/png", "video/mp4"}
policy := templates.GetContentTypeRestrictedPolicy(bucketName, allowedTypes)
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 2)
// First statement: Upload with content type restriction
uploadStmt := policy.Statement[0]
assert.Equal(t, "Allow", uploadStmt.Effect)
assert.Equal(t, "ContentTypeRestrictedUpload", uploadStmt.Sid)
assert.Contains(t, uploadStmt.Action, "s3:PutObject")
assert.Contains(t, uploadStmt.Action, "s3:CreateMultipartUpload")
assert.NotNil(t, uploadStmt.Condition)
// Check content type condition
condition := uploadStmt.Condition
stringEquals, exists := condition["StringEquals"]
assert.True(t, exists)
contentType, exists := stringEquals["s3:content-type"]
assert.True(t, exists)
assert.Equal(t, allowedTypes, contentType)
// Second statement: Read access without restrictions
readStmt := policy.Statement[1]
assert.Equal(t, "Allow", readStmt.Effect)
assert.Equal(t, "ReadAccess", readStmt.Sid)
assert.Contains(t, readStmt.Action, "s3:GetObject")
assert.Contains(t, readStmt.Action, "s3:ListBucket")
assert.Nil(t, readStmt.Condition) // No conditions for read access
}
func TestDenyDeletePolicy(t *testing.T) {
templates := NewS3PolicyTemplates()
policy := templates.GetDenyDeletePolicy()
require.NotNil(t, policy)
assert.Equal(t, "2012-10-17", policy.Version)
assert.Len(t, policy.Statement, 2)
// First statement: Allow everything except delete
allowStmt := policy.Statement[0]
assert.Equal(t, "Allow", allowStmt.Effect)
assert.Equal(t, "AllowAllExceptDelete", allowStmt.Sid)
assert.Contains(t, allowStmt.Action, "s3:GetObject")
assert.Contains(t, allowStmt.Action, "s3:PutObject")
assert.Contains(t, allowStmt.Action, "s3:ListBucket")
assert.NotContains(t, allowStmt.Action, "s3:DeleteObject")
assert.NotContains(t, allowStmt.Action, "s3:DeleteBucket")
// Second statement: Explicitly deny delete operations
denyStmt := policy.Statement[1]
assert.Equal(t, "Deny", denyStmt.Effect)
assert.Equal(t, "DenyDeleteOperations", denyStmt.Sid)
assert.Contains(t, denyStmt.Action, "s3:DeleteObject")
assert.Contains(t, denyStmt.Action, "s3:DeleteObjectVersion")
assert.Contains(t, denyStmt.Action, "s3:DeleteBucket")
}
func TestPolicyTemplateMetadata(t *testing.T) {
templates := NewS3PolicyTemplates()
t.Run("GetAllPolicyTemplates", func(t *testing.T) {
allTemplates := templates.GetAllPolicyTemplates()
assert.Greater(t, len(allTemplates), 10) // Should have many templates
// Check that each template has required fields
for _, template := range allTemplates {
assert.NotEmpty(t, template.Name)
assert.NotEmpty(t, template.Description)
assert.NotEmpty(t, template.Category)
assert.NotEmpty(t, template.UseCase)
assert.NotNil(t, template.Policy)
assert.Equal(t, "2012-10-17", template.Policy.Version)
}
})
t.Run("GetPolicyTemplateByName", func(t *testing.T) {
// Test existing template
template := templates.GetPolicyTemplateByName("S3ReadOnlyAccess")
require.NotNil(t, template)
assert.Equal(t, "S3ReadOnlyAccess", template.Name)
assert.Equal(t, "Basic Access", template.Category)
// Test non-existing template
nonExistent := templates.GetPolicyTemplateByName("NonExistentTemplate")
assert.Nil(t, nonExistent)
})
t.Run("GetPolicyTemplatesByCategory", func(t *testing.T) {
basicAccessTemplates := templates.GetPolicyTemplatesByCategory("Basic Access")
assert.GreaterOrEqual(t, len(basicAccessTemplates), 2)
for _, template := range basicAccessTemplates {
assert.Equal(t, "Basic Access", template.Category)
}
// Test non-existing category
emptyCategory := templates.GetPolicyTemplatesByCategory("NonExistentCategory")
assert.Empty(t, emptyCategory)
})
t.Run("PolicyTemplateParameters", func(t *testing.T) {
allTemplates := templates.GetAllPolicyTemplates()
// Find a template with parameters (like BucketSpecificRead)
var templateWithParams *PolicyTemplateDefinition
for _, template := range allTemplates {
if template.Name == "BucketSpecificRead" {
templateWithParams = &template
break
}
}
require.NotNil(t, templateWithParams)
assert.Greater(t, len(templateWithParams.Parameters), 0)
param := templateWithParams.Parameters[0]
assert.Equal(t, "bucketName", param.Name)
assert.Equal(t, "string", param.Type)
assert.True(t, param.Required)
assert.NotEmpty(t, param.Description)
assert.NotEmpty(t, param.Example)
})
}
func TestFormatHourHelper(t *testing.T) {
tests := []struct {
hour int
expected string
}{
{0, "00"},
{5, "05"},
{9, "09"},
{10, "10"},
{15, "15"},
{23, "23"},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("Hour_%d", tt.hour), func(t *testing.T) {
result := formatHour(tt.hour)
assert.Equal(t, tt.expected, result)
})
}
}
func TestPolicyTemplateCategories(t *testing.T) {
templates := NewS3PolicyTemplates()
allTemplates := templates.GetAllPolicyTemplates()
// Extract all categories
categoryMap := make(map[string]int)
for _, template := range allTemplates {
categoryMap[template.Category]++
}
// Expected categories
expectedCategories := []string{
"Basic Access",
"Administrative",
"Bucket-Specific",
"Path-Restricted",
"Security",
"Upload-Specific",
"Presigned URLs",
"Content Control",
"Data Protection",
}
for _, expectedCategory := range expectedCategories {
count, exists := categoryMap[expectedCategory]
assert.True(t, exists, "Category %s should exist", expectedCategory)
assert.Greater(t, count, 0, "Category %s should have at least one template", expectedCategory)
}
}
func TestPolicyValidation(t *testing.T) {
templates := NewS3PolicyTemplates()
allTemplates := templates.GetAllPolicyTemplates()
// Test that all policies have valid structure
for _, template := range allTemplates {
t.Run("Policy_"+template.Name, func(t *testing.T) {
policy := template.Policy
// Basic validation
assert.Equal(t, "2012-10-17", policy.Version)
assert.Greater(t, len(policy.Statement), 0)
// Validate each statement
for i, stmt := range policy.Statement {
assert.NotEmpty(t, stmt.Effect, "Statement %d should have effect", i)
assert.Contains(t, []string{"Allow", "Deny"}, stmt.Effect, "Statement %d effect should be Allow or Deny", i)
assert.Greater(t, len(stmt.Action), 0, "Statement %d should have actions", i)
assert.Greater(t, len(stmt.Resource), 0, "Statement %d should have resources", i)
// Check resource format
for _, resource := range stmt.Resource {
if resource != "*" {
assert.Contains(t, resource, "arn:aws:s3:::", "Resource should be valid AWS S3 ARN: %s", resource)
}
}
}
})
}
}

355
weed/s3api/s3_presigned_url_iam.go
View File

@@ -1,355 +0,0 @@
package s3api
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// S3PresignedURLManager handles IAM integration for presigned URLs
type S3PresignedURLManager struct {
s3iam *S3IAMIntegration
}
// NewS3PresignedURLManager creates a new presigned URL manager with IAM integration
func NewS3PresignedURLManager(s3iam *S3IAMIntegration) *S3PresignedURLManager {
return &S3PresignedURLManager{
s3iam: s3iam,
}
}
// PresignedURLRequest represents a request to generate a presigned URL
type PresignedURLRequest struct {
Method string `json:"method"` // HTTP method (GET, PUT, POST, DELETE)
Bucket string `json:"bucket"` // S3 bucket name
ObjectKey string `json:"object_key"` // S3 object key
Expiration time.Duration `json:"expiration"` // URL expiration duration
SessionToken string `json:"session_token"` // JWT session token for IAM
Headers map[string]string `json:"headers"` // Additional headers to sign
QueryParams map[string]string `json:"query_params"` // Additional query parameters
}
// PresignedURLResponse represents the generated presigned URL
type PresignedURLResponse struct {
URL string `json:"url"` // The presigned URL
Method string `json:"method"` // HTTP method
Headers map[string]string `json:"headers"` // Required headers
ExpiresAt time.Time `json:"expires_at"` // URL expiration time
SignedHeaders []string `json:"signed_headers"` // List of signed headers
CanonicalQuery string `json:"canonical_query"` // Canonical query string
}
// ValidatePresignedURLWithIAM validates a presigned URL request using IAM policies
func (iam *IdentityAccessManagement) ValidatePresignedURLWithIAM(r *http.Request, identity *Identity) s3err.ErrorCode {
if iam.iamIntegration == nil {
// Fall back to standard validation
return s3err.ErrNone
}
// Extract bucket and object from request
bucket, object := s3_constants.GetBucketAndObject(r)
// Determine the S3 action from HTTP method and path
action := determineS3ActionFromRequest(r, bucket, object)
// Check if the user has permission for this action
ctx := r.Context()
sessionToken := extractSessionTokenFromPresignedURL(r)
if sessionToken == "" {
// No session token in presigned URL - use standard auth
return s3err.ErrNone
}
// Create a temporary cloned request with Authorization header to reuse the secure AuthenticateJWT logic
// This ensures we use the same robust validation (STS vs OIDC, signature verification, etc.)
// as standard requests, preventing security regressions.
authReq := r.Clone(ctx)
authReq.Header.Set("Authorization", "Bearer "+sessionToken)
// Authenticate the token using the centralized IAM integration
iamIdentity, errCode := iam.iamIntegration.AuthenticateJWT(ctx, authReq)
if errCode != s3err.ErrNone {
glog.V(3).Infof("JWT authentication failed for presigned URL: %v", errCode)
return errCode
}
// Authorize using IAM
errCode = iam.iamIntegration.AuthorizeAction(ctx, iamIdentity, action, bucket, object, r)
if errCode != s3err.ErrNone {
glog.V(3).Infof("IAM authorization failed for presigned URL: principal=%s action=%s bucket=%s object=%s",
iamIdentity.Principal, action, bucket, object)
return errCode
}
glog.V(3).Infof("IAM authorization succeeded for presigned URL: principal=%s action=%s bucket=%s object=%s",
iamIdentity.Principal, action, bucket, object)
return s3err.ErrNone
}
// GeneratePresignedURLWithIAM generates a presigned URL with IAM policy validation
func (pm *S3PresignedURLManager) GeneratePresignedURLWithIAM(ctx context.Context, req *PresignedURLRequest, baseURL string) (*PresignedURLResponse, error) {
if pm.s3iam == nil || !pm.s3iam.enabled {
return nil, fmt.Errorf("IAM integration not enabled")
}
if req == nil || strings.TrimSpace(req.SessionToken) == "" {
return nil, fmt.Errorf("IAM authorization failed: session token is required")
}
authRequest := &http.Request{
Method: req.Method,
URL: &url.URL{Path: "/" + req.Bucket + "/" + req.ObjectKey},
Header: make(http.Header),
}
authRequest.Header.Set("Authorization", "Bearer "+req.SessionToken)
authRequest = authRequest.WithContext(ctx)
iamIdentity, errCode := pm.s3iam.AuthenticateJWT(ctx, authRequest)
if errCode != s3err.ErrNone {
return nil, fmt.Errorf("IAM authorization failed: invalid session token")
}
// Determine S3 action from method
action := determineS3ActionFromMethodAndPath(req.Method, req.Bucket, req.ObjectKey)
// Check IAM permissions before generating URL
errCode = pm.s3iam.AuthorizeAction(ctx, iamIdentity, action, req.Bucket, req.ObjectKey, authRequest)
if errCode != s3err.ErrNone {
return nil, fmt.Errorf("IAM authorization failed: user does not have permission for action %s on resource %s/%s", action, req.Bucket, req.ObjectKey)
}
// Generate presigned URL with validated permissions
return pm.generatePresignedURL(req, baseURL, iamIdentity)
}
// generatePresignedURL creates the actual presigned URL
func (pm *S3PresignedURLManager) generatePresignedURL(req *PresignedURLRequest, baseURL string, identity *IAMIdentity) (*PresignedURLResponse, error) {
// Calculate expiration time
expiresAt := time.Now().Add(req.Expiration)
// Build the base URL
urlPath := "/" + req.Bucket
if req.ObjectKey != "" {
urlPath += "/" + req.ObjectKey
}
// Create query parameters for AWS signature v4
queryParams := make(map[string]string)
for k, v := range req.QueryParams {
queryParams[k] = v
}
// Add AWS signature v4 parameters
queryParams["X-Amz-Algorithm"] = "AWS4-HMAC-SHA256"
queryParams["X-Amz-Credential"] = fmt.Sprintf("seaweedfs/%s/us-east-1/s3/aws4_request", expiresAt.Format("20060102"))
queryParams["X-Amz-Date"] = expiresAt.Format("20060102T150405Z")
queryParams["X-Amz-Expires"] = strconv.Itoa(int(req.Expiration.Seconds()))
queryParams["X-Amz-SignedHeaders"] = "host"
// Add session token if available
if identity.SessionToken != "" {
queryParams["X-Amz-Security-Token"] = identity.SessionToken
}
// Build canonical query string
canonicalQuery := buildCanonicalQuery(queryParams)
// For now, we'll create a mock signature
// In production, this would use proper AWS signature v4 signing
mockSignature := generateMockSignature(req.Method, urlPath, canonicalQuery, identity.SessionToken)
queryParams["X-Amz-Signature"] = mockSignature
// Build final URL
finalQuery := buildCanonicalQuery(queryParams)
fullURL := baseURL + urlPath + "?" + finalQuery
// Prepare response
headers := make(map[string]string)
for k, v := range req.Headers {
headers[k] = v
}
return &PresignedURLResponse{
URL: fullURL,
Method: req.Method,
Headers: headers,
ExpiresAt: expiresAt,
SignedHeaders: []string{"host"},
CanonicalQuery: canonicalQuery,
}, nil
}
// Helper functions
// determineS3ActionFromRequest determines the S3 action based on HTTP request
func determineS3ActionFromRequest(r *http.Request, bucket, object string) Action {
return determineS3ActionFromMethodAndPath(r.Method, bucket, object)
}
// determineS3ActionFromMethodAndPath determines the S3 action based on method and path
func determineS3ActionFromMethodAndPath(method, bucket, object string) Action {
switch method {
case "GET":
if object == "" {
return s3_constants.ACTION_LIST // ListBucket
} else {
return s3_constants.ACTION_READ // GetObject
}
case "PUT", "POST":
return s3_constants.ACTION_WRITE // PutObject
case "DELETE":
if object == "" {
return s3_constants.ACTION_DELETE_BUCKET // DeleteBucket
} else {
return s3_constants.ACTION_WRITE // DeleteObject (uses WRITE action)
}
case "HEAD":
if object == "" {
return s3_constants.ACTION_LIST // HeadBucket
} else {
return s3_constants.ACTION_READ // HeadObject
}
default:
return s3_constants.ACTION_READ // Default to read
}
}
// extractSessionTokenFromPresignedURL extracts session token from presigned URL query parameters
func extractSessionTokenFromPresignedURL(r *http.Request) string {
// Check for X-Amz-Security-Token in query parameters
if token := r.URL.Query().Get("X-Amz-Security-Token"); token != "" {
return token
}
// Check for session token in other possible locations
if token := r.URL.Query().Get("SessionToken"); token != "" {
return token
}
return ""
}
// buildCanonicalQuery builds a canonical query string for AWS signature
func buildCanonicalQuery(params map[string]string) string {
var keys []string
for k := range params {
keys = append(keys, k)
}
// Sort keys for canonical order
for i := 0; i < len(keys); i++ {
for j := i + 1; j < len(keys); j++ {
if keys[i] > keys[j] {
keys[i], keys[j] = keys[j], keys[i]
}
}
}
var parts []string
for _, k := range keys {
parts = append(parts, fmt.Sprintf("%s=%s", url.QueryEscape(k), url.QueryEscape(params[k])))
}
return strings.Join(parts, "&")
}
// generateMockSignature generates a mock signature for testing purposes
func generateMockSignature(method, path, query, sessionToken string) string {
// This is a simplified signature for demonstration
// In production, use proper AWS signature v4 calculation
data := fmt.Sprintf("%s\n%s\n%s\n%s", method, path, query, sessionToken)
hash := sha256.Sum256([]byte(data))
return hex.EncodeToString(hash[:])[:16] // Truncate for readability
}
// ValidatePresignedURLExpiration validates that a presigned URL hasn't expired
func ValidatePresignedURLExpiration(r *http.Request) error {
query := r.URL.Query()
// Get X-Amz-Date and X-Amz-Expires
dateStr := query.Get("X-Amz-Date")
expiresStr := query.Get("X-Amz-Expires")
if dateStr == "" || expiresStr == "" {
return fmt.Errorf("missing required presigned URL parameters")
}
// Parse date (always in UTC)
signedDate, err := time.Parse("20060102T150405Z", dateStr)
if err != nil {
return fmt.Errorf("invalid X-Amz-Date format: %v", err)
}
// Parse expires
expires, err := strconv.Atoi(expiresStr)
if err != nil {
return fmt.Errorf("invalid X-Amz-Expires format: %v", err)
}
// Check expiration - compare in UTC
expirationTime := signedDate.Add(time.Duration(expires) * time.Second)
now := time.Now().UTC()
if now.After(expirationTime) {
return fmt.Errorf("presigned URL has expired")
}
return nil
}
// PresignedURLSecurityPolicy represents security constraints for presigned URL generation
type PresignedURLSecurityPolicy struct {
MaxExpirationDuration time.Duration `json:"max_expiration_duration"` // Maximum allowed expiration
AllowedMethods []string `json:"allowed_methods"` // Allowed HTTP methods
RequiredHeaders []string `json:"required_headers"` // Headers that must be present
IPWhitelist []string `json:"ip_whitelist"` // Allowed IP addresses/ranges
MaxFileSize int64 `json:"max_file_size"` // Maximum file size for uploads
}
// DefaultPresignedURLSecurityPolicy returns a default security policy
func DefaultPresignedURLSecurityPolicy() *PresignedURLSecurityPolicy {
return &PresignedURLSecurityPolicy{
MaxExpirationDuration: 7 * 24 * time.Hour, // 7 days max
AllowedMethods: []string{"GET", "PUT", "POST", "HEAD"},
RequiredHeaders: []string{},
IPWhitelist: []string{}, // Empty means no IP restrictions
MaxFileSize: 5 * 1024 * 1024 * 1024, // 5GB default
}
}
// ValidatePresignedURLRequest validates a presigned URL request against security policy
func (policy *PresignedURLSecurityPolicy) ValidatePresignedURLRequest(req *PresignedURLRequest) error {
// Check expiration duration
if req.Expiration > policy.MaxExpirationDuration {
return fmt.Errorf("expiration duration %v exceeds maximum allowed %v", req.Expiration, policy.MaxExpirationDuration)
}
// Check HTTP method
methodAllowed := false
for _, allowedMethod := range policy.AllowedMethods {
if req.Method == allowedMethod {
methodAllowed = true
break
}
}
if !methodAllowed {
return fmt.Errorf("HTTP method %s is not allowed", req.Method)
}
// Check required headers
for _, requiredHeader := range policy.RequiredHeaders {
if _, exists := req.Headers[requiredHeader]; !exists {
return fmt.Errorf("required header %s is missing", requiredHeader)
}
}
return nil
}

631
weed/s3api/s3_presigned_url_iam_test.go
View File

@@ -1,631 +0,0 @@
package s3api
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/golang-jwt/jwt/v5"
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/iam/ldap"
"github.com/seaweedfs/seaweedfs/weed/iam/oidc"
"github.com/seaweedfs/seaweedfs/weed/iam/policy"
"github.com/seaweedfs/seaweedfs/weed/iam/sts"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// createTestJWTPresigned creates a test JWT token with the specified issuer, subject and signing key
func createTestJWTPresigned(t *testing.T, issuer, subject, signingKey string) string {
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"iss": issuer,
"sub": subject,
"aud": "test-client-id",
"exp": time.Now().Add(time.Hour).Unix(),
"iat": time.Now().Unix(),
// Add claims that trust policy validation expects
"idp": "test-oidc", // Identity provider claim for trust policy matching
})
tokenString, err := token.SignedString([]byte(signingKey))
require.NoError(t, err)
return tokenString
}
// TestPresignedURLIAMValidation tests IAM validation for presigned URLs
func TestPresignedURLIAMValidation(t *testing.T) {
// Set up IAM system
iamManager := setupTestIAMManagerForPresigned(t)
s3iam := NewS3IAMIntegration(iamManager, "localhost:8888")
// Create IAM with integration
iam := &IdentityAccessManagement{
isAuthEnabled: true,
}
iam.SetIAMIntegration(s3iam)
// Set up roles
ctx := context.Background()
setupTestRolesForPresigned(ctx, iamManager)
// Create a valid JWT token for testing
validJWTToken := createTestJWTPresigned(t, "https://test-issuer.com", "test-user-123", "test-signing-key")
// Get session token
response, err := iamManager.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
RoleArn: "arn:aws:iam::role/S3ReadOnlyRole",
WebIdentityToken: validJWTToken,
RoleSessionName: "presigned-test-session",
})
require.NoError(t, err)
sessionToken := response.Credentials.SessionToken
tests := []struct {
name string
method string
path string
sessionToken string
expectedResult s3err.ErrorCode
}{
{
name: "GET object with read permissions",
method: "GET",
path: "/test-bucket/test-file.txt",
sessionToken: sessionToken,
expectedResult: s3err.ErrNone,
},
{
name: "PUT object with read-only permissions (should fail)",
method: "PUT",
path: "/test-bucket/new-file.txt",
sessionToken: sessionToken,
expectedResult: s3err.ErrAccessDenied,
},
{
name: "GET object without session token",
method: "GET",
path: "/test-bucket/test-file.txt",
sessionToken: "",
expectedResult: s3err.ErrNone, // Falls back to standard auth
},
{
name: "Invalid session token",
method: "GET",
path: "/test-bucket/test-file.txt",
sessionToken: "invalid-token",
expectedResult: s3err.ErrAccessDenied,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create request with presigned URL parameters
req := createPresignedURLRequest(t, tt.method, tt.path, tt.sessionToken)
// Create identity for testing
identity := &Identity{
Name: "test-user",
Account: &AccountAdmin,
}
// Test validation
result := iam.ValidatePresignedURLWithIAM(req, identity)
assert.Equal(t, tt.expectedResult, result, "IAM validation result should match expected")
})
}
}
// TestPresignedURLGeneration tests IAM-aware presigned URL generation
func TestPresignedURLGeneration(t *testing.T) {
// Set up IAM system
iamManager := setupTestIAMManagerForPresigned(t)
s3iam := NewS3IAMIntegration(iamManager, "localhost:8888")
s3iam.enabled = true // Enable IAM integration
presignedManager := NewS3PresignedURLManager(s3iam)
ctx := context.Background()
setupTestRolesForPresigned(ctx, iamManager)
// Create a valid JWT token for testing
validJWTToken := createTestJWTPresigned(t, "https://test-issuer.com", "test-user-123", "test-signing-key")
// Get session token
response, err := iamManager.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
RoleArn: "arn:aws:iam::role/S3AdminRole",
WebIdentityToken: validJWTToken,
RoleSessionName: "presigned-gen-test-session",
})
require.NoError(t, err)
sessionToken := response.Credentials.SessionToken
tests := []struct {
name string
request *PresignedURLRequest
shouldSucceed bool
expectedError string
}{
{
name: "Generate valid presigned GET URL",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: time.Hour,
SessionToken: sessionToken,
},
shouldSucceed: true,
},
{
name: "Generate valid presigned PUT URL",
request: &PresignedURLRequest{
Method: "PUT",
Bucket: "test-bucket",
ObjectKey: "new-file.txt",
Expiration: time.Hour,
SessionToken: sessionToken,
},
shouldSucceed: true,
},
{
name: "Generate URL with invalid session token",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: time.Hour,
SessionToken: "invalid-token",
},
shouldSucceed: false,
expectedError: "IAM authorization failed",
},
{
name: "Generate URL without session token",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: time.Hour,
},
shouldSucceed: false,
expectedError: "IAM authorization failed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
response, err := presignedManager.GeneratePresignedURLWithIAM(ctx, tt.request, "http://localhost:8333")
if tt.shouldSucceed {
assert.NoError(t, err, "Presigned URL generation should succeed")
if response != nil {
assert.NotEmpty(t, response.URL, "URL should not be empty")
assert.Equal(t, tt.request.Method, response.Method, "Method should match")
assert.True(t, response.ExpiresAt.After(time.Now()), "URL should not be expired")
} else {
t.Errorf("Response should not be nil when generation should succeed")
}
} else {
assert.Error(t, err, "Presigned URL generation should fail")
if tt.expectedError != "" {
assert.Contains(t, err.Error(), tt.expectedError, "Error message should contain expected text")
}
}
})
}
}
func TestPresignedURLGenerationUsesAuthenticatedPrincipal(t *testing.T) {
iamManager := setupTestIAMManagerForPresigned(t)
s3iam := NewS3IAMIntegration(iamManager, "localhost:8888")
s3iam.enabled = true
presignedManager := NewS3PresignedURLManager(s3iam)
ctx := context.Background()
setupTestRolesForPresigned(ctx, iamManager)
validJWTToken := createTestJWTPresigned(t, "https://test-issuer.com", "test-user-123", "test-signing-key")
response, err := iamManager.AssumeRoleWithWebIdentity(ctx, &sts.AssumeRoleWithWebIdentityRequest{
RoleArn: "arn:aws:iam::role/S3ReadOnlyRole",
WebIdentityToken: validJWTToken,
RoleSessionName: "presigned-read-only-session",
})
require.NoError(t, err)
_, err = presignedManager.GeneratePresignedURLWithIAM(ctx, &PresignedURLRequest{
Method: "PUT",
Bucket: "test-bucket",
ObjectKey: "new-file.txt",
Expiration: time.Hour,
SessionToken: response.Credentials.SessionToken,
}, "http://localhost:8333")
require.Error(t, err)
assert.Contains(t, err.Error(), "IAM authorization failed")
}
// TestPresignedURLExpiration tests URL expiration validation
func TestPresignedURLExpiration(t *testing.T) {
tests := []struct {
name string
setupRequest func() *http.Request
expectedError string
}{
{
name: "Valid non-expired URL",
setupRequest: func() *http.Request {
req := httptest.NewRequest("GET", "/test-bucket/test-file.txt", nil)
q := req.URL.Query()
// Set date to 30 minutes ago with 2 hours expiration for safe margin
q.Set("X-Amz-Date", time.Now().UTC().Add(-30*time.Minute).Format("20060102T150405Z"))
q.Set("X-Amz-Expires", "7200") // 2 hours
req.URL.RawQuery = q.Encode()
return req
},
expectedError: "",
},
{
name: "Expired URL",
setupRequest: func() *http.Request {
req := httptest.NewRequest("GET", "/test-bucket/test-file.txt", nil)
q := req.URL.Query()
// Set date to 2 hours ago with 1 hour expiration
q.Set("X-Amz-Date", time.Now().UTC().Add(-2*time.Hour).Format("20060102T150405Z"))
q.Set("X-Amz-Expires", "3600") // 1 hour
req.URL.RawQuery = q.Encode()
return req
},
expectedError: "presigned URL has expired",
},
{
name: "Missing date parameter",
setupRequest: func() *http.Request {
req := httptest.NewRequest("GET", "/test-bucket/test-file.txt", nil)
q := req.URL.Query()
q.Set("X-Amz-Expires", "3600")
req.URL.RawQuery = q.Encode()
return req
},
expectedError: "missing required presigned URL parameters",
},
{
name: "Invalid date format",
setupRequest: func() *http.Request {
req := httptest.NewRequest("GET", "/test-bucket/test-file.txt", nil)
q := req.URL.Query()
q.Set("X-Amz-Date", "invalid-date")
q.Set("X-Amz-Expires", "3600")
req.URL.RawQuery = q.Encode()
return req
},
expectedError: "invalid X-Amz-Date format",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := tt.setupRequest()
err := ValidatePresignedURLExpiration(req)
if tt.expectedError == "" {
assert.NoError(t, err, "Validation should succeed")
} else {
assert.Error(t, err, "Validation should fail")
assert.Contains(t, err.Error(), tt.expectedError, "Error message should contain expected text")
}
})
}
}
// TestPresignedURLSecurityPolicy tests security policy enforcement
func TestPresignedURLSecurityPolicy(t *testing.T) {
policy := &PresignedURLSecurityPolicy{
MaxExpirationDuration: 24 * time.Hour,
AllowedMethods: []string{"GET", "PUT"},
RequiredHeaders: []string{"Content-Type"},
MaxFileSize: 1024 * 1024, // 1MB
}
tests := []struct {
name string
request *PresignedURLRequest
expectedError string
}{
{
name: "Valid request",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: 12 * time.Hour,
Headers: map[string]string{"Content-Type": "application/json"},
},
expectedError: "",
},
{
name: "Expiration too long",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: 48 * time.Hour, // Exceeds 24h limit
Headers: map[string]string{"Content-Type": "application/json"},
},
expectedError: "expiration duration",
},
{
name: "Method not allowed",
request: &PresignedURLRequest{
Method: "DELETE", // Not in allowed methods
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: 12 * time.Hour,
Headers: map[string]string{"Content-Type": "application/json"},
},
expectedError: "HTTP method DELETE is not allowed",
},
{
name: "Missing required header",
request: &PresignedURLRequest{
Method: "GET",
Bucket: "test-bucket",
ObjectKey: "test-file.txt",
Expiration: 12 * time.Hour,
Headers: map[string]string{}, // Missing Content-Type
},
expectedError: "required header Content-Type is missing",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := policy.ValidatePresignedURLRequest(tt.request)
if tt.expectedError == "" {
assert.NoError(t, err, "Policy validation should succeed")
} else {
assert.Error(t, err, "Policy validation should fail")
assert.Contains(t, err.Error(), tt.expectedError, "Error message should contain expected text")
}
})
}
}
// TestS3ActionDetermination tests action determination from HTTP methods
func TestS3ActionDetermination(t *testing.T) {
tests := []struct {
name string
method string
bucket string
object string
expectedAction Action
}{
{
name: "GET object",
method: "GET",
bucket: "test-bucket",
object: "test-file.txt",
expectedAction: s3_constants.ACTION_READ,
},
{
name: "GET bucket (list)",
method: "GET",
bucket: "test-bucket",
object: "",
expectedAction: s3_constants.ACTION_LIST,
},
{
name: "PUT object",
method: "PUT",
bucket: "test-bucket",
object: "new-file.txt",
expectedAction: s3_constants.ACTION_WRITE,
},
{
name: "DELETE object",
method: "DELETE",
bucket: "test-bucket",
object: "old-file.txt",
expectedAction: s3_constants.ACTION_WRITE,
},
{
name: "DELETE bucket",
method: "DELETE",
bucket: "test-bucket",
object: "",
expectedAction: s3_constants.ACTION_DELETE_BUCKET,
},
{
name: "HEAD object",
method: "HEAD",
bucket: "test-bucket",
object: "test-file.txt",
expectedAction: s3_constants.ACTION_READ,
},
{
name: "POST object",
method: "POST",
bucket: "test-bucket",
object: "upload-file.txt",
expectedAction: s3_constants.ACTION_WRITE,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
action := determineS3ActionFromMethodAndPath(tt.method, tt.bucket, tt.object)
assert.Equal(t, tt.expectedAction, action, "S3 action should match expected")
})
}
}
// Helper functions for tests
func setupTestIAMManagerForPresigned(t *testing.T) *integration.IAMManager {
// Create IAM manager
manager := integration.NewIAMManager()
// Initialize with test configuration
config := &integration.IAMConfig{
STS: &sts.STSConfig{
TokenDuration: sts.FlexibleDuration{Duration: time.Hour},
MaxSessionLength: sts.FlexibleDuration{Duration: time.Hour * 12},
Issuer: "test-sts",
SigningKey: []byte("test-signing-key-32-characters-long"),
},
Policy: &policy.PolicyEngineConfig{
DefaultEffect: "Deny",
StoreType: "memory",
},
Roles: &integration.RoleStoreConfig{
StoreType: "memory",
},
}
err := manager.Initialize(config, func() string {
return "localhost:8888" // Mock filer address for testing
})
require.NoError(t, err)
// Set up test identity providers
setupTestProvidersForPresigned(t, manager)
return manager
}
func setupTestProvidersForPresigned(t *testing.T, manager *integration.IAMManager) {
// Set up OIDC provider
oidcProvider := oidc.NewMockOIDCProvider("test-oidc")
oidcConfig := &oidc.OIDCConfig{
Issuer: "https://test-issuer.com",
ClientID: "test-client-id",
}
err := oidcProvider.Initialize(oidcConfig)
require.NoError(t, err)
oidcProvider.SetupDefaultTestData()
// Set up LDAP provider
ldapProvider := ldap.NewMockLDAPProvider("test-ldap")
err = ldapProvider.Initialize(nil) // Mock doesn't need real config
require.NoError(t, err)
ldapProvider.SetupDefaultTestData()
// Register providers
err = manager.RegisterIdentityProvider(oidcProvider)
require.NoError(t, err)
err = manager.RegisterIdentityProvider(ldapProvider)
require.NoError(t, err)
}
func setupTestRolesForPresigned(ctx context.Context, manager *integration.IAMManager) {
// Create read-only policy
readOnlyPolicy := &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "AllowS3ReadOperations",
Effect: "Allow",
Action: []string{"s3:GetObject", "s3:ListBucket", "s3:HeadObject"},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
manager.CreatePolicy(ctx, "", "S3ReadOnlyPolicy", readOnlyPolicy)
// Create read-only role
manager.CreateRole(ctx, "", "S3ReadOnlyRole", &integration.RoleDefinition{
RoleName: "S3ReadOnlyRole",
TrustPolicy: &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Principal: map[string]interface{}{
"Federated": "test-oidc",
},
Action: []string{"sts:AssumeRoleWithWebIdentity"},
},
},
},
AttachedPolicies: []string{"S3ReadOnlyPolicy"},
})
// Create admin policy
adminPolicy := &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Sid: "AllowAllS3Operations",
Effect: "Allow",
Action: []string{"s3:*"},
Resource: []string{
"arn:aws:s3:::*",
"arn:aws:s3:::*/*",
},
},
},
}
manager.CreatePolicy(ctx, "", "S3AdminPolicy", adminPolicy)
// Create admin role
manager.CreateRole(ctx, "", "S3AdminRole", &integration.RoleDefinition{
RoleName: "S3AdminRole",
TrustPolicy: &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Principal: map[string]interface{}{
"Federated": "test-oidc",
},
Action: []string{"sts:AssumeRoleWithWebIdentity"},
},
},
},
AttachedPolicies: []string{"S3AdminPolicy"},
})
// Create a role for presigned URL users with admin permissions for testing
manager.CreateRole(ctx, "", "PresignedUser", &integration.RoleDefinition{
RoleName: "PresignedUser",
TrustPolicy: &policy.PolicyDocument{
Version: "2012-10-17",
Statement: []policy.Statement{
{
Effect: "Allow",
Principal: map[string]interface{}{
"Federated": "test-oidc",
},
Action: []string{"sts:AssumeRoleWithWebIdentity"},
},
},
},
AttachedPolicies: []string{"S3AdminPolicy"}, // Use admin policy for testing
})
}
func createPresignedURLRequest(t *testing.T, method, path, sessionToken string) *http.Request {
req := httptest.NewRequest(method, path, nil)
// Add presigned URL parameters if session token is provided
if sessionToken != "" {
q := req.URL.Query()
q.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
q.Set("X-Amz-Security-Token", sessionToken)
q.Set("X-Amz-Date", time.Now().Format("20060102T150405Z"))
q.Set("X-Amz-Expires", "3600")
req.URL.RawQuery = q.Encode()
}
return req
}

401
weed/s3api/s3_sse_bucket_test.go
View File

@@ -1,401 +0,0 @@
package s3api
import (
"fmt"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
)
// TestBucketDefaultSSEKMSEnforcement tests bucket default encryption enforcement
func TestBucketDefaultSSEKMSEnforcement(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Create bucket encryption configuration
config := &s3_pb.EncryptionConfiguration{
SseAlgorithm: "aws:kms",
KmsKeyId: kmsKey.KeyID,
BucketKeyEnabled: false,
}
t.Run("Bucket with SSE-KMS default encryption", func(t *testing.T) {
// Test that default encryption config is properly stored and retrieved
if config.SseAlgorithm != "aws:kms" {
t.Errorf("Expected SSE algorithm aws:kms, got %s", config.SseAlgorithm)
}
if config.KmsKeyId != kmsKey.KeyID {
t.Errorf("Expected KMS key ID %s, got %s", kmsKey.KeyID, config.KmsKeyId)
}
})
t.Run("Default encryption headers generation", func(t *testing.T) {
// Test generating default encryption headers for objects
headers := GetDefaultEncryptionHeaders(config)
if headers == nil {
t.Fatal("Expected default headers, got nil")
}
expectedAlgorithm := headers["X-Amz-Server-Side-Encryption"]
if expectedAlgorithm != "aws:kms" {
t.Errorf("Expected X-Amz-Server-Side-Encryption header aws:kms, got %s", expectedAlgorithm)
}
expectedKeyID := headers["X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"]
if expectedKeyID != kmsKey.KeyID {
t.Errorf("Expected X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id header %s, got %s", kmsKey.KeyID, expectedKeyID)
}
})
t.Run("Default encryption detection", func(t *testing.T) {
// Test IsDefaultEncryptionEnabled
enabled := IsDefaultEncryptionEnabled(config)
if !enabled {
t.Error("Should detect default encryption as enabled")
}
// Test with nil config
enabled = IsDefaultEncryptionEnabled(nil)
if enabled {
t.Error("Should detect default encryption as disabled for nil config")
}
// Test with empty config
emptyConfig := &s3_pb.EncryptionConfiguration{}
enabled = IsDefaultEncryptionEnabled(emptyConfig)
if enabled {
t.Error("Should detect default encryption as disabled for empty config")
}
})
}
// TestBucketEncryptionConfigValidation tests XML validation of bucket encryption configurations
func TestBucketEncryptionConfigValidation(t *testing.T) {
testCases := []struct {
name string
xml string
expectError bool
description string
}{
{
name: "Valid SSE-S3 configuration",
xml: `<ServerSideEncryptionConfiguration>
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>AES256</SSEAlgorithm>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`,
expectError: false,
description: "Basic SSE-S3 configuration should be valid",
},
{
name: "Valid SSE-KMS configuration",
xml: `<ServerSideEncryptionConfiguration>
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>aws:kms</SSEAlgorithm>
<KMSMasterKeyID>test-key-id</KMSMasterKeyID>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`,
expectError: false,
description: "SSE-KMS configuration with key ID should be valid",
},
{
name: "Valid SSE-KMS without key ID",
xml: `<ServerSideEncryptionConfiguration>
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>aws:kms</SSEAlgorithm>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`,
expectError: false,
description: "SSE-KMS without key ID should use default key",
},
{
name: "Invalid XML structure",
xml: `<ServerSideEncryptionConfiguration>
<InvalidRule>
<SSEAlgorithm>AES256</SSEAlgorithm>
</InvalidRule>
</ServerSideEncryptionConfiguration>`,
expectError: true,
description: "Invalid XML structure should be rejected",
},
{
name: "Empty configuration",
xml: `<ServerSideEncryptionConfiguration>
</ServerSideEncryptionConfiguration>`,
expectError: true,
description: "Empty configuration should be rejected",
},
{
name: "Invalid algorithm",
xml: `<ServerSideEncryptionConfiguration>
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>INVALID</SSEAlgorithm>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`,
expectError: true,
description: "Invalid algorithm should be rejected",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
config, err := encryptionConfigFromXMLBytes([]byte(tc.xml))
if tc.expectError && err == nil {
t.Errorf("Expected error for %s, but got none. %s", tc.name, tc.description)
}
if !tc.expectError && err != nil {
t.Errorf("Expected no error for %s, but got: %v. %s", tc.name, err, tc.description)
}
if !tc.expectError && config != nil {
// Validate the parsed configuration
t.Logf("Successfully parsed config: Algorithm=%s, KeyID=%s",
config.SseAlgorithm, config.KmsKeyId)
}
})
}
}
// TestBucketEncryptionAPIOperations tests the bucket encryption API operations
func TestBucketEncryptionAPIOperations(t *testing.T) {
// Note: These tests would normally require a full S3 API server setup
// For now, we test the individual components
t.Run("PUT bucket encryption", func(t *testing.T) {
xml := `<ServerSideEncryptionConfiguration>
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>aws:kms</SSEAlgorithm>
<KMSMasterKeyID>test-key-id</KMSMasterKeyID>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`
// Parse the XML to protobuf
config, err := encryptionConfigFromXMLBytes([]byte(xml))
if err != nil {
t.Fatalf("Failed to parse encryption config: %v", err)
}
// Verify the parsed configuration
if config.SseAlgorithm != "aws:kms" {
t.Errorf("Expected algorithm aws:kms, got %s", config.SseAlgorithm)
}
if config.KmsKeyId != "test-key-id" {
t.Errorf("Expected key ID test-key-id, got %s", config.KmsKeyId)
}
// Convert back to XML
xmlBytes, err := encryptionConfigToXMLBytes(config)
if err != nil {
t.Fatalf("Failed to convert config to XML: %v", err)
}
// Verify round-trip
if len(xmlBytes) == 0 {
t.Error("Generated XML should not be empty")
}
// Parse again to verify
roundTripConfig, err := encryptionConfigFromXMLBytes(xmlBytes)
if err != nil {
t.Fatalf("Failed to parse round-trip XML: %v", err)
}
if roundTripConfig.SseAlgorithm != config.SseAlgorithm {
t.Error("Round-trip algorithm doesn't match")
}
if roundTripConfig.KmsKeyId != config.KmsKeyId {
t.Error("Round-trip key ID doesn't match")
}
})
t.Run("GET bucket encryption", func(t *testing.T) {
// Test getting encryption configuration
config := &s3_pb.EncryptionConfiguration{
SseAlgorithm: "AES256",
KmsKeyId: "",
BucketKeyEnabled: false,
}
// Convert to XML for GET response
xmlBytes, err := encryptionConfigToXMLBytes(config)
if err != nil {
t.Fatalf("Failed to convert config to XML: %v", err)
}
if len(xmlBytes) == 0 {
t.Error("Generated XML should not be empty")
}
// Verify XML contains expected elements
xmlStr := string(xmlBytes)
if !strings.Contains(xmlStr, "AES256") {
t.Error("XML should contain AES256 algorithm")
}
})
t.Run("DELETE bucket encryption", func(t *testing.T) {
// Test deleting encryption configuration
// This would typically involve removing the configuration from metadata
// Simulate checking if encryption is enabled after deletion
enabled := IsDefaultEncryptionEnabled(nil)
if enabled {
t.Error("Encryption should be disabled after deletion")
}
})
}
// TestBucketEncryptionEdgeCases tests edge cases in bucket encryption
func TestBucketEncryptionEdgeCases(t *testing.T) {
t.Run("Large XML configuration", func(t *testing.T) {
// Test with a large but valid XML
largeXML := `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>aws:kms</SSEAlgorithm>
<KMSMasterKeyID>arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012</KMSMasterKeyID>
</ApplyServerSideEncryptionByDefault>
<BucketKeyEnabled>true</BucketKeyEnabled>
</Rule>
</ServerSideEncryptionConfiguration>`
config, err := encryptionConfigFromXMLBytes([]byte(largeXML))
if err != nil {
t.Fatalf("Failed to parse large XML: %v", err)
}
if config.SseAlgorithm != "aws:kms" {
t.Error("Should parse large XML correctly")
}
})
t.Run("XML with namespaces", func(t *testing.T) {
// Test XML with namespaces
namespacedXML := `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>AES256</SSEAlgorithm>
</ApplyServerSideEncryptionByDefault>
</Rule>
</ServerSideEncryptionConfiguration>`
config, err := encryptionConfigFromXMLBytes([]byte(namespacedXML))
if err != nil {
t.Fatalf("Failed to parse namespaced XML: %v", err)
}
if config.SseAlgorithm != "AES256" {
t.Error("Should parse namespaced XML correctly")
}
})
t.Run("Malformed XML", func(t *testing.T) {
malformedXMLs := []string{
`<ServerSideEncryptionConfiguration><Rule><SSEAlgorithm>AES256</Rule>`, // Unclosed tags
`<ServerSideEncryptionConfiguration><Rule></Rule></ServerSideEncryptionConfiguration>`, // Empty rule
`not-xml-at-all`, // Not XML
`<ServerSideEncryptionConfiguration xmlns="invalid-namespace"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`, // Invalid namespace
}
for i, malformedXML := range malformedXMLs {
t.Run(fmt.Sprintf("Malformed XML %d", i), func(t *testing.T) {
_, err := encryptionConfigFromXMLBytes([]byte(malformedXML))
if err == nil {
t.Errorf("Expected error for malformed XML %d, but got none", i)
}
})
}
})
}
// TestGetDefaultEncryptionHeaders tests generation of default encryption headers
func TestGetDefaultEncryptionHeaders(t *testing.T) {
testCases := []struct {
name string
config *s3_pb.EncryptionConfiguration
expectedHeaders map[string]string
}{
{
name: "Nil configuration",
config: nil,
expectedHeaders: nil,
},
{
name: "SSE-S3 configuration",
config: &s3_pb.EncryptionConfiguration{
SseAlgorithm: "AES256",
},
expectedHeaders: map[string]string{
"X-Amz-Server-Side-Encryption": "AES256",
},
},
{
name: "SSE-KMS configuration with key",
config: &s3_pb.EncryptionConfiguration{
SseAlgorithm: "aws:kms",
KmsKeyId: "test-key-id",
},
expectedHeaders: map[string]string{
"X-Amz-Server-Side-Encryption": "aws:kms",
"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "test-key-id",
},
},
{
name: "SSE-KMS configuration without key",
config: &s3_pb.EncryptionConfiguration{
SseAlgorithm: "aws:kms",
},
expectedHeaders: map[string]string{
"X-Amz-Server-Side-Encryption": "aws:kms",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
headers := GetDefaultEncryptionHeaders(tc.config)
if tc.expectedHeaders == nil && headers != nil {
t.Error("Expected nil headers but got some")
}
if tc.expectedHeaders != nil && headers == nil {
t.Error("Expected headers but got nil")
}
if tc.expectedHeaders != nil && headers != nil {
for key, expectedValue := range tc.expectedHeaders {
if actualValue, exists := headers[key]; !exists {
t.Errorf("Expected header %s not found", key)
} else if actualValue != expectedValue {
t.Errorf("Header %s: expected %s, got %s", key, expectedValue, actualValue)
}
}
// Check for unexpected headers
for key := range headers {
if _, expected := tc.expectedHeaders[key]; !expected {
t.Errorf("Unexpected header found: %s", key)
}
}
}
})
}
}

16
weed/s3api/s3_sse_c.go
View File

@@ -58,9 +58,9 @@ var (
// SSECustomerKey represents a customer-provided encryption key for SSE-C
type SSECustomerKey struct {
Algorithm string
Key []byte
KeyMD5 string
Algorithm string
Key []byte
KeyMD5 string
}
// IsSSECRequest checks if the request contains SSE-C headers
@@ -134,16 +134,6 @@ func validateAndParseSSECHeaders(algorithm, key, keyMD5 string) (*SSECustomerKey
}, nil
}
// ValidateSSECHeaders validates SSE-C headers in the request
func ValidateSSECHeaders(r *http.Request) error {
algorithm := r.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm)
key := r.Header.Get(s3_constants.AmzServerSideEncryptionCustomerKey)
keyMD5 := r.Header.Get(s3_constants.AmzServerSideEncryptionCustomerKeyMD5)
_, err := validateAndParseSSECHeaders(algorithm, key, keyMD5)
return err
}
// ParseSSECHeaders parses and validates SSE-C headers from the request
func ParseSSECHeaders(r *http.Request) (*SSECustomerKey, error) {
algorithm := r.Header.Get(s3_constants.AmzServerSideEncryptionCustomerAlgorithm)

407
weed/s3api/s3_sse_c_test.go
View File

@@ -1,407 +0,0 @@
package s3api
import (
"bytes"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"net/http"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
func base64MD5(b []byte) string {
s := md5.Sum(b)
return base64.StdEncoding.EncodeToString(s[:])
}
func TestSSECHeaderValidation(t *testing.T) {
// Test valid SSE-C headers
req := &http.Request{Header: make(http.Header)}
key := make([]byte, 32) // 256-bit key
for i := range key {
key[i] = byte(i)
}
keyBase64 := base64.StdEncoding.EncodeToString(key)
md5sum := md5.Sum(key)
keyMD5 := base64.StdEncoding.EncodeToString(md5sum[:])
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, keyBase64)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyMD5)
// Test validation
err := ValidateSSECHeaders(req)
if err != nil {
t.Errorf("Expected valid headers, got error: %v", err)
}
// Test parsing
customerKey, err := ParseSSECHeaders(req)
if err != nil {
t.Errorf("Expected successful parsing, got error: %v", err)
}
if customerKey == nil {
t.Error("Expected customer key, got nil")
}
if customerKey.Algorithm != "AES256" {
t.Errorf("Expected algorithm AES256, got %s", customerKey.Algorithm)
}
if !bytes.Equal(customerKey.Key, key) {
t.Error("Key doesn't match original")
}
if customerKey.KeyMD5 != keyMD5 {
t.Errorf("Expected key MD5 %s, got %s", keyMD5, customerKey.KeyMD5)
}
}
func TestSSECCopySourceHeaders(t *testing.T) {
// Test valid SSE-C copy source headers
req := &http.Request{Header: make(http.Header)}
key := make([]byte, 32) // 256-bit key
for i := range key {
key[i] = byte(i) + 1 // Different from regular test
}
keyBase64 := base64.StdEncoding.EncodeToString(key)
md5sum2 := md5.Sum(key)
keyMD5 := base64.StdEncoding.EncodeToString(md5sum2[:])
req.Header.Set(s3_constants.AmzCopySourceServerSideEncryptionCustomerAlgorithm, "AES256")
req.Header.Set(s3_constants.AmzCopySourceServerSideEncryptionCustomerKey, keyBase64)
req.Header.Set(s3_constants.AmzCopySourceServerSideEncryptionCustomerKeyMD5, keyMD5)
// Test parsing copy source headers
customerKey, err := ParseSSECCopySourceHeaders(req)
if err != nil {
t.Errorf("Expected successful copy source parsing, got error: %v", err)
}
if customerKey == nil {
t.Error("Expected customer key from copy source headers, got nil")
}
if customerKey.Algorithm != "AES256" {
t.Errorf("Expected algorithm AES256, got %s", customerKey.Algorithm)
}
if !bytes.Equal(customerKey.Key, key) {
t.Error("Copy source key doesn't match original")
}
// Test that regular headers don't interfere with copy source headers
regularKey, err := ParseSSECHeaders(req)
if err != nil {
t.Errorf("Regular header parsing should not fail: %v", err)
}
if regularKey != nil {
t.Error("Expected nil for regular headers when only copy source headers are present")
}
}
func TestSSECHeaderValidationErrors(t *testing.T) {
tests := []struct {
name string
algorithm string
key string
keyMD5 string
wantErr error
}{
{
name: "invalid algorithm",
algorithm: "AES128",
key: base64.StdEncoding.EncodeToString(make([]byte, 32)),
keyMD5: base64MD5(make([]byte, 32)),
wantErr: ErrInvalidEncryptionAlgorithm,
},
{
name: "invalid key length",
algorithm: "AES256",
key: base64.StdEncoding.EncodeToString(make([]byte, 16)),
keyMD5: base64MD5(make([]byte, 16)),
wantErr: ErrInvalidEncryptionKey,
},
{
name: "mismatched MD5",
algorithm: "AES256",
key: base64.StdEncoding.EncodeToString(make([]byte, 32)),
keyMD5: "wrong==md5",
wantErr: ErrSSECustomerKeyMD5Mismatch,
},
{
name: "incomplete headers",
algorithm: "AES256",
key: "",
keyMD5: "",
wantErr: ErrInvalidRequest,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := &http.Request{Header: make(http.Header)}
if tt.algorithm != "" {
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, tt.algorithm)
}
if tt.key != "" {
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, tt.key)
}
if tt.keyMD5 != "" {
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, tt.keyMD5)
}
err := ValidateSSECHeaders(req)
if err != tt.wantErr {
t.Errorf("Expected error %v, got %v", tt.wantErr, err)
}
})
}
}
func TestSSECEncryptionDecryption(t *testing.T) {
// Create customer key
key := make([]byte, 32)
for i := range key {
key[i] = byte(i)
}
md5sumKey := md5.Sum(key)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: key,
KeyMD5: base64.StdEncoding.EncodeToString(md5sumKey[:]),
}
// Test data
testData := []byte("Hello, World! This is a test of SSE-C encryption.")
// Create encrypted reader
dataReader := bytes.NewReader(testData)
encryptedReader, iv, err := CreateSSECEncryptedReader(dataReader, customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Read encrypted data
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Verify data is actually encrypted (different from original)
if bytes.Equal(encryptedData[16:], testData) { // Skip IV
t.Error("Data doesn't appear to be encrypted")
}
// Create decrypted reader
encryptedReader2 := bytes.NewReader(encryptedData)
decryptedReader, err := CreateSSECDecryptedReader(encryptedReader2, customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Read decrypted data
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
// Verify decrypted data matches original
if !bytes.Equal(decryptedData, testData) {
t.Errorf("Decrypted data doesn't match original.\nOriginal: %s\nDecrypted: %s", testData, decryptedData)
}
}
func TestSSECIsSSECRequest(t *testing.T) {
// Test with SSE-C headers
req := &http.Request{Header: make(http.Header)}
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
if !IsSSECRequest(req) {
t.Error("Expected IsSSECRequest to return true when SSE-C headers are present")
}
// Test without SSE-C headers
req2 := &http.Request{Header: make(http.Header)}
if IsSSECRequest(req2) {
t.Error("Expected IsSSECRequest to return false when no SSE-C headers are present")
}
}
// Test encryption with different data sizes (similar to s3tests)
func TestSSECEncryptionVariousSizes(t *testing.T) {
sizes := []int{1, 13, 1024, 1024 * 1024} // 1B, 13B, 1KB, 1MB
for _, size := range sizes {
t.Run(fmt.Sprintf("size_%d", size), func(t *testing.T) {
// Create customer key
key := make([]byte, 32)
for i := range key {
key[i] = byte(i + size) // Make key unique per test
}
md5sumDyn := md5.Sum(key)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: key,
KeyMD5: base64.StdEncoding.EncodeToString(md5sumDyn[:]),
}
// Create test data of specified size
testData := make([]byte, size)
for i := range testData {
testData[i] = byte('A' + (i % 26)) // Pattern of A-Z
}
// Encrypt
dataReader := bytes.NewReader(testData)
encryptedReader, iv, err := CreateSSECEncryptedReader(dataReader, customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Verify encrypted data has same size as original (IV is stored in metadata, not in stream)
if len(encryptedData) != size {
t.Errorf("Expected encrypted data length %d (same as original), got %d", size, len(encryptedData))
}
// Decrypt
encryptedReader2 := bytes.NewReader(encryptedData)
decryptedReader, err := CreateSSECDecryptedReader(encryptedReader2, customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
// Verify decrypted data matches original
if !bytes.Equal(decryptedData, testData) {
t.Errorf("Decrypted data doesn't match original for size %d", size)
}
})
}
}
func TestSSECEncryptionWithNilKey(t *testing.T) {
testData := []byte("test data")
dataReader := bytes.NewReader(testData)
// Test encryption with nil key (should pass through)
encryptedReader, iv, err := CreateSSECEncryptedReader(dataReader, nil)
if err != nil {
t.Fatalf("Failed to create encrypted reader with nil key: %v", err)
}
result, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read from pass-through reader: %v", err)
}
if !bytes.Equal(result, testData) {
t.Error("Data should pass through unchanged when key is nil")
}
// Test decryption with nil key (should pass through)
dataReader2 := bytes.NewReader(testData)
decryptedReader, err := CreateSSECDecryptedReader(dataReader2, nil, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader with nil key: %v", err)
}
result2, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read from pass-through reader: %v", err)
}
if !bytes.Equal(result2, testData) {
t.Error("Data should pass through unchanged when key is nil")
}
}
// TestSSECEncryptionSmallBuffers tests the fix for the critical bug where small buffers
// could corrupt the data stream when reading in chunks smaller than the IV size
func TestSSECEncryptionSmallBuffers(t *testing.T) {
testData := []byte("This is a test message for small buffer reads")
// Create customer key
key := make([]byte, 32)
for i := range key {
key[i] = byte(i)
}
md5sumKey3 := md5.Sum(key)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: key,
KeyMD5: base64.StdEncoding.EncodeToString(md5sumKey3[:]),
}
// Create encrypted reader
dataReader := bytes.NewReader(testData)
encryptedReader, iv, err := CreateSSECEncryptedReader(dataReader, customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Read with very small buffers (smaller than IV size of 16 bytes)
var encryptedData []byte
smallBuffer := make([]byte, 5) // Much smaller than 16-byte IV
for {
n, err := encryptedReader.Read(smallBuffer)
if n > 0 {
encryptedData = append(encryptedData, smallBuffer[:n]...)
}
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("Error reading encrypted data: %v", err)
}
}
// Verify we have some encrypted data (IV is in metadata, not in stream)
if len(encryptedData) == 0 && len(testData) > 0 {
t.Fatal("Expected encrypted data but got none")
}
// Expected size: same as original data (IV is stored in metadata, not in stream)
if len(encryptedData) != len(testData) {
t.Errorf("Expected encrypted data size %d (same as original), got %d", len(testData), len(encryptedData))
}
// Decrypt and verify
encryptedReader2 := bytes.NewReader(encryptedData)
decryptedReader, err := CreateSSECDecryptedReader(encryptedReader2, customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
if !bytes.Equal(decryptedData, testData) {
t.Errorf("Decrypted data doesn't match original.\nOriginal: %s\nDecrypted: %s", testData, decryptedData)
}
}

628
weed/s3api/s3_sse_copy_test.go
View File

@@ -1,628 +0,0 @@
package s3api
import (
"bytes"
"io"
"net/http"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// TestSSECObjectCopy tests copying SSE-C encrypted objects with different keys
func TestSSECObjectCopy(t *testing.T) {
// Original key for source object
sourceKey := GenerateTestSSECKey(1)
sourceCustomerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: sourceKey.Key,
KeyMD5: sourceKey.KeyMD5,
}
// Destination key for target object
destKey := GenerateTestSSECKey(2)
destCustomerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: destKey.Key,
KeyMD5: destKey.KeyMD5,
}
testData := "Hello, SSE-C copy world!"
// Encrypt with source key
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(testData), sourceCustomerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Test copy strategy determination
sourceMetadata := make(map[string][]byte)
StoreSSECIVInMetadata(sourceMetadata, iv)
sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerAlgorithm] = []byte("AES256")
sourceMetadata[s3_constants.AmzServerSideEncryptionCustomerKeyMD5] = []byte(sourceKey.KeyMD5)
t.Run("Same key copy (direct copy)", func(t *testing.T) {
strategy, err := DetermineSSECCopyStrategy(sourceMetadata, sourceCustomerKey, sourceCustomerKey)
if err != nil {
t.Fatalf("Failed to determine copy strategy: %v", err)
}
if strategy != SSECCopyStrategyDirect {
t.Errorf("Expected direct copy strategy for same key, got %v", strategy)
}
})
t.Run("Different key copy (decrypt-encrypt)", func(t *testing.T) {
strategy, err := DetermineSSECCopyStrategy(sourceMetadata, sourceCustomerKey, destCustomerKey)
if err != nil {
t.Fatalf("Failed to determine copy strategy: %v", err)
}
if strategy != SSECCopyStrategyDecryptEncrypt {
t.Errorf("Expected decrypt-encrypt copy strategy for different keys, got %v", strategy)
}
})
t.Run("Can direct copy check", func(t *testing.T) {
// Same key should allow direct copy
canDirect := CanDirectCopySSEC(sourceMetadata, sourceCustomerKey, sourceCustomerKey)
if !canDirect {
t.Error("Should allow direct copy with same key")
}
// Different key should not allow direct copy
canDirect = CanDirectCopySSEC(sourceMetadata, sourceCustomerKey, destCustomerKey)
if canDirect {
t.Error("Should not allow direct copy with different keys")
}
})
// Test actual copy operation (decrypt with source key, encrypt with dest key)
t.Run("Full copy operation", func(t *testing.T) {
// Decrypt with source key
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), sourceCustomerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Re-encrypt with destination key
reEncryptedReader, destIV, err := CreateSSECEncryptedReader(decryptedReader, destCustomerKey)
if err != nil {
t.Fatalf("Failed to create re-encrypted reader: %v", err)
}
reEncryptedData, err := io.ReadAll(reEncryptedReader)
if err != nil {
t.Fatalf("Failed to read re-encrypted data: %v", err)
}
// Verify we can decrypt with destination key
finalDecryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(reEncryptedData), destCustomerKey, destIV)
if err != nil {
t.Fatalf("Failed to create final decrypted reader: %v", err)
}
finalData, err := io.ReadAll(finalDecryptedReader)
if err != nil {
t.Fatalf("Failed to read final decrypted data: %v", err)
}
if string(finalData) != testData {
t.Errorf("Expected %s, got %s", testData, string(finalData))
}
})
}
// TestSSEKMSObjectCopy tests copying SSE-KMS encrypted objects
func TestSSEKMSObjectCopy(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
testData := "Hello, SSE-KMS copy world!"
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
// Encrypt with SSE-KMS
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(testData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
t.Run("Same KMS key copy", func(t *testing.T) {
// Decrypt with original key
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Re-encrypt with same KMS key
reEncryptedReader, newSseKey, err := CreateSSEKMSEncryptedReader(decryptedReader, kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create re-encrypted reader: %v", err)
}
reEncryptedData, err := io.ReadAll(reEncryptedReader)
if err != nil {
t.Fatalf("Failed to read re-encrypted data: %v", err)
}
// Verify we can decrypt with new key
finalDecryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(reEncryptedData), newSseKey)
if err != nil {
t.Fatalf("Failed to create final decrypted reader: %v", err)
}
finalData, err := io.ReadAll(finalDecryptedReader)
if err != nil {
t.Fatalf("Failed to read final decrypted data: %v", err)
}
if string(finalData) != testData {
t.Errorf("Expected %s, got %s", testData, string(finalData))
}
})
}
// TestSSECToSSEKMSCopy tests cross-encryption copy (SSE-C to SSE-KMS)
func TestSSECToSSEKMSCopy(t *testing.T) {
// Setup SSE-C key
ssecKey := GenerateTestSSECKey(1)
ssecCustomerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: ssecKey.Key,
KeyMD5: ssecKey.KeyMD5,
}
// Setup SSE-KMS
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
testData := "Hello, cross-encryption copy world!"
// Encrypt with SSE-C
encryptedReader, ssecIV, err := CreateSSECEncryptedReader(strings.NewReader(testData), ssecCustomerKey)
if err != nil {
t.Fatalf("Failed to create SSE-C encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read SSE-C encrypted data: %v", err)
}
// Decrypt SSE-C data
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), ssecCustomerKey, ssecIV)
if err != nil {
t.Fatalf("Failed to create SSE-C decrypted reader: %v", err)
}
// Re-encrypt with SSE-KMS
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
reEncryptedReader, sseKmsKey, err := CreateSSEKMSEncryptedReader(decryptedReader, kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create SSE-KMS encrypted reader: %v", err)
}
reEncryptedData, err := io.ReadAll(reEncryptedReader)
if err != nil {
t.Fatalf("Failed to read SSE-KMS encrypted data: %v", err)
}
// Decrypt with SSE-KMS
finalDecryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(reEncryptedData), sseKmsKey)
if err != nil {
t.Fatalf("Failed to create SSE-KMS decrypted reader: %v", err)
}
finalData, err := io.ReadAll(finalDecryptedReader)
if err != nil {
t.Fatalf("Failed to read final decrypted data: %v", err)
}
if string(finalData) != testData {
t.Errorf("Expected %s, got %s", testData, string(finalData))
}
}
// TestSSEKMSToSSECCopy tests cross-encryption copy (SSE-KMS to SSE-C)
func TestSSEKMSToSSECCopy(t *testing.T) {
// Setup SSE-KMS
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Setup SSE-C key
ssecKey := GenerateTestSSECKey(1)
ssecCustomerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: ssecKey.Key,
KeyMD5: ssecKey.KeyMD5,
}
testData := "Hello, reverse cross-encryption copy world!"
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
// Encrypt with SSE-KMS
encryptedReader, sseKmsKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(testData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create SSE-KMS encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read SSE-KMS encrypted data: %v", err)
}
// Decrypt SSE-KMS data
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKmsKey)
if err != nil {
t.Fatalf("Failed to create SSE-KMS decrypted reader: %v", err)
}
// Re-encrypt with SSE-C
reEncryptedReader, reEncryptedIV, err := CreateSSECEncryptedReader(decryptedReader, ssecCustomerKey)
if err != nil {
t.Fatalf("Failed to create SSE-C encrypted reader: %v", err)
}
reEncryptedData, err := io.ReadAll(reEncryptedReader)
if err != nil {
t.Fatalf("Failed to read SSE-C encrypted data: %v", err)
}
// Decrypt with SSE-C
finalDecryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(reEncryptedData), ssecCustomerKey, reEncryptedIV)
if err != nil {
t.Fatalf("Failed to create SSE-C decrypted reader: %v", err)
}
finalData, err := io.ReadAll(finalDecryptedReader)
if err != nil {
t.Fatalf("Failed to read final decrypted data: %v", err)
}
if string(finalData) != testData {
t.Errorf("Expected %s, got %s", testData, string(finalData))
}
}
// TestSSECopyWithCorruptedSource tests copy operations with corrupted source data
func TestSSECopyWithCorruptedSource(t *testing.T) {
ssecKey := GenerateTestSSECKey(1)
ssecCustomerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: ssecKey.Key,
KeyMD5: ssecKey.KeyMD5,
}
testData := "Hello, corruption test!"
// Encrypt data
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(testData), ssecCustomerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Corrupt the encrypted data
corruptedData := make([]byte, len(encryptedData))
copy(corruptedData, encryptedData)
if len(corruptedData) > s3_constants.AESBlockSize {
// Corrupt a byte after the IV
corruptedData[s3_constants.AESBlockSize] ^= 0xFF
}
// Try to decrypt corrupted data
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(corruptedData), ssecCustomerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader for corrupted data: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
// This is okay - corrupted data might cause read errors
t.Logf("Read error for corrupted data (expected): %v", err)
return
}
// If we can read it, the data should be different from original
if string(decryptedData) == testData {
t.Error("Decrypted corrupted data should not match original")
}
}
// TestSSEKMSCopyStrategy tests SSE-KMS copy strategy determination
func TestSSEKMSCopyStrategy(t *testing.T) {
tests := []struct {
name string
srcMetadata map[string][]byte
destKeyID string
expectedStrategy SSEKMSCopyStrategy
}{
{
name: "Unencrypted to unencrypted",
srcMetadata: map[string][]byte{},
destKeyID: "",
expectedStrategy: SSEKMSCopyStrategyDirect,
},
{
name: "Same KMS key",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "test-key-123",
expectedStrategy: SSEKMSCopyStrategyDirect,
},
{
name: "Different KMS keys",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "test-key-456",
expectedStrategy: SSEKMSCopyStrategyDecryptEncrypt,
},
{
name: "Encrypted to unencrypted",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "",
expectedStrategy: SSEKMSCopyStrategyDecryptEncrypt,
},
{
name: "Unencrypted to encrypted",
srcMetadata: map[string][]byte{},
destKeyID: "test-key-123",
expectedStrategy: SSEKMSCopyStrategyDecryptEncrypt,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
strategy, err := DetermineSSEKMSCopyStrategy(tt.srcMetadata, tt.destKeyID)
if err != nil {
t.Fatalf("DetermineSSEKMSCopyStrategy failed: %v", err)
}
if strategy != tt.expectedStrategy {
t.Errorf("Expected strategy %v, got %v", tt.expectedStrategy, strategy)
}
})
}
}
// TestSSEKMSCopyHeaders tests SSE-KMS copy header parsing
func TestSSEKMSCopyHeaders(t *testing.T) {
tests := []struct {
name string
headers map[string]string
expectedKeyID string
expectedContext map[string]string
expectedBucketKey bool
expectError bool
}{
{
name: "No SSE-KMS headers",
headers: map[string]string{},
expectedKeyID: "",
expectedContext: nil,
expectedBucketKey: false,
expectError: false,
},
{
name: "SSE-KMS with key ID",
headers: map[string]string{
s3_constants.AmzServerSideEncryption: "aws:kms",
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "test-key-123",
},
expectedKeyID: "test-key-123",
expectedContext: nil,
expectedBucketKey: false,
expectError: false,
},
{
name: "SSE-KMS with all options",
headers: map[string]string{
s3_constants.AmzServerSideEncryption: "aws:kms",
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "test-key-123",
s3_constants.AmzServerSideEncryptionContext: "eyJ0ZXN0IjoidmFsdWUifQ==", // base64 of {"test":"value"}
s3_constants.AmzServerSideEncryptionBucketKeyEnabled: "true",
},
expectedKeyID: "test-key-123",
expectedContext: map[string]string{"test": "value"},
expectedBucketKey: true,
expectError: false,
},
{
name: "Invalid key ID",
headers: map[string]string{
s3_constants.AmzServerSideEncryption: "aws:kms",
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "invalid key id",
},
expectError: true,
},
{
name: "Invalid encryption context",
headers: map[string]string{
s3_constants.AmzServerSideEncryption: "aws:kms",
s3_constants.AmzServerSideEncryptionContext: "invalid-base64!",
},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, _ := http.NewRequest("PUT", "/test", nil)
for k, v := range tt.headers {
req.Header.Set(k, v)
}
keyID, context, bucketKey, err := ParseSSEKMSCopyHeaders(req)
if tt.expectError {
if err == nil {
t.Error("Expected error but got none")
}
return
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if keyID != tt.expectedKeyID {
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, keyID)
}
if !mapsEqual(context, tt.expectedContext) {
t.Errorf("Expected context %v, got %v", tt.expectedContext, context)
}
if bucketKey != tt.expectedBucketKey {
t.Errorf("Expected bucketKey %v, got %v", tt.expectedBucketKey, bucketKey)
}
})
}
}
// TestSSEKMSDirectCopy tests direct copy scenarios
func TestSSEKMSDirectCopy(t *testing.T) {
tests := []struct {
name string
srcMetadata map[string][]byte
destKeyID string
canDirect bool
}{
{
name: "Both unencrypted",
srcMetadata: map[string][]byte{},
destKeyID: "",
canDirect: true,
},
{
name: "Same key ID",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "test-key-123",
canDirect: true,
},
{
name: "Different key IDs",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "test-key-456",
canDirect: false,
},
{
name: "Source encrypted, dest unencrypted",
srcMetadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
destKeyID: "",
canDirect: false,
},
{
name: "Source unencrypted, dest encrypted",
srcMetadata: map[string][]byte{},
destKeyID: "test-key-123",
canDirect: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
canDirect := CanDirectCopySSEKMS(tt.srcMetadata, tt.destKeyID)
if canDirect != tt.canDirect {
t.Errorf("Expected canDirect %v, got %v", tt.canDirect, canDirect)
}
})
}
}
// TestGetSourceSSEKMSInfo tests extraction of SSE-KMS info from metadata
func TestGetSourceSSEKMSInfo(t *testing.T) {
tests := []struct {
name string
metadata map[string][]byte
expectedKeyID string
expectedEncrypted bool
}{
{
name: "No encryption",
metadata: map[string][]byte{},
expectedKeyID: "",
expectedEncrypted: false,
},
{
name: "SSE-KMS with key ID",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzServerSideEncryptionAwsKmsKeyId: []byte("test-key-123"),
},
expectedKeyID: "test-key-123",
expectedEncrypted: true,
},
{
name: "SSE-KMS without key ID (default key)",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
},
expectedKeyID: "",
expectedEncrypted: true,
},
{
name: "Non-KMS encryption",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
},
expectedKeyID: "",
expectedEncrypted: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
keyID, encrypted := GetSourceSSEKMSInfo(tt.metadata)
if keyID != tt.expectedKeyID {
t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, keyID)
}
if encrypted != tt.expectedEncrypted {
t.Errorf("Expected encrypted %v, got %v", tt.expectedEncrypted, encrypted)
}
})
}
}
// Helper function to compare maps
func mapsEqual(a, b map[string]string) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if b[k] != v {
return false
}
}
return true
}

400
weed/s3api/s3_sse_error_test.go
View File

@@ -1,400 +0,0 @@
package s3api
import (
"bytes"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// TestSSECWrongKeyDecryption tests decryption with wrong SSE-C key
func TestSSECWrongKeyDecryption(t *testing.T) {
// Setup original key and encrypt data
originalKey := GenerateTestSSECKey(1)
testData := "Hello, SSE-C world!"
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(testData), &SSECustomerKey{
Algorithm: "AES256",
Key: originalKey.Key,
KeyMD5: originalKey.KeyMD5,
})
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Read encrypted data
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Try to decrypt with wrong key
wrongKey := GenerateTestSSECKey(2) // Different seed = different key
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), &SSECustomerKey{
Algorithm: "AES256",
Key: wrongKey.Key,
KeyMD5: wrongKey.KeyMD5,
}, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Read decrypted data - should be garbage/different from original
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
// Verify the decrypted data is NOT the same as original (wrong key used)
if string(decryptedData) == testData {
t.Error("Decryption with wrong key should not produce original data")
}
}
// TestSSEKMSKeyNotFound tests handling of missing KMS key
func TestSSEKMSKeyNotFound(t *testing.T) {
// Note: The local KMS provider creates keys on-demand by design.
// This test validates that when on-demand creation fails or is disabled,
// appropriate errors are returned.
// Test with an invalid key ID that would fail even on-demand creation
invalidKeyID := "" // Empty key ID should fail
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
_, _, err := CreateSSEKMSEncryptedReader(strings.NewReader("test data"), invalidKeyID, encryptionContext)
// Should get an error for invalid/empty key
if err == nil {
t.Error("Expected error for empty KMS key ID, got none")
}
// For local KMS with on-demand creation, we test what we can realistically test
if err != nil {
t.Logf("Got expected error for empty key ID: %v", err)
}
}
// TestSSEHeadersWithoutEncryption tests inconsistent state where headers are present but no encryption
func TestSSEHeadersWithoutEncryption(t *testing.T) {
testCases := []struct {
name string
setupReq func() *http.Request
}{
{
name: "SSE-C algorithm without key",
setupReq: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
// Missing key and MD5
return req
},
},
{
name: "SSE-C key without algorithm",
setupReq: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
keyPair := GenerateTestSSECKey(1)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, keyPair.KeyB64)
// Missing algorithm
return req
},
},
{
name: "SSE-KMS key ID without algorithm",
setupReq: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, "test-key-id")
// Missing algorithm
return req
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := tc.setupReq()
// Validate headers - should catch incomplete configurations
if strings.Contains(tc.name, "SSE-C") {
err := ValidateSSECHeaders(req)
if err == nil {
t.Error("Expected validation error for incomplete SSE-C headers")
}
}
})
}
}
// TestSSECInvalidKeyFormats tests various invalid SSE-C key formats
func TestSSECInvalidKeyFormats(t *testing.T) {
testCases := []struct {
name string
algorithm string
key string
keyMD5 string
expectErr bool
}{
{
name: "Invalid algorithm",
algorithm: "AES128",
key: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3RrZXk=", // 32 bytes base64
keyMD5: "valid-md5-hash",
expectErr: true,
},
{
name: "Invalid key length (too short)",
algorithm: "AES256",
key: "c2hvcnRrZXk=", // "shortkey" base64 - too short
keyMD5: "valid-md5-hash",
expectErr: true,
},
{
name: "Invalid key length (too long)",
algorithm: "AES256",
key: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleQ==", // too long
keyMD5: "valid-md5-hash",
expectErr: true,
},
{
name: "Invalid base64 key",
algorithm: "AES256",
key: "invalid-base64!",
keyMD5: "valid-md5-hash",
expectErr: true,
},
{
name: "Invalid base64 MD5",
algorithm: "AES256",
key: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3RrZXk=",
keyMD5: "invalid-base64!",
expectErr: true,
},
{
name: "Mismatched MD5",
algorithm: "AES256",
key: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3RrZXk=",
keyMD5: "d29uZy1tZDUtaGFzaA==", // "wrong-md5-hash" base64
expectErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, tc.algorithm)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, tc.key)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, tc.keyMD5)
err := ValidateSSECHeaders(req)
if tc.expectErr && err == nil {
t.Errorf("Expected error for %s, but got none", tc.name)
}
if !tc.expectErr && err != nil {
t.Errorf("Expected no error for %s, but got: %v", tc.name, err)
}
})
}
}
// TestSSEKMSInvalidConfigurations tests various invalid SSE-KMS configurations
func TestSSEKMSInvalidConfigurations(t *testing.T) {
testCases := []struct {
name string
setupRequest func() *http.Request
expectError bool
}{
{
name: "Invalid algorithm",
setupRequest: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryption, "invalid-algorithm")
return req
},
expectError: true,
},
{
name: "Empty key ID",
setupRequest: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryption, "aws:kms")
req.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, "")
return req
},
expectError: false, // Empty key ID might be valid (use default)
},
{
name: "Invalid key ID format",
setupRequest: func() *http.Request {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
req.Header.Set(s3_constants.AmzServerSideEncryption, "aws:kms")
req.Header.Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, "invalid key id with spaces")
return req
},
expectError: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := tc.setupRequest()
_, err := ParseSSEKMSHeaders(req)
if tc.expectError && err == nil {
t.Errorf("Expected error for %s, but got none", tc.name)
}
if !tc.expectError && err != nil {
t.Errorf("Expected no error for %s, but got: %v", tc.name, err)
}
})
}
}
// TestSSEEmptyDataHandling tests handling of empty data with SSE
func TestSSEEmptyDataHandling(t *testing.T) {
t.Run("SSE-C with empty data", func(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
// Encrypt empty data
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(""), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for empty data: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted empty data: %v", err)
}
// Should have IV for empty data
if len(iv) != s3_constants.AESBlockSize {
t.Error("IV should be present even for empty data")
}
// Decrypt and verify
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader for empty data: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted empty data: %v", err)
}
if len(decryptedData) != 0 {
t.Errorf("Expected empty decrypted data, got %d bytes", len(decryptedData))
}
})
t.Run("SSE-KMS with empty data", func(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
// Encrypt empty data
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(""), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader for empty data: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted empty data: %v", err)
}
// Empty data should produce empty encrypted data (IV is stored in metadata)
if len(encryptedData) != 0 {
t.Errorf("Encrypted empty data should be empty, got %d bytes", len(encryptedData))
}
// Decrypt and verify
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader for empty data: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted empty data: %v", err)
}
if len(decryptedData) != 0 {
t.Errorf("Expected empty decrypted data, got %d bytes", len(decryptedData))
}
})
}
// TestSSEConcurrentAccess tests SSE operations under concurrent access
func TestSSEConcurrentAccess(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
const numGoroutines = 10
done := make(chan bool, numGoroutines)
errors := make(chan error, numGoroutines)
// Run multiple encryption/decryption operations concurrently
for i := 0; i < numGoroutines; i++ {
go func(id int) {
defer func() { done <- true }()
testData := fmt.Sprintf("test data %d", id)
// Encrypt
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(testData), customerKey)
if err != nil {
errors <- fmt.Errorf("goroutine %d encrypt error: %v", id, err)
return
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
errors <- fmt.Errorf("goroutine %d read encrypted error: %v", id, err)
return
}
// Decrypt
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
errors <- fmt.Errorf("goroutine %d decrypt error: %v", id, err)
return
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
errors <- fmt.Errorf("goroutine %d read decrypted error: %v", id, err)
return
}
if string(decryptedData) != testData {
errors <- fmt.Errorf("goroutine %d data mismatch: expected %s, got %s", id, testData, string(decryptedData))
return
}
}(i)
}
// Wait for all goroutines to complete
for i := 0; i < numGoroutines; i++ {
<-done
}
// Check for errors
close(errors)
for err := range errors {
t.Error(err)
}
}

401
weed/s3api/s3_sse_http_test.go
View File

@@ -1,401 +0,0 @@
package s3api
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// TestPutObjectWithSSEC tests PUT object with SSE-C through HTTP handler
func TestPutObjectWithSSEC(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
testData := "Hello, SSE-C PUT object!"
// Create HTTP request
req := CreateTestHTTPRequest("PUT", "/test-bucket/test-object", []byte(testData))
SetupTestSSECHeaders(req, keyPair)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Create response recorder
w := CreateTestHTTPResponse()
// Test header validation
err := ValidateSSECHeaders(req)
if err != nil {
t.Fatalf("Header validation failed: %v", err)
}
// Parse SSE-C headers
customerKey, err := ParseSSECHeaders(req)
if err != nil {
t.Fatalf("Failed to parse SSE-C headers: %v", err)
}
if customerKey == nil {
t.Fatal("Expected customer key, got nil")
}
// Verify parsed key matches input
if !bytes.Equal(customerKey.Key, keyPair.Key) {
t.Error("Parsed key doesn't match input key")
}
if customerKey.KeyMD5 != keyPair.KeyMD5 {
t.Errorf("Parsed key MD5 doesn't match: expected %s, got %s", keyPair.KeyMD5, customerKey.KeyMD5)
}
// Simulate setting response headers
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyPair.KeyMD5)
// Verify response headers
AssertSSECHeaders(t, w, keyPair)
}
// TestGetObjectWithSSEC tests GET object with SSE-C through HTTP handler
func TestGetObjectWithSSEC(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
// Create HTTP request for GET
req := CreateTestHTTPRequest("GET", "/test-bucket/test-object", nil)
SetupTestSSECHeaders(req, keyPair)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Create response recorder
w := CreateTestHTTPResponse()
// Test that SSE-C is detected for GET requests
if !IsSSECRequest(req) {
t.Error("Should detect SSE-C request for GET with SSE-C headers")
}
// Validate headers
err := ValidateSSECHeaders(req)
if err != nil {
t.Fatalf("Header validation failed: %v", err)
}
// Simulate response with SSE-C headers
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
w.Header().Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyPair.KeyMD5)
w.WriteHeader(http.StatusOK)
// Verify response
if w.Code != http.StatusOK {
t.Errorf("Expected status 200, got %d", w.Code)
}
AssertSSECHeaders(t, w, keyPair)
}
// TestPutObjectWithSSEKMS tests PUT object with SSE-KMS through HTTP handler
func TestPutObjectWithSSEKMS(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
testData := "Hello, SSE-KMS PUT object!"
// Create HTTP request
req := CreateTestHTTPRequest("PUT", "/test-bucket/test-object", []byte(testData))
SetupTestSSEKMSHeaders(req, kmsKey.KeyID)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Create response recorder
w := CreateTestHTTPResponse()
// Test that SSE-KMS is detected
if !IsSSEKMSRequest(req) {
t.Error("Should detect SSE-KMS request")
}
// Parse SSE-KMS headers
sseKmsKey, err := ParseSSEKMSHeaders(req)
if err != nil {
t.Fatalf("Failed to parse SSE-KMS headers: %v", err)
}
if sseKmsKey == nil {
t.Fatal("Expected SSE-KMS key, got nil")
}
if sseKmsKey.KeyID != kmsKey.KeyID {
t.Errorf("Parsed key ID doesn't match: expected %s, got %s", kmsKey.KeyID, sseKmsKey.KeyID)
}
// Simulate setting response headers
w.Header().Set(s3_constants.AmzServerSideEncryption, "aws:kms")
w.Header().Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, kmsKey.KeyID)
// Verify response headers
AssertSSEKMSHeaders(t, w, kmsKey.KeyID)
}
// TestGetObjectWithSSEKMS tests GET object with SSE-KMS through HTTP handler
func TestGetObjectWithSSEKMS(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Create HTTP request for GET (no SSE headers needed for GET)
req := CreateTestHTTPRequest("GET", "/test-bucket/test-object", nil)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Create response recorder
w := CreateTestHTTPResponse()
// Simulate response with SSE-KMS headers (would come from stored metadata)
w.Header().Set(s3_constants.AmzServerSideEncryption, "aws:kms")
w.Header().Set(s3_constants.AmzServerSideEncryptionAwsKmsKeyId, kmsKey.KeyID)
w.WriteHeader(http.StatusOK)
// Verify response
if w.Code != http.StatusOK {
t.Errorf("Expected status 200, got %d", w.Code)
}
AssertSSEKMSHeaders(t, w, kmsKey.KeyID)
}
// TestSSECRangeRequestSupport tests that range requests are now supported for SSE-C
func TestSSECRangeRequestSupport(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
// Create HTTP request with Range header
req := CreateTestHTTPRequest("GET", "/test-bucket/test-object", nil)
req.Header.Set("Range", "bytes=0-100")
SetupTestSSECHeaders(req, keyPair)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Create a mock proxy response with SSE-C headers
proxyResponse := httptest.NewRecorder()
proxyResponse.Header().Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
proxyResponse.Header().Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyPair.KeyMD5)
proxyResponse.Header().Set("Content-Length", "1000")
// Test the detection logic - these should all still work
// Should detect as SSE-C request
if !IsSSECRequest(req) {
t.Error("Should detect SSE-C request")
}
// Should detect range request
if req.Header.Get("Range") == "" {
t.Error("Range header should be present")
}
// The combination should now be allowed and handled by the filer layer
// Range requests with SSE-C are now supported since IV is stored in metadata
}
// TestSSEHeaderConflicts tests conflicting SSE headers
func TestSSEHeaderConflicts(t *testing.T) {
testCases := []struct {
name string
setupFn func(*http.Request)
valid bool
}{
{
name: "SSE-C and SSE-KMS conflict",
setupFn: func(req *http.Request) {
keyPair := GenerateTestSSECKey(1)
SetupTestSSECHeaders(req, keyPair)
SetupTestSSEKMSHeaders(req, "test-key-id")
},
valid: false,
},
{
name: "Valid SSE-C only",
setupFn: func(req *http.Request) {
keyPair := GenerateTestSSECKey(1)
SetupTestSSECHeaders(req, keyPair)
},
valid: true,
},
{
name: "Valid SSE-KMS only",
setupFn: func(req *http.Request) {
SetupTestSSEKMSHeaders(req, "test-key-id")
},
valid: true,
},
{
name: "No SSE headers",
setupFn: func(req *http.Request) {
// No SSE headers
},
valid: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := CreateTestHTTPRequest("PUT", "/test-bucket/test-object", []byte("test"))
tc.setupFn(req)
ssecDetected := IsSSECRequest(req)
sseKmsDetected := IsSSEKMSRequest(req)
// Both shouldn't be detected simultaneously
if ssecDetected && sseKmsDetected {
t.Error("Both SSE-C and SSE-KMS should not be detected simultaneously")
}
// Test validation if SSE-C is detected
if ssecDetected {
err := ValidateSSECHeaders(req)
if tc.valid && err != nil {
t.Errorf("Expected valid SSE-C headers, got error: %v", err)
}
if !tc.valid && err == nil && tc.name == "SSE-C and SSE-KMS conflict" {
// This specific test case should probably be handled at a higher level
t.Log("Conflict detection should be handled by higher-level validation")
}
}
})
}
}
// TestSSECopySourceHeaders tests copy operations with SSE headers
func TestSSECopySourceHeaders(t *testing.T) {
sourceKey := GenerateTestSSECKey(1)
destKey := GenerateTestSSECKey(2)
// Create copy request with both source and destination SSE-C headers
req := CreateTestHTTPRequest("PUT", "/dest-bucket/dest-object", nil)
// Set copy source headers
SetupTestSSECCopyHeaders(req, sourceKey)
// Set destination headers
SetupTestSSECHeaders(req, destKey)
// Set copy source
req.Header.Set("X-Amz-Copy-Source", "/source-bucket/source-object")
SetupTestMuxVars(req, map[string]string{
"bucket": "dest-bucket",
"object": "dest-object",
})
// Parse copy source headers
copySourceKey, err := ParseSSECCopySourceHeaders(req)
if err != nil {
t.Fatalf("Failed to parse copy source headers: %v", err)
}
if copySourceKey == nil {
t.Fatal("Expected copy source key, got nil")
}
if !bytes.Equal(copySourceKey.Key, sourceKey.Key) {
t.Error("Copy source key doesn't match")
}
// Parse destination headers
destCustomerKey, err := ParseSSECHeaders(req)
if err != nil {
t.Fatalf("Failed to parse destination headers: %v", err)
}
if destCustomerKey == nil {
t.Fatal("Expected destination key, got nil")
}
if !bytes.Equal(destCustomerKey.Key, destKey.Key) {
t.Error("Destination key doesn't match")
}
}
// TestSSERequestValidation tests comprehensive request validation
func TestSSERequestValidation(t *testing.T) {
testCases := []struct {
name string
method string
setupFn func(*http.Request)
expectError bool
errorType string
}{
{
name: "Valid PUT with SSE-C",
method: "PUT",
setupFn: func(req *http.Request) {
keyPair := GenerateTestSSECKey(1)
SetupTestSSECHeaders(req, keyPair)
},
expectError: false,
},
{
name: "Valid GET with SSE-C",
method: "GET",
setupFn: func(req *http.Request) {
keyPair := GenerateTestSSECKey(1)
SetupTestSSECHeaders(req, keyPair)
},
expectError: false,
},
{
name: "Invalid SSE-C key format",
method: "PUT",
setupFn: func(req *http.Request) {
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, "invalid-key")
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, "invalid-md5")
},
expectError: true,
errorType: "InvalidRequest",
},
{
name: "Missing SSE-C key MD5",
method: "PUT",
setupFn: func(req *http.Request) {
keyPair := GenerateTestSSECKey(1)
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, keyPair.KeyB64)
// Missing MD5
},
expectError: true,
errorType: "InvalidRequest",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := CreateTestHTTPRequest(tc.method, "/test-bucket/test-object", []byte("test data"))
tc.setupFn(req)
SetupTestMuxVars(req, map[string]string{
"bucket": "test-bucket",
"object": "test-object",
})
// Test header validation
if IsSSECRequest(req) {
err := ValidateSSECHeaders(req)
if tc.expectError && err == nil {
t.Errorf("Expected error for %s, but got none", tc.name)
}
if !tc.expectError && err != nil {
t.Errorf("Expected no error for %s, but got: %v", tc.name, err)
}
}
})
}
}

139
weed/s3api/s3_sse_kms.go
View File

@@ -59,11 +59,6 @@ const (
// Bucket key cache TTL (moved to be used with per-bucket cache)
const BucketKeyCacheTTL = time.Hour
// CreateSSEKMSEncryptedReader creates an encrypted reader using KMS envelope encryption
func CreateSSEKMSEncryptedReader(r io.Reader, keyID string, encryptionContext map[string]string) (io.Reader, *SSEKMSKey, error) {
return CreateSSEKMSEncryptedReaderWithBucketKey(r, keyID, encryptionContext, false)
}
// CreateSSEKMSEncryptedReaderWithBucketKey creates an encrypted reader with optional S3 Bucket Keys optimization
func CreateSSEKMSEncryptedReaderWithBucketKey(r io.Reader, keyID string, encryptionContext map[string]string, bucketKeyEnabled bool) (io.Reader, *SSEKMSKey, error) {
if bucketKeyEnabled {
@@ -111,42 +106,6 @@ func CreateSSEKMSEncryptedReaderWithBucketKey(r io.Reader, keyID string, encrypt
return encryptedReader, sseKey, nil
}
// CreateSSEKMSEncryptedReaderWithBaseIV creates an SSE-KMS encrypted reader using a provided base IV
// This is used for multipart uploads where all chunks need to use the same base IV
func CreateSSEKMSEncryptedReaderWithBaseIV(r io.Reader, keyID string, encryptionContext map[string]string, bucketKeyEnabled bool, baseIV []byte) (io.Reader, *SSEKMSKey, error) {
if err := ValidateIV(baseIV, "base IV"); err != nil {
return nil, nil, err
}
// Generate data key using common utility
dataKeyResult, err := generateKMSDataKey(keyID, encryptionContext)
if err != nil {
return nil, nil, err
}
// Ensure we clear the plaintext data key from memory when done
defer clearKMSDataKey(dataKeyResult)
// Use the provided base IV instead of generating a new one
iv := make([]byte, s3_constants.AESBlockSize)
copy(iv, baseIV)
// Create CTR mode cipher stream
stream := cipher.NewCTR(dataKeyResult.Block, iv)
// Create the SSE-KMS metadata using utility function
sseKey := createSSEKMSKey(dataKeyResult, encryptionContext, bucketKeyEnabled, iv, 0)
// The IV is stored in SSE key metadata, so the encrypted stream does not need to prepend the IV
// This ensures correct Content-Length for clients
encryptedReader := &cipher.StreamReader{S: stream, R: r}
// Store the base IV in the SSE key for metadata storage
sseKey.IV = iv
return encryptedReader, sseKey, nil
}
// CreateSSEKMSEncryptedReaderWithBaseIVAndOffset creates an SSE-KMS encrypted reader using a provided base IV and offset
// This is used for multipart uploads where all chunks need unique IVs to prevent IV reuse vulnerabilities
func CreateSSEKMSEncryptedReaderWithBaseIVAndOffset(r io.Reader, keyID string, encryptionContext map[string]string, bucketKeyEnabled bool, baseIV []byte, offset int64) (io.Reader, *SSEKMSKey, error) {
@@ -453,67 +412,6 @@ func CreateSSEKMSDecryptedReader(r io.Reader, sseKey *SSEKMSKey) (io.Reader, err
return decryptReader, nil
}
// ParseSSEKMSHeaders parses SSE-KMS headers from an HTTP request
func ParseSSEKMSHeaders(r *http.Request) (*SSEKMSKey, error) {
sseAlgorithm := r.Header.Get(s3_constants.AmzServerSideEncryption)
// Check if SSE-KMS is requested
if sseAlgorithm == "" {
return nil, nil // No SSE headers present
}
if sseAlgorithm != s3_constants.SSEAlgorithmKMS {
return nil, fmt.Errorf("invalid SSE algorithm: %s", sseAlgorithm)
}
keyID := r.Header.Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId)
encryptionContextHeader := r.Header.Get(s3_constants.AmzServerSideEncryptionContext)
bucketKeyEnabledHeader := r.Header.Get(s3_constants.AmzServerSideEncryptionBucketKeyEnabled)
// Parse encryption context if provided
var encryptionContext map[string]string
if encryptionContextHeader != "" {
// Decode base64-encoded JSON encryption context
contextBytes, err := base64.StdEncoding.DecodeString(encryptionContextHeader)
if err != nil {
return nil, fmt.Errorf("invalid encryption context format: %v", err)
}
if err := json.Unmarshal(contextBytes, &encryptionContext); err != nil {
return nil, fmt.Errorf("invalid encryption context JSON: %v", err)
}
}
// Parse bucket key enabled flag
bucketKeyEnabled := strings.ToLower(bucketKeyEnabledHeader) == "true"
sseKey := &SSEKMSKey{
KeyID: keyID,
EncryptionContext: encryptionContext,
BucketKeyEnabled: bucketKeyEnabled,
}
// Validate the parsed key including key ID format
if err := ValidateSSEKMSKeyInternal(sseKey); err != nil {
return nil, err
}
return sseKey, nil
}
// ValidateSSEKMSKey validates an SSE-KMS key configuration
func ValidateSSEKMSKeyInternal(sseKey *SSEKMSKey) error {
if err := ValidateSSEKMSKey(sseKey); err != nil {
return err
}
// An empty key ID is valid and means the default KMS key should be used.
if sseKey.KeyID != "" && !isValidKMSKeyID(sseKey.KeyID) {
return fmt.Errorf("invalid KMS key ID format: %s", sseKey.KeyID)
}
return nil
}
// BuildEncryptionContext creates the encryption context for S3 objects
func BuildEncryptionContext(bucketName, objectKey string, useBucketKey bool) map[string]string {
return kms.BuildS3EncryptionContext(bucketName, objectKey, useBucketKey)
@@ -732,28 +630,6 @@ func IsSSEKMSEncrypted(metadata map[string][]byte) bool {
return false
}
// IsAnySSEEncrypted checks if metadata indicates any type of SSE encryption
func IsAnySSEEncrypted(metadata map[string][]byte) bool {
if metadata == nil {
return false
}
// Check for any SSE type
if IsSSECEncrypted(metadata) {
return true
}
if IsSSEKMSEncrypted(metadata) {
return true
}
// Check for SSE-S3
if sseAlgorithm, exists := metadata[s3_constants.AmzServerSideEncryption]; exists {
return string(sseAlgorithm) == s3_constants.SSEAlgorithmAES256
}
return false
}
// MapKMSErrorToS3Error maps KMS errors to appropriate S3 error codes
func MapKMSErrorToS3Error(err error) s3err.ErrorCode {
if err == nil {
@@ -990,21 +866,6 @@ func DetermineUnifiedCopyStrategy(state *EncryptionState, srcMetadata map[string
return CopyStrategyDirect, nil
}
// DetectEncryptionState analyzes the source metadata and request headers to determine encryption state
func DetectEncryptionState(srcMetadata map[string][]byte, r *http.Request, srcPath, dstPath string) *EncryptionState {
state := &EncryptionState{
SrcSSEC: IsSSECEncrypted(srcMetadata),
SrcSSEKMS: IsSSEKMSEncrypted(srcMetadata),
SrcSSES3: IsSSES3EncryptedInternal(srcMetadata),
DstSSEC: IsSSECRequest(r),
DstSSEKMS: IsSSEKMSRequest(r),
DstSSES3: IsSSES3RequestInternal(r),
SameObject: srcPath == dstPath,
}
return state
}
// DetectEncryptionStateWithEntry analyzes the source entry and request headers to determine encryption state
// This version can detect multipart encrypted objects by examining chunks
func DetectEncryptionStateWithEntry(entry *filer_pb.Entry, r *http.Request, srcPath, dstPath string) *EncryptionState {

399
weed/s3api/s3_sse_kms_test.go
View File

@@ -1,399 +0,0 @@
package s3api
import (
"bytes"
"encoding/json"
"io"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/kms"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
func TestSSEKMSEncryptionDecryption(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Test data
testData := "Hello, SSE-KMS world! This is a test of envelope encryption."
testReader := strings.NewReader(testData)
// Create encryption context
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
// Encrypt the data
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(testReader, kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Verify SSE key metadata
if sseKey.KeyID != kmsKey.KeyID {
t.Errorf("Expected key ID %s, got %s", kmsKey.KeyID, sseKey.KeyID)
}
if len(sseKey.EncryptedDataKey) == 0 {
t.Error("Encrypted data key should not be empty")
}
if sseKey.EncryptionContext == nil {
t.Error("Encryption context should not be nil")
}
// Read the encrypted data
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Verify the encrypted data is different from original
if string(encryptedData) == testData {
t.Error("Encrypted data should be different from original data")
}
// The encrypted data should be same size as original (IV is stored in metadata, not in stream)
if len(encryptedData) != len(testData) {
t.Errorf("Encrypted data should be same size as original: expected %d, got %d", len(testData), len(encryptedData))
}
// Decrypt the data
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Read the decrypted data
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
// Verify the decrypted data matches the original
if string(decryptedData) != testData {
t.Errorf("Decrypted data does not match original.\nExpected: %s\nGot: %s", testData, string(decryptedData))
}
}
func TestSSEKMSKeyValidation(t *testing.T) {
tests := []struct {
name string
keyID string
wantValid bool
}{
{
name: "Valid UUID key ID",
keyID: "12345678-1234-1234-1234-123456789012",
wantValid: true,
},
{
name: "Valid alias",
keyID: "alias/my-test-key",
wantValid: true,
},
{
name: "Valid ARN",
keyID: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012",
wantValid: true,
},
{
name: "Valid alias ARN",
keyID: "arn:aws:kms:us-east-1:123456789012:alias/my-test-key",
wantValid: true,
},
{
name: "Valid test key format",
keyID: "invalid-key-format",
wantValid: true, // Now valid - following Minio's permissive approach
},
{
name: "Valid short key",
keyID: "12345678-1234",
wantValid: true, // Now valid - following Minio's permissive approach
},
{
name: "Invalid - leading space",
keyID: " leading-space",
wantValid: false,
},
{
name: "Invalid - trailing space",
keyID: "trailing-space ",
wantValid: false,
},
{
name: "Invalid - empty",
keyID: "",
wantValid: false,
},
{
name: "Invalid - internal spaces",
keyID: "invalid key id",
wantValid: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
valid := isValidKMSKeyID(tt.keyID)
if valid != tt.wantValid {
t.Errorf("isValidKMSKeyID(%s) = %v, want %v", tt.keyID, valid, tt.wantValid)
}
})
}
}
func TestSSEKMSMetadataSerialization(t *testing.T) {
// Create test SSE key
sseKey := &SSEKMSKey{
KeyID: "test-key-id",
EncryptedDataKey: []byte("encrypted-data-key"),
EncryptionContext: map[string]string{
"aws:s3:arn": "arn:aws:s3:::test-bucket/test-object",
},
BucketKeyEnabled: true,
}
// Serialize metadata
serialized, err := SerializeSSEKMSMetadata(sseKey)
if err != nil {
t.Fatalf("Failed to serialize SSE-KMS metadata: %v", err)
}
// Verify it's valid JSON
var jsonData map[string]interface{}
if err := json.Unmarshal(serialized, &jsonData); err != nil {
t.Fatalf("Serialized data is not valid JSON: %v", err)
}
// Deserialize metadata
deserializedKey, err := DeserializeSSEKMSMetadata(serialized)
if err != nil {
t.Fatalf("Failed to deserialize SSE-KMS metadata: %v", err)
}
// Verify the deserialized data matches original
if deserializedKey.KeyID != sseKey.KeyID {
t.Errorf("KeyID mismatch: expected %s, got %s", sseKey.KeyID, deserializedKey.KeyID)
}
if !bytes.Equal(deserializedKey.EncryptedDataKey, sseKey.EncryptedDataKey) {
t.Error("EncryptedDataKey mismatch")
}
if len(deserializedKey.EncryptionContext) != len(sseKey.EncryptionContext) {
t.Error("EncryptionContext length mismatch")
}
for k, v := range sseKey.EncryptionContext {
if deserializedKey.EncryptionContext[k] != v {
t.Errorf("EncryptionContext mismatch for key %s: expected %s, got %s", k, v, deserializedKey.EncryptionContext[k])
}
}
if deserializedKey.BucketKeyEnabled != sseKey.BucketKeyEnabled {
t.Errorf("BucketKeyEnabled mismatch: expected %v, got %v", sseKey.BucketKeyEnabled, deserializedKey.BucketKeyEnabled)
}
}
func TestBuildEncryptionContext(t *testing.T) {
tests := []struct {
name string
bucket string
object string
useBucketKey bool
expectedARN string
}{
{
name: "Object-level encryption",
bucket: "test-bucket",
object: "test-object",
useBucketKey: false,
expectedARN: "arn:aws:s3:::test-bucket/test-object",
},
{
name: "Bucket-level encryption",
bucket: "test-bucket",
object: "test-object",
useBucketKey: true,
expectedARN: "arn:aws:s3:::test-bucket",
},
{
name: "Nested object path",
bucket: "my-bucket",
object: "folder/subfolder/file.txt",
useBucketKey: false,
expectedARN: "arn:aws:s3:::my-bucket/folder/subfolder/file.txt",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
context := BuildEncryptionContext(tt.bucket, tt.object, tt.useBucketKey)
if context == nil {
t.Fatal("Encryption context should not be nil")
}
arn, exists := context[kms.EncryptionContextS3ARN]
if !exists {
t.Error("Encryption context should contain S3 ARN")
}
if arn != tt.expectedARN {
t.Errorf("Expected ARN %s, got %s", tt.expectedARN, arn)
}
})
}
}
func TestKMSErrorMapping(t *testing.T) {
tests := []struct {
name string
kmsError *kms.KMSError
expectedErr string
}{
{
name: "Key not found",
kmsError: &kms.KMSError{
Code: kms.ErrCodeNotFoundException,
Message: "Key not found",
},
expectedErr: "KMSKeyNotFoundException",
},
{
name: "Access denied",
kmsError: &kms.KMSError{
Code: kms.ErrCodeAccessDenied,
Message: "Access denied",
},
expectedErr: "KMSAccessDeniedException",
},
{
name: "Key unavailable",
kmsError: &kms.KMSError{
Code: kms.ErrCodeKeyUnavailable,
Message: "Key is disabled",
},
expectedErr: "KMSKeyDisabledException",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
errorCode := MapKMSErrorToS3Error(tt.kmsError)
// Get the actual error description
apiError := s3err.GetAPIError(errorCode)
if apiError.Code != tt.expectedErr {
t.Errorf("Expected error code %s, got %s", tt.expectedErr, apiError.Code)
}
})
}
}
// TestLargeDataEncryption tests encryption/decryption of larger data streams
func TestSSEKMSLargeDataEncryption(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Create a larger test dataset (1MB)
testData := strings.Repeat("This is a test of SSE-KMS with larger data streams. ", 20000)
testReader := strings.NewReader(testData)
// Create encryption context
encryptionContext := BuildEncryptionContext("large-bucket", "large-object", false)
// Encrypt the data
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(testReader, kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Read the encrypted data
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Decrypt the data
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
// Read the decrypted data
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
// Verify the decrypted data matches the original
if string(decryptedData) != testData {
t.Errorf("Decrypted data length: %d, original data length: %d", len(decryptedData), len(testData))
t.Error("Decrypted large data does not match original")
}
t.Logf("Successfully encrypted/decrypted %d bytes of data", len(testData))
}
// TestValidateSSEKMSKey tests the ValidateSSEKMSKey function, which correctly handles empty key IDs
func TestValidateSSEKMSKey(t *testing.T) {
tests := []struct {
name string
sseKey *SSEKMSKey
wantErr bool
}{
{
name: "nil SSE-KMS key",
sseKey: nil,
wantErr: true,
},
{
name: "empty key ID (valid - represents default KMS key)",
sseKey: &SSEKMSKey{
KeyID: "",
EncryptionContext: map[string]string{"test": "value"},
BucketKeyEnabled: false,
},
wantErr: false,
},
{
name: "valid UUID key ID",
sseKey: &SSEKMSKey{
KeyID: "12345678-1234-1234-1234-123456789012",
EncryptionContext: map[string]string{"test": "value"},
BucketKeyEnabled: true,
},
wantErr: false,
},
{
name: "valid alias",
sseKey: &SSEKMSKey{
KeyID: "alias/my-test-key",
EncryptionContext: map[string]string{},
BucketKeyEnabled: false,
},
wantErr: false,
},
{
name: "valid flexible key ID format",
sseKey: &SSEKMSKey{
KeyID: "invalid-format",
EncryptionContext: map[string]string{},
BucketKeyEnabled: false,
},
wantErr: false, // Now valid - following Minio's permissive approach
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateSSEKMSKey(tt.sseKey)
if (err != nil) != tt.wantErr {
t.Errorf("ValidateSSEKMSKey() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

328
weed/s3api/s3_sse_metadata_test.go
View File

@@ -1,328 +0,0 @@
package s3api
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// TestSSECIsEncrypted tests detection of SSE-C encryption from metadata
func TestSSECIsEncrypted(t *testing.T) {
testCases := []struct {
name string
metadata map[string][]byte
expected bool
}{
{
name: "Empty metadata",
metadata: CreateTestMetadata(),
expected: false,
},
{
name: "Valid SSE-C metadata",
metadata: CreateTestMetadataWithSSEC(GenerateTestSSECKey(1)),
expected: true,
},
{
name: "SSE-C algorithm only",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"),
},
expected: true,
},
{
name: "SSE-C key MD5 only",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerKeyMD5: []byte("somemd5"),
},
expected: true,
},
{
name: "Other encryption type (SSE-KMS)",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
},
expected: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := IsSSECEncrypted(tc.metadata)
if result != tc.expected {
t.Errorf("Expected %v, got %v", tc.expected, result)
}
})
}
}
// TestSSEKMSIsEncrypted tests detection of SSE-KMS encryption from metadata
func TestSSEKMSIsEncrypted(t *testing.T) {
testCases := []struct {
name string
metadata map[string][]byte
expected bool
}{
{
name: "Empty metadata",
metadata: CreateTestMetadata(),
expected: false,
},
{
name: "Valid SSE-KMS metadata",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzEncryptedDataKey: []byte("encrypted-key"),
},
expected: true,
},
{
name: "SSE-KMS algorithm only",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
},
expected: true,
},
{
name: "SSE-KMS encrypted data key only",
metadata: map[string][]byte{
s3_constants.AmzEncryptedDataKey: []byte("encrypted-key"),
},
expected: false, // Only encrypted data key without algorithm header should not be considered SSE-KMS
},
{
name: "Other encryption type (SSE-C)",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"),
},
expected: false,
},
{
name: "SSE-S3 (AES256)",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
},
expected: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := IsSSEKMSEncrypted(tc.metadata)
if result != tc.expected {
t.Errorf("Expected %v, got %v", tc.expected, result)
}
})
}
}
// TestSSETypeDiscrimination tests that SSE types don't interfere with each other
func TestSSETypeDiscrimination(t *testing.T) {
// Test SSE-C headers don't trigger SSE-KMS detection
t.Run("SSE-C headers don't trigger SSE-KMS", func(t *testing.T) {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
keyPair := GenerateTestSSECKey(1)
SetupTestSSECHeaders(req, keyPair)
// Should detect SSE-C, not SSE-KMS
if !IsSSECRequest(req) {
t.Error("Should detect SSE-C request")
}
if IsSSEKMSRequest(req) {
t.Error("Should not detect SSE-KMS request for SSE-C headers")
}
})
// Test SSE-KMS headers don't trigger SSE-C detection
t.Run("SSE-KMS headers don't trigger SSE-C", func(t *testing.T) {
req := CreateTestHTTPRequest("PUT", "/bucket/object", nil)
SetupTestSSEKMSHeaders(req, "test-key-id")
// Should detect SSE-KMS, not SSE-C
if IsSSECRequest(req) {
t.Error("Should not detect SSE-C request for SSE-KMS headers")
}
if !IsSSEKMSRequest(req) {
t.Error("Should detect SSE-KMS request")
}
})
// Test metadata discrimination
t.Run("Metadata type discrimination", func(t *testing.T) {
ssecMetadata := CreateTestMetadataWithSSEC(GenerateTestSSECKey(1))
// Should detect as SSE-C, not SSE-KMS
if !IsSSECEncrypted(ssecMetadata) {
t.Error("Should detect SSE-C encrypted metadata")
}
if IsSSEKMSEncrypted(ssecMetadata) {
t.Error("Should not detect SSE-KMS for SSE-C metadata")
}
})
}
// TestSSECParseCorruptedMetadata tests handling of corrupted SSE-C metadata
func TestSSECParseCorruptedMetadata(t *testing.T) {
testCases := []struct {
name string
metadata map[string][]byte
expectError bool
errorMessage string
}{
{
name: "Missing algorithm",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerKeyMD5: []byte("valid-md5"),
},
expectError: false, // Detection should still work with partial metadata
},
{
name: "Invalid key MD5 format",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte("AES256"),
s3_constants.AmzServerSideEncryptionCustomerKeyMD5: []byte("invalid-base64!"),
},
expectError: false, // Detection should work, validation happens later
},
{
name: "Empty values",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryptionCustomerAlgorithm: []byte(""),
s3_constants.AmzServerSideEncryptionCustomerKeyMD5: []byte(""),
},
expectError: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Test that detection doesn't panic on corrupted metadata
result := IsSSECEncrypted(tc.metadata)
// The detection should be robust and not crash
t.Logf("Detection result for %s: %v", tc.name, result)
})
}
}
// TestSSEKMSParseCorruptedMetadata tests handling of corrupted SSE-KMS metadata
func TestSSEKMSParseCorruptedMetadata(t *testing.T) {
testCases := []struct {
name string
metadata map[string][]byte
}{
{
name: "Invalid encrypted data key",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzEncryptedDataKey: []byte("invalid-base64!"),
},
},
{
name: "Invalid encryption context",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
s3_constants.AmzEncryptionContextMeta: []byte("invalid-json"),
},
},
{
name: "Empty values",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte(""),
s3_constants.AmzEncryptedDataKey: []byte(""),
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Test that detection doesn't panic on corrupted metadata
result := IsSSEKMSEncrypted(tc.metadata)
t.Logf("Detection result for %s: %v", tc.name, result)
})
}
}
// TestSSEMetadataDeserialization tests SSE-KMS metadata deserialization with various inputs
func TestSSEMetadataDeserialization(t *testing.T) {
testCases := []struct {
name string
data []byte
expectError bool
}{
{
name: "Empty data",
data: []byte{},
expectError: true,
},
{
name: "Invalid JSON",
data: []byte("invalid-json"),
expectError: true,
},
{
name: "Valid JSON but wrong structure",
data: []byte(`{"wrong": "structure"}`),
expectError: false, // Our deserialization might be lenient
},
{
name: "Null data",
data: nil,
expectError: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
_, err := DeserializeSSEKMSMetadata(tc.data)
if tc.expectError && err == nil {
t.Error("Expected error but got none")
}
if !tc.expectError && err != nil {
t.Errorf("Expected no error but got: %v", err)
}
})
}
}
// TestGeneralSSEDetection tests the general SSE detection that works across types
func TestGeneralSSEDetection(t *testing.T) {
testCases := []struct {
name string
metadata map[string][]byte
expected bool
}{
{
name: "No encryption",
metadata: CreateTestMetadata(),
expected: false,
},
{
name: "SSE-C encrypted",
metadata: CreateTestMetadataWithSSEC(GenerateTestSSECKey(1)),
expected: true,
},
{
name: "SSE-KMS encrypted",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
},
expected: true,
},
{
name: "SSE-S3 encrypted",
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
},
expected: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := IsAnySSEEncrypted(tc.metadata)
if result != tc.expected {
t.Errorf("Expected %v, got %v", tc.expected, result)
}
})
}
}

569
weed/s3api/s3_sse_multipart_test.go
View File

@@ -1,569 +0,0 @@
package s3api
import (
"bytes"
"fmt"
"io"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// TestSSECMultipartUpload tests SSE-C with multipart uploads
func TestSSECMultipartUpload(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
// Test data larger than typical part size
testData := strings.Repeat("Hello, SSE-C multipart world! ", 1000) // ~30KB
t.Run("Single part encryption/decryption", func(t *testing.T) {
// Encrypt the data
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(testData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Decrypt the data
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
if string(decryptedData) != testData {
t.Error("Decrypted data doesn't match original")
}
})
t.Run("Simulated multipart upload parts", func(t *testing.T) {
// Simulate multiple parts (each part gets encrypted separately)
partSize := 5 * 1024 // 5KB parts
var encryptedParts [][]byte
var partIVs [][]byte
for i := 0; i < len(testData); i += partSize {
end := i + partSize
if end > len(testData) {
end = len(testData)
}
partData := testData[i:end]
// Each part is encrypted separately in multipart uploads
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(partData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for part %d: %v", i/partSize, err)
}
encryptedPart, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted part %d: %v", i/partSize, err)
}
encryptedParts = append(encryptedParts, encryptedPart)
partIVs = append(partIVs, iv)
}
// Simulate reading back the multipart object
var reconstructedData strings.Builder
for i, encryptedPart := range encryptedParts {
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedPart), customerKey, partIVs[i])
if err != nil {
t.Fatalf("Failed to create decrypted reader for part %d: %v", i, err)
}
decryptedPart, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted part %d: %v", i, err)
}
reconstructedData.Write(decryptedPart)
}
if reconstructedData.String() != testData {
t.Error("Reconstructed multipart data doesn't match original")
}
})
t.Run("Multipart with different part sizes", func(t *testing.T) {
partSizes := []int{1024, 2048, 4096, 8192} // Various part sizes
for _, partSize := range partSizes {
t.Run(fmt.Sprintf("PartSize_%d", partSize), func(t *testing.T) {
var encryptedParts [][]byte
var partIVs [][]byte
for i := 0; i < len(testData); i += partSize {
end := i + partSize
if end > len(testData) {
end = len(testData)
}
partData := testData[i:end]
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(partData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedPart, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted part: %v", err)
}
encryptedParts = append(encryptedParts, encryptedPart)
partIVs = append(partIVs, iv)
}
// Verify reconstruction
var reconstructedData strings.Builder
for j, encryptedPart := range encryptedParts {
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedPart), customerKey, partIVs[j])
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
decryptedPart, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted part: %v", err)
}
reconstructedData.Write(decryptedPart)
}
if reconstructedData.String() != testData {
t.Errorf("Reconstructed data doesn't match original for part size %d", partSize)
}
})
}
})
}
// TestSSEKMSMultipartUpload tests SSE-KMS with multipart uploads
func TestSSEKMSMultipartUpload(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Test data larger than typical part size
testData := strings.Repeat("Hello, SSE-KMS multipart world! ", 1000) // ~30KB
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
t.Run("Single part encryption/decryption", func(t *testing.T) {
// Encrypt the data
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(testData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
// Decrypt the data
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data: %v", err)
}
if string(decryptedData) != testData {
t.Error("Decrypted data doesn't match original")
}
})
t.Run("Simulated multipart upload parts", func(t *testing.T) {
// Simulate multiple parts (each part might use the same or different KMS operations)
partSize := 5 * 1024 // 5KB parts
var encryptedParts [][]byte
var sseKeys []*SSEKMSKey
for i := 0; i < len(testData); i += partSize {
end := i + partSize
if end > len(testData) {
end = len(testData)
}
partData := testData[i:end]
// Each part might get its own data key in KMS multipart uploads
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(partData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader for part %d: %v", i/partSize, err)
}
encryptedPart, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted part %d: %v", i/partSize, err)
}
encryptedParts = append(encryptedParts, encryptedPart)
sseKeys = append(sseKeys, sseKey)
}
// Simulate reading back the multipart object
var reconstructedData strings.Builder
for i, encryptedPart := range encryptedParts {
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedPart), sseKeys[i])
if err != nil {
t.Fatalf("Failed to create decrypted reader for part %d: %v", i, err)
}
decryptedPart, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted part %d: %v", i, err)
}
reconstructedData.Write(decryptedPart)
}
if reconstructedData.String() != testData {
t.Error("Reconstructed multipart data doesn't match original")
}
})
t.Run("Multipart consistency checks", func(t *testing.T) {
// Test that all parts use the same KMS key ID but different data keys
partSize := 5 * 1024
var sseKeys []*SSEKMSKey
for i := 0; i < len(testData); i += partSize {
end := i + partSize
if end > len(testData) {
end = len(testData)
}
partData := testData[i:end]
_, sseKey, err := CreateSSEKMSEncryptedReader(strings.NewReader(partData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
sseKeys = append(sseKeys, sseKey)
}
// Verify all parts use the same KMS key ID
for i, sseKey := range sseKeys {
if sseKey.KeyID != kmsKey.KeyID {
t.Errorf("Part %d has wrong KMS key ID: expected %s, got %s", i, kmsKey.KeyID, sseKey.KeyID)
}
}
// Verify each part has different encrypted data keys (they should be unique)
for i := 0; i < len(sseKeys); i++ {
for j := i + 1; j < len(sseKeys); j++ {
if bytes.Equal(sseKeys[i].EncryptedDataKey, sseKeys[j].EncryptedDataKey) {
t.Errorf("Parts %d and %d have identical encrypted data keys (should be unique)", i, j)
}
}
}
})
}
// TestMultipartSSEMixedScenarios tests edge cases with multipart and SSE
func TestMultipartSSEMixedScenarios(t *testing.T) {
t.Run("Empty parts handling", func(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
// Test empty part
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(""), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for empty data: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted empty data: %v", err)
}
// Empty part should produce empty encrypted data, but still have a valid IV
if len(encryptedData) != 0 {
t.Errorf("Expected empty encrypted data for empty part, got %d bytes", len(encryptedData))
}
if len(iv) != s3_constants.AESBlockSize {
t.Errorf("Expected IV of size %d, got %d", s3_constants.AESBlockSize, len(iv))
}
// Decrypt and verify
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader for empty data: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted empty data: %v", err)
}
if len(decryptedData) != 0 {
t.Errorf("Expected empty decrypted data, got %d bytes", len(decryptedData))
}
})
t.Run("Single byte parts", func(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
testData := "ABCDEFGHIJ"
var encryptedParts [][]byte
var partIVs [][]byte
// Encrypt each byte as a separate part
for i, b := range []byte(testData) {
partData := string(b)
encryptedReader, iv, err := CreateSSECEncryptedReader(strings.NewReader(partData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for byte %d: %v", i, err)
}
encryptedPart, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted byte %d: %v", i, err)
}
encryptedParts = append(encryptedParts, encryptedPart)
partIVs = append(partIVs, iv)
}
// Reconstruct
var reconstructedData strings.Builder
for i, encryptedPart := range encryptedParts {
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedPart), customerKey, partIVs[i])
if err != nil {
t.Fatalf("Failed to create decrypted reader for byte %d: %v", i, err)
}
decryptedPart, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted byte %d: %v", i, err)
}
reconstructedData.Write(decryptedPart)
}
if reconstructedData.String() != testData {
t.Errorf("Expected %s, got %s", testData, reconstructedData.String())
}
})
t.Run("Very large parts", func(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
// Create a large part (1MB)
largeData := make([]byte, 1024*1024)
for i := range largeData {
largeData[i] = byte(i % 256)
}
// Encrypt
encryptedReader, iv, err := CreateSSECEncryptedReader(bytes.NewReader(largeData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for large data: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted large data: %v", err)
}
// Decrypt
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader for large data: %v", err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted large data: %v", err)
}
if !bytes.Equal(decryptedData, largeData) {
t.Error("Large data doesn't match after encryption/decryption")
}
})
}
func TestSSECLargeObjectChunkReassembly(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
const chunkSize = 8 * 1024 * 1024 // matches putToFiler chunk size
totalSize := chunkSize*2 + 3*1024*1024
plaintext := make([]byte, totalSize)
for i := range plaintext {
plaintext[i] = byte(i % 251)
}
encryptedReader, iv, err := CreateSSECEncryptedReader(bytes.NewReader(plaintext), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data: %v", err)
}
var reconstructed bytes.Buffer
offset := int64(0)
for offset < int64(len(encryptedData)) {
end := offset + chunkSize
if end > int64(len(encryptedData)) {
end = int64(len(encryptedData))
}
chunkIV := make([]byte, len(iv))
copy(chunkIV, iv)
chunkReader := bytes.NewReader(encryptedData[offset:end])
decryptedReader, decErr := CreateSSECDecryptedReaderWithOffset(chunkReader, customerKey, chunkIV, uint64(offset))
if decErr != nil {
t.Fatalf("Failed to create decrypted reader for offset %d: %v", offset, decErr)
}
decryptedChunk, decErr := io.ReadAll(decryptedReader)
if decErr != nil {
t.Fatalf("Failed to read decrypted chunk at offset %d: %v", offset, decErr)
}
reconstructed.Write(decryptedChunk)
offset = end
}
if !bytes.Equal(reconstructed.Bytes(), plaintext) {
t.Fatalf("Reconstructed data mismatch: expected %d bytes, got %d", len(plaintext), reconstructed.Len())
}
}
// TestMultipartSSEPerformance tests performance characteristics of SSE with multipart
func TestMultipartSSEPerformance(t *testing.T) {
if testing.Short() {
t.Skip("Skipping performance test in short mode")
}
t.Run("SSE-C performance with multiple parts", func(t *testing.T) {
keyPair := GenerateTestSSECKey(1)
customerKey := &SSECustomerKey{
Algorithm: "AES256",
Key: keyPair.Key,
KeyMD5: keyPair.KeyMD5,
}
partSize := 64 * 1024 // 64KB parts
numParts := 10
for partNum := 0; partNum < numParts; partNum++ {
partData := make([]byte, partSize)
for i := range partData {
partData[i] = byte((partNum + i) % 256)
}
// Encrypt
encryptedReader, iv, err := CreateSSECEncryptedReader(bytes.NewReader(partData), customerKey)
if err != nil {
t.Fatalf("Failed to create encrypted reader for part %d: %v", partNum, err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data for part %d: %v", partNum, err)
}
// Decrypt
decryptedReader, err := CreateSSECDecryptedReader(bytes.NewReader(encryptedData), customerKey, iv)
if err != nil {
t.Fatalf("Failed to create decrypted reader for part %d: %v", partNum, err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data for part %d: %v", partNum, err)
}
if !bytes.Equal(decryptedData, partData) {
t.Errorf("Data mismatch for part %d", partNum)
}
}
})
t.Run("SSE-KMS performance with multiple parts", func(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
partSize := 64 * 1024 // 64KB parts
numParts := 5 // Fewer parts for KMS due to overhead
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
for partNum := 0; partNum < numParts; partNum++ {
partData := make([]byte, partSize)
for i := range partData {
partData[i] = byte((partNum + i) % 256)
}
// Encrypt
encryptedReader, sseKey, err := CreateSSEKMSEncryptedReader(bytes.NewReader(partData), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader for part %d: %v", partNum, err)
}
encryptedData, err := io.ReadAll(encryptedReader)
if err != nil {
t.Fatalf("Failed to read encrypted data for part %d: %v", partNum, err)
}
// Decrypt
decryptedReader, err := CreateSSEKMSDecryptedReader(bytes.NewReader(encryptedData), sseKey)
if err != nil {
t.Fatalf("Failed to create decrypted reader for part %d: %v", partNum, err)
}
decryptedData, err := io.ReadAll(decryptedReader)
if err != nil {
t.Fatalf("Failed to read decrypted data for part %d: %v", partNum, err)
}
if !bytes.Equal(decryptedData, partData) {
t.Errorf("Data mismatch for part %d", partNum)
}
}
})
}

59
weed/s3api/s3_sse_s3.go
View File

@@ -137,13 +137,6 @@ func CreateSSES3DecryptedReader(reader io.Reader, key *SSES3Key, iv []byte) (io.
return decryptReader, nil
}
// GetSSES3Headers returns the headers for SSE-S3 encrypted objects
func GetSSES3Headers() map[string]string {
return map[string]string{
s3_constants.AmzServerSideEncryption: SSES3Algorithm,
}
}
// SerializeSSES3Metadata serializes SSE-S3 metadata for storage using envelope encryption
func SerializeSSES3Metadata(key *SSES3Key) ([]byte, error) {
if err := ValidateSSES3Key(key); err != nil {
@@ -339,7 +332,7 @@ func (km *SSES3KeyManager) InitializeWithFiler(filerClient filer_pb.FilerClient)
v := util.GetViper()
cfgKEK := v.GetString(sseS3KEKConfigKey) // hex-encoded, drop-in for filer file
cfgKey := v.GetString(sseS3KeyConfigKey) // any string, HKDF-derived
cfgKey := v.GetString(sseS3KeyConfigKey) // any string, HKDF-derived
if cfgKEK != "" && cfgKey != "" {
return fmt.Errorf("only one of %s and %s may be set, not both", sseS3KEKConfigKey, sseS3KeyConfigKey)
@@ -454,7 +447,6 @@ func (km *SSES3KeyManager) loadSuperKeyFromFiler() error {
return nil
}
// GetOrCreateKey gets an existing key or creates a new one
// With envelope encryption, we always generate a new DEK since we don't store them
func (km *SSES3KeyManager) GetOrCreateKey(keyID string) (*SSES3Key, error) {
@@ -532,14 +524,6 @@ func (km *SSES3KeyManager) StoreKey(key *SSES3Key) {
// The DEK is encrypted with the super key and stored in object metadata
}
// GetKey is now a no-op since we don't cache keys
// Keys are retrieved by decrypting the encrypted DEK from object metadata
func (km *SSES3KeyManager) GetKey(keyID string) (*SSES3Key, bool) {
// No-op: With envelope encryption, keys are not cached
// Each object's metadata contains the encrypted DEK
return nil, false
}
// GetMasterKey returns a derived key from the master KEK for STS signing
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
func (km *SSES3KeyManager) GetMasterKey() []byte {
@@ -596,47 +580,6 @@ func InitializeGlobalSSES3KeyManager(filerClient *wdclient.FilerClient, grpcDial
return globalSSES3KeyManager.InitializeWithFiler(wrapper)
}
// ProcessSSES3Request processes an SSE-S3 request and returns encryption metadata
func ProcessSSES3Request(r *http.Request) (map[string][]byte, error) {
if !IsSSES3RequestInternal(r) {
return nil, nil
}
// Generate or retrieve encryption key
keyManager := GetSSES3KeyManager()
key, err := keyManager.GetOrCreateKey("")
if err != nil {
return nil, fmt.Errorf("get SSE-S3 key: %w", err)
}
// Serialize key metadata
keyData, err := SerializeSSES3Metadata(key)
if err != nil {
return nil, fmt.Errorf("serialize SSE-S3 metadata: %w", err)
}
// Store key in manager
keyManager.StoreKey(key)
// Return metadata
metadata := map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte(SSES3Algorithm),
s3_constants.SeaweedFSSSES3Key: keyData,
}
return metadata, nil
}
// GetSSES3KeyFromMetadata extracts SSE-S3 key from object metadata
func GetSSES3KeyFromMetadata(metadata map[string][]byte, keyManager *SSES3KeyManager) (*SSES3Key, error) {
keyData, exists := metadata[s3_constants.SeaweedFSSSES3Key]
if !exists {
return nil, fmt.Errorf("SSE-S3 key not found in metadata")
}
return DeserializeSSES3Metadata(keyData, keyManager)
}
// GetSSES3IV extracts the IV for single-part SSE-S3 objects
// Priority: 1) object-level metadata (for inline/small files), 2) first chunk metadata
func GetSSES3IV(entry *filer_pb.Entry, sseS3Key *SSES3Key, keyManager *SSES3KeyManager) ([]byte, error) {

1079
weed/s3api/s3_sse_s3_test.go
View File

File diff suppressed because it is too large Load Diff

8
weed/s3api/s3_validation_utils.go
View File

@@ -58,14 +58,6 @@ func ValidateSSEKMSKey(sseKey *SSEKMSKey) error {
return nil
}
// ValidateSSECKey validates that an SSE-C key is not nil
func ValidateSSECKey(customerKey *SSECustomerKey) error {
if customerKey == nil {
return fmt.Errorf("SSE-C customer key cannot be nil")
}
return nil
}
// ValidateSSES3Key validates that an SSE-S3 key has valid structure and contents
func ValidateSSES3Key(sseKey *SSES3Key) error {
if sseKey == nil {

166
weed/s3api/s3api_acl_helper.go
View File

@@ -20,16 +20,6 @@ type AccountManager interface {
GetAccountIdByEmail(email string) string
}
// GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen
func GetAccountId(r *http.Request) string {
id := r.Header.Get(s3_constants.AmzAccountId)
if len(id) == 0 {
return s3_constants.AccountAnonymousId
} else {
return id
}
}
// ExtractAcl extracts the acl from the request body, or from the header if request body is empty
func ExtractAcl(r *http.Request, accountManager AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) {
if r.Body != nil && r.Body != http.NoBody {
@@ -318,83 +308,6 @@ func ValidateAndTransferGrants(accountManager AccountManager, grants []*s3.Grant
return result, s3err.ErrNone
}
// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission.
func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
// group grantee (AllUsers)
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
})
// canonical grantee (accountId)
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
})
// group grantee (AuthenticateUsers)
if accountId != s3_constants.AccountAnonymousId {
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &s3_constants.PermissionFullControl,
})
}
return
}
func SetAcpOwnerHeader(r *http.Request, acpOwnerId string) {
r.Header.Set(s3_constants.ExtAmzOwnerKey, acpOwnerId)
}
func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string {
ownerIdBytes, ok := entryExtended[s3_constants.ExtAmzOwnerKey]
if ok && len(ownerIdBytes) > 0 {
return string(ownerIdBytes)
}
return defaultOwner
}
func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) {
if len(acpGrants) > 0 {
a, err := json.Marshal(acpGrants)
if err == nil {
r.Header.Set(s3_constants.ExtAmzAclKey, string(a))
} else {
glog.Warning("Marshal acp grants err", err)
}
}
}
// GetAcpGrants return grants parsed from entry
func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant {
acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey]
@@ -433,82 +346,3 @@ func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grant
return s3err.ErrNone
}
// GrantEquals Compare whether two Grants are equal in meaning, not completely
// equal (compare Grantee.Type and the corresponding Value for equality, other
// fields of Grantee are ignored)
func GrantEquals(a, b *s3.Grant) bool {
// grant
if a == b {
return true
}
if a == nil || b == nil {
return false
}
// grant.Permission
if a.Permission != b.Permission {
if a.Permission == nil || b.Permission == nil {
return false
}
if *a.Permission != *b.Permission {
return false
}
}
// grant.Grantee
ag := a.Grantee
bg := b.Grantee
if ag != bg {
if ag == nil || bg == nil {
return false
}
// grantee.Type
if ag.Type != bg.Type {
if ag.Type == nil || bg.Type == nil {
return false
}
if *ag.Type != *bg.Type {
return false
}
}
// value corresponding to granteeType
if ag.Type != nil {
switch *ag.Type {
case s3_constants.GrantTypeGroup:
if ag.URI != bg.URI {
if ag.URI == nil || bg.URI == nil {
return false
}
if *ag.URI != *bg.URI {
return false
}
}
case s3_constants.GrantTypeCanonicalUser:
if ag.ID != bg.ID {
if ag.ID == nil || bg.ID == nil {
return false
}
if *ag.ID != *bg.ID {
return false
}
}
case s3_constants.GrantTypeAmazonCustomerByEmail:
if ag.EmailAddress != bg.EmailAddress {
if ag.EmailAddress == nil || bg.EmailAddress == nil {
return false
}
if *ag.EmailAddress != *bg.EmailAddress {
return false
}
}
}
}
}
return true
}

710
weed/s3api/s3api_acl_helper_test.go
View File

@@ -1,710 +0,0 @@
package s3api
import (
"bytes"
"encoding/json"
"io"
"net/http"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
var accountManager *IdentityAccessManagement
func init() {
accountManager = &IdentityAccessManagement{}
_ = accountManager.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Accounts: []*iam_pb.Account{
{
Id: "accountA",
DisplayName: "accountAName",
EmailAddress: "accountA@example.com",
},
{
Id: "accountB",
DisplayName: "accountBName",
EmailAddress: "accountB@example.com",
},
},
})
}
func TestGetAccountId(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
//case1
//accountId: "admin"
req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAdminId)
if GetAccountId(req) != s3_constants.AccountAdminId {
t.Fatal("expect accountId: admin")
}
//case2
//accountId: "anoymous"
req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAnonymousId)
if GetAccountId(req) != s3_constants.AccountAnonymousId {
t.Fatal("expect accountId: anonymous")
}
//case3
//accountId is nil => "anonymous"
req.Header.Del(s3_constants.AmzAccountId)
if GetAccountId(req) != s3_constants.AccountAnonymousId {
t.Fatal("expect accountId: anonymous")
}
}
func TestExtractAcl(t *testing.T) {
type Case struct {
id int
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
accountAdminId := "admin"
{
//case1 (good case)
//parse acp from request body
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
1,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountAdminId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse acp from header (cannedAcl)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
2,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (bad case)
//parse acp from request body (content is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf")))
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
id: 3,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
//case4 (bad case)
//parse acp from header (cannedAcl is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk")
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
{
//case5 (bad case)
//parse acp from request body: owner is inconsistent
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter = "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied,
})
}
for _, tc := range testCases {
if tc.resultErrCode != tc.expectErrCode {
t.Fatalf("case[%d]: errorCode not expect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func TestParseAndValidateAclHeaders(t *testing.T) {
type Case struct {
id int
resultOwner, expectOwner string
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
bucketOwner := "admin"
{
//case1 (good case)
//parse custom acl
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
1,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: aws.String(s3_constants.AccountAnonymousId),
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: aws.String(s3_constants.AccountAdminId),
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse canned acl (ownership=ObjectWriter)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
2,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (good case)
//parse canned acl (ownership=OwnershipBucketOwnerPreferred)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
3,
ownerId, bucketOwner,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case4 (bad case)
//parse custom acl (grantee id not exists)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case5 (bad case)
//parse custom acl (invalid format)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case6 (bad case)
//parse canned acl (invalid value)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
for _, tc := range testCases {
if tc.expectErrCode != tc.resultErrCode {
t.Errorf("case[%d]: errCode unexpect", tc.id)
}
if tc.resultOwner != tc.expectOwner {
t.Errorf("case[%d]: ownerId unexpect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func grantsEquals(a, b []*s3.Grant) bool {
if len(a) != len(b) {
return false
}
for i, grant := range a {
if !GrantEquals(grant, b[i]) {
return false
}
}
return true
}
func TestDetermineReqGrants(t *testing.T) {
{
//case1: request account is anonymous
accountId := s3_constants.AccountAnonymousId
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
{
//case2: request account is not anonymous (Iam authed)
accountId := "accountX"
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
}
func TestAssembleEntryWithAcp(t *testing.T) {
defaultOwner := "admin"
//case1
//assemble with non-empty grants
expectOwner := "accountS"
expectGrants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
entry := &filer_pb.Entry{}
AssembleEntryWithAcp(entry, expectOwner, expectGrants)
resultOwner := GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != expectOwner {
t.Fatalf("owner not expect")
}
resultGrants := GetAcpGrants(entry.Extended)
if !grantsEquals(resultGrants, expectGrants) {
t.Fatal("grants not expect")
}
//case2
//assemble with empty grants (override)
AssembleEntryWithAcp(entry, "", nil)
resultOwner = GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != defaultOwner {
t.Fatalf("owner not expect")
}
resultGrants = GetAcpGrants(entry.Extended)
if len(resultGrants) != 0 {
t.Fatal("grants not expect")
}
}
func TestGrantEquals(t *testing.T) {
testCases := map[bool]bool{
GrantEquals(nil, nil): true,
GrantEquals(&s3.Grant{}, nil): false,
GrantEquals(&s3.Grant{}, &s3.Grant{}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): false,
//type not present, compare other fields of grant is meaningless
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: aws.String(s3_constants.AccountAdminId),
//EmailAddress: &s3account.AccountAdmin.EmailAddress,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: aws.String(s3_constants.AccountAdminId),
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionWrite,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
}
for tc, expect := range testCases {
if tc != expect {
t.Fatal("TestGrantEquals not expect!")
}
}
}
func TestSetAcpOwnerHeader(t *testing.T) {
ownerId := "accountZ"
req := &http.Request{
Header: make(map[string][]string),
}
SetAcpOwnerHeader(req, ownerId)
if req.Header.Get(s3_constants.ExtAmzOwnerKey) != ownerId {
t.Fatalf("owner unexpect")
}
}
func TestSetAcpGrantsHeader(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
grants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
SetAcpGrantsHeader(req, grants)
grantsJson, _ := json.Marshal(grants)
if req.Header.Get(s3_constants.ExtAmzAclKey) != string(grantsJson) {
t.Fatalf("owner unexpect")
}
}

8
weed/s3api/s3api_bucket_handlers.go
View File

@@ -150,14 +150,6 @@ func isBucketOwnedByIdentity(entry *filer_pb.Entry, identity *Identity) bool {
return true
}
// isBucketVisibleToIdentity is kept for backward compatibility with tests.
// It checks if a bucket should be visible based on ownership only.
// Deprecated: Use isBucketOwnedByIdentity instead. The ListBucketsHandler
// now uses OR logic: a bucket is visible if user owns it OR has List permission.
func isBucketVisibleToIdentity(entry *filer_pb.Entry, identity *Identity) bool {
return isBucketOwnedByIdentity(entry, identity)
}
func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters

1085
weed/s3api/s3api_bucket_handlers_test.go
View File

File diff suppressed because it is too large Load Diff

984
weed/s3api/s3api_conditional_headers_test.go
View File

@@ -1,984 +0,0 @@
package s3api
import (
"bytes"
"encoding/hex"
"fmt"
"net/http"
"net/url"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// TestConditionalHeadersWithExistingObjects tests conditional headers against existing objects
// This addresses the PR feedback about missing test coverage for object existence scenarios
func TestConditionalHeadersWithExistingObjects(t *testing.T) {
bucket := "test-bucket"
object := "/test-object"
// Mock object with known ETag and modification time
testObject := &filer_pb.Entry{
Name: "test-object",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"abc123\""),
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC).Unix(), // June 15, 2024
FileSize: 1024, // Add file size
},
Chunks: []*filer_pb.FileChunk{
// Add a mock chunk to make calculateETagFromChunks work
{
FileId: "test-file-id",
Offset: 0,
Size: 1024,
},
},
}
// Test If-None-Match with existing object
t.Run("IfNoneMatch_ObjectExists", func(t *testing.T) {
// Test case 1: If-None-Match=* when object exists (should fail)
t.Run("Asterisk_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object exists with If-None-Match=*, got %v", errCode)
}
})
// Test case 2: If-None-Match with matching ETag (should fail)
t.Run("MatchingETag_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "\"abc123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when ETag matches, got %v", errCode)
}
})
// Test case 3: If-None-Match with non-matching ETag (should succeed)
t.Run("NonMatchingETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "\"xyz789\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when ETag doesn't match, got %v", errCode)
}
})
// Test case 4: If-None-Match with multiple ETags, one matching (should fail)
t.Run("MultipleETags_OneMatches_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "\"xyz789\", \"abc123\", \"def456\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when one ETag matches, got %v", errCode)
}
})
// Test case 5: If-None-Match with multiple ETags, none matching (should succeed)
t.Run("MultipleETags_NoneMatch_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "\"xyz789\", \"def456\", \"ghi123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when no ETags match, got %v", errCode)
}
})
})
// Test If-Match with existing object
t.Run("IfMatch_ObjectExists", func(t *testing.T) {
// Test case 1: If-Match with matching ETag (should succeed)
t.Run("MatchingETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "\"abc123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when ETag matches, got %v", errCode)
}
})
// Test case 2: If-Match with non-matching ETag (should fail)
t.Run("NonMatchingETag_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "\"xyz789\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when ETag doesn't match, got %v", errCode)
}
})
// Test case 3: If-Match with multiple ETags, one matching (should succeed)
t.Run("MultipleETags_OneMatches_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "\"xyz789\", \"abc123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when one ETag matches, got %v", errCode)
}
})
// Test case 4: If-Match with wildcard * (should succeed if object exists)
t.Run("Wildcard_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when If-Match=* and object exists, got %v", errCode)
}
})
})
// Test If-Modified-Since with existing object
t.Run("IfModifiedSince_ObjectExists", func(t *testing.T) {
// Test case 1: If-Modified-Since with date before object modification (should succeed)
t.Run("DateBefore_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
dateBeforeModification := time.Date(2024, 6, 14, 12, 0, 0, 0, time.UTC)
req.Header.Set(s3_constants.IfModifiedSince, dateBeforeModification.Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object was modified after date, got %v", errCode)
}
})
// Test case 2: If-Modified-Since with date after object modification (should fail)
t.Run("DateAfter_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
dateAfterModification := time.Date(2024, 6, 16, 12, 0, 0, 0, time.UTC)
req.Header.Set(s3_constants.IfModifiedSince, dateAfterModification.Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object wasn't modified since date, got %v", errCode)
}
})
// Test case 3: If-Modified-Since with exact modification date (should fail - not after)
t.Run("ExactDate_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
exactDate := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC)
req.Header.Set(s3_constants.IfModifiedSince, exactDate.Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object modification time equals header date, got %v", errCode)
}
})
})
// Test If-Unmodified-Since with existing object
t.Run("IfUnmodifiedSince_ObjectExists", func(t *testing.T) {
// Test case 1: If-Unmodified-Since with date after object modification (should succeed)
t.Run("DateAfter_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
dateAfterModification := time.Date(2024, 6, 16, 12, 0, 0, 0, time.UTC)
req.Header.Set(s3_constants.IfUnmodifiedSince, dateAfterModification.Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object wasn't modified after date, got %v", errCode)
}
})
// Test case 2: If-Unmodified-Since with date before object modification (should fail)
t.Run("DateBefore_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(testObject)
req := createTestPutRequest(bucket, object, "test content")
dateBeforeModification := time.Date(2024, 6, 14, 12, 0, 0, 0, time.UTC)
req.Header.Set(s3_constants.IfUnmodifiedSince, dateBeforeModification.Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object was modified after date, got %v", errCode)
}
})
})
}
// TestConditionalHeadersForReads tests conditional headers for read operations (GET, HEAD)
// This implements AWS S3 conditional reads behavior where different conditions return different status codes
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-reads.html
func TestConditionalHeadersForReads(t *testing.T) {
bucket := "test-bucket"
object := "/test-read-object"
// Mock existing object to test conditional headers against
existingObject := &filer_pb.Entry{
Name: "test-read-object",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"read123\""),
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC).Unix(),
FileSize: 1024,
},
Chunks: []*filer_pb.FileChunk{
{
FileId: "read-file-id",
Offset: 0,
Size: 1024,
},
},
}
// Test conditional reads with existing object
t.Run("ConditionalReads_ObjectExists", func(t *testing.T) {
// Test If-None-Match with existing object (should return 304 Not Modified)
t.Run("IfNoneMatch_ObjectExists_ShouldReturn304", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfNoneMatch, "\"read123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNotModified {
t.Errorf("Expected ErrNotModified when If-None-Match matches, got %v", errCode)
}
})
// Test If-None-Match=* with existing object (should return 304 Not Modified)
t.Run("IfNoneMatchAsterisk_ObjectExists_ShouldReturn304", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNotModified {
t.Errorf("Expected ErrNotModified when If-None-Match=* with existing object, got %v", errCode)
}
})
// Test If-None-Match with non-matching ETag (should succeed)
t.Run("IfNoneMatch_NonMatchingETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfNoneMatch, "\"different-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when If-None-Match doesn't match, got %v", errCode)
}
})
// Test If-Match with matching ETag (should succeed)
t.Run("IfMatch_MatchingETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "\"read123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when If-Match matches, got %v", errCode)
}
})
// Test If-Match with non-matching ETag (should return 412 Precondition Failed)
t.Run("IfMatch_NonMatchingETag_ShouldReturn412", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "\"different-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when If-Match doesn't match, got %v", errCode)
}
})
// Test If-Match=* with existing object (should succeed)
t.Run("IfMatchAsterisk_ObjectExists_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when If-Match=* with existing object, got %v", errCode)
}
})
// Test If-Modified-Since (object modified after date - should succeed)
t.Run("IfModifiedSince_ObjectModifiedAfter_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfModifiedSince, "Sat, 14 Jun 2024 12:00:00 GMT") // Before object mtime
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object modified after If-Modified-Since date, got %v", errCode)
}
})
// Test If-Modified-Since (object not modified since date - should return 304)
t.Run("IfModifiedSince_ObjectNotModified_ShouldReturn304", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfModifiedSince, "Sun, 16 Jun 2024 12:00:00 GMT") // After object mtime
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNotModified {
t.Errorf("Expected ErrNotModified when object not modified since If-Modified-Since date, got %v", errCode)
}
})
// Test If-Unmodified-Since (object not modified since date - should succeed)
t.Run("IfUnmodifiedSince_ObjectNotModified_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfUnmodifiedSince, "Sun, 16 Jun 2024 12:00:00 GMT") // After object mtime
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object not modified since If-Unmodified-Since date, got %v", errCode)
}
})
// Test If-Unmodified-Since (object modified since date - should return 412)
t.Run("IfUnmodifiedSince_ObjectModified_ShouldReturn412", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfUnmodifiedSince, "Fri, 14 Jun 2024 12:00:00 GMT") // Before object mtime
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object modified since If-Unmodified-Since date, got %v", errCode)
}
})
})
// Test conditional reads with non-existent object
t.Run("ConditionalReads_ObjectNotExists", func(t *testing.T) {
// Test If-None-Match with non-existent object (should succeed)
t.Run("IfNoneMatch_ObjectNotExists_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfNoneMatch, "\"any-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object doesn't exist with If-None-Match, got %v", errCode)
}
})
// Test If-Match with non-existent object (should return 412)
t.Run("IfMatch_ObjectNotExists_ShouldReturn412", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "\"any-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object doesn't exist with If-Match, got %v", errCode)
}
})
// Test If-Modified-Since with non-existent object (should succeed)
t.Run("IfModifiedSince_ObjectNotExists_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfModifiedSince, "Sat, 15 Jun 2024 12:00:00 GMT")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object doesn't exist with If-Modified-Since, got %v", errCode)
}
})
// Test If-Unmodified-Since with non-existent object (should return 412)
t.Run("IfUnmodifiedSince_ObjectNotExists_ShouldReturn412", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfUnmodifiedSince, "Sat, 15 Jun 2024 12:00:00 GMT")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if errCode.ErrorCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object doesn't exist with If-Unmodified-Since, got %v", errCode)
}
})
})
}
// Helper function to create a GET request for testing
func createTestGetRequest(bucket, object string) *http.Request {
return &http.Request{
Method: "GET",
Header: make(http.Header),
URL: &url.URL{
Path: fmt.Sprintf("/%s/%s", bucket, object),
},
}
}
// TestConditionalHeadersWithNonExistentObjects tests the original scenarios (object doesn't exist)
func TestConditionalHeadersWithNonExistentObjects(t *testing.T) {
s3a := NewS3ApiServerForTest()
if s3a == nil {
t.Skip("S3ApiServer not available for testing")
}
bucket := "test-bucket"
object := "/test-object"
// Test If-None-Match header when object doesn't exist
t.Run("IfNoneMatch_ObjectDoesNotExist", func(t *testing.T) {
// Test case 1: If-None-Match=* when object doesn't exist (should return ErrNone)
t.Run("Asterisk_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object doesn't exist, got %v", errCode)
}
})
// Test case 2: If-None-Match with specific ETag when object doesn't exist
t.Run("SpecificETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfNoneMatch, "\"some-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object doesn't exist, got %v", errCode)
}
})
})
// Test If-Match header when object doesn't exist
t.Run("IfMatch_ObjectDoesNotExist", func(t *testing.T) {
// Test case 1: If-Match with specific ETag when object doesn't exist (should fail - critical bug fix)
t.Run("SpecificETag_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "\"some-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object doesn't exist with If-Match header, got %v", errCode)
}
})
// Test case 2: If-Match with wildcard * when object doesn't exist (should fail)
t.Run("Wildcard_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object doesn't exist with If-Match=*, got %v", errCode)
}
})
})
// Test date format validation (works regardless of object existence)
t.Run("DateFormatValidation", func(t *testing.T) {
// Test case 1: Valid If-Modified-Since date format
t.Run("IfModifiedSince_ValidFormat", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfModifiedSince, time.Now().Format(time.RFC1123))
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone with valid date format, got %v", errCode)
}
})
// Test case 2: Invalid If-Modified-Since date format
t.Run("IfModifiedSince_InvalidFormat", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfModifiedSince, "invalid-date")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrInvalidRequest {
t.Errorf("Expected ErrInvalidRequest for invalid date format, got %v", errCode)
}
})
// Test case 3: Invalid If-Unmodified-Since date format
t.Run("IfUnmodifiedSince_InvalidFormat", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
req.Header.Set(s3_constants.IfUnmodifiedSince, "invalid-date")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrInvalidRequest {
t.Errorf("Expected ErrInvalidRequest for invalid date format, got %v", errCode)
}
})
})
// Test no conditional headers
t.Run("NoConditionalHeaders", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No object exists
req := createTestPutRequest(bucket, object, "test content")
// Don't set any conditional headers
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when no conditional headers, got %v", errCode)
}
})
}
// TestETagMatching tests the etagMatches helper function
func TestETagMatching(t *testing.T) {
s3a := NewS3ApiServerForTest()
if s3a == nil {
t.Skip("S3ApiServer not available for testing")
}
testCases := []struct {
name string
headerValue string
objectETag string
expected bool
}{
{
name: "ExactMatch",
headerValue: "\"abc123\"",
objectETag: "abc123",
expected: true,
},
{
name: "ExactMatchWithQuotes",
headerValue: "\"abc123\"",
objectETag: "\"abc123\"",
expected: true,
},
{
name: "NoMatch",
headerValue: "\"abc123\"",
objectETag: "def456",
expected: false,
},
{
name: "MultipleETags_FirstMatch",
headerValue: "\"abc123\", \"def456\"",
objectETag: "abc123",
expected: true,
},
{
name: "MultipleETags_SecondMatch",
headerValue: "\"abc123\", \"def456\"",
objectETag: "def456",
expected: true,
},
{
name: "MultipleETags_NoMatch",
headerValue: "\"abc123\", \"def456\"",
objectETag: "ghi789",
expected: false,
},
{
name: "WithSpaces",
headerValue: " \"abc123\" , \"def456\" ",
objectETag: "def456",
expected: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := s3a.etagMatches(tc.headerValue, tc.objectETag)
if result != tc.expected {
t.Errorf("Expected %v, got %v for headerValue='%s', objectETag='%s'",
tc.expected, result, tc.headerValue, tc.objectETag)
}
})
}
}
// TestGetObjectETagWithMd5AndChunks tests the fix for issue #7274
// When an object has both Attributes.Md5 and multiple chunks, getObjectETag should
// prefer Attributes.Md5 to match the behavior of HeadObject and filer.ETag
func TestGetObjectETagWithMd5AndChunks(t *testing.T) {
s3a := NewS3ApiServerForTest()
if s3a == nil {
t.Skip("S3ApiServer not available for testing")
}
// Create an object with both Md5 and multiple chunks (like in issue #7274)
// Md5: ZjcmMwrCVGNVgb4HoqHe9g== (base64) = 663726330ac254635581be07a2a1def6 (hex)
md5HexString := "663726330ac254635581be07a2a1def6"
md5Bytes, err := hex.DecodeString(md5HexString)
if err != nil {
t.Fatalf("failed to decode md5 hex string: %v", err)
}
entry := &filer_pb.Entry{
Name: "test-multipart-object",
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 5597744,
Md5: md5Bytes,
},
// Two chunks - if we only used ETagChunks, it would return format "hash-2"
Chunks: []*filer_pb.FileChunk{
{
FileId: "chunk1",
Offset: 0,
Size: 4194304,
ETag: "9+yCD2DGwMG5uKwAd+y04Q==",
},
{
FileId: "chunk2",
Offset: 4194304,
Size: 1403440,
ETag: "cs6SVSTgZ8W3IbIrAKmklg==",
},
},
}
// getObjectETag should return the Md5 in hex with quotes
expectedETag := "\"" + md5HexString + "\""
actualETag := s3a.getObjectETag(entry)
if actualETag != expectedETag {
t.Errorf("Expected ETag %s, got %s", expectedETag, actualETag)
}
// Now test that conditional headers work with this ETag
bucket := "test-bucket"
object := "/test-object"
// Test If-Match with the Md5-based ETag (should succeed)
t.Run("IfMatch_WithMd5BasedETag_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(entry)
req := createTestGetRequest(bucket, object)
// Client sends the ETag from HeadObject (without quotes)
req.Header.Set(s3_constants.IfMatch, md5HexString)
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when If-Match uses Md5-based ETag, got %v (ETag was %s)", result.ErrorCode, actualETag)
}
})
// Test If-Match with chunk-based ETag format (should fail - this was the old incorrect behavior)
t.Run("IfMatch_WithChunkBasedETag_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(entry)
req := createTestGetRequest(bucket, object)
// If we incorrectly calculated ETag from chunks, it would be in format "hash-2"
req.Header.Set(s3_constants.IfMatch, "123294de680f28bde364b81477549f7d-2")
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when If-Match uses chunk-based ETag format, got %v", result.ErrorCode)
}
})
}
// TestConditionalHeadersIntegration tests conditional headers with full integration
func TestConditionalHeadersIntegration(t *testing.T) {
// This would be a full integration test that requires a running SeaweedFS instance
t.Skip("Integration test - requires running SeaweedFS instance")
}
// createTestPutRequest creates a test HTTP PUT request
func createTestPutRequest(bucket, object, content string) *http.Request {
req, _ := http.NewRequest("PUT", "/"+bucket+object, bytes.NewReader([]byte(content)))
req.Header.Set("Content-Type", "application/octet-stream")
// Set up mux vars to simulate the bucket and object extraction
// In real tests, this would be handled by the gorilla mux router
return req
}
// NewS3ApiServerForTest creates a minimal S3ApiServer for testing
// Note: This is a simplified version for unit testing conditional logic
func NewS3ApiServerForTest() *S3ApiServer {
// In a real test environment, this would set up a proper S3ApiServer
// with filer connection, etc. For unit testing conditional header logic,
// we create a minimal instance
return &S3ApiServer{
option: &S3ApiServerOption{
BucketsPath: "/buckets",
},
bucketConfigCache: NewBucketConfigCache(60 * time.Minute),
}
}
// MockEntryGetter implements the simplified EntryGetter interface for testing
// Only mocks the data access dependency - tests use production getObjectETag and etagMatches
type MockEntryGetter struct {
mockEntry *filer_pb.Entry
}
// Implement only the simplified EntryGetter interface
func (m *MockEntryGetter) getEntry(parentDirectoryPath, entryName string) (*filer_pb.Entry, error) {
if m.mockEntry != nil {
return m.mockEntry, nil
}
return nil, filer_pb.ErrNotFound
}
// createMockEntryGetter creates a mock EntryGetter for testing
func createMockEntryGetter(mockEntry *filer_pb.Entry) *MockEntryGetter {
return &MockEntryGetter{
mockEntry: mockEntry,
}
}
// TestConditionalHeadersMultipartUpload tests conditional headers with multipart uploads
// This verifies AWS S3 compatibility where conditional headers only apply to CompleteMultipartUpload
func TestConditionalHeadersMultipartUpload(t *testing.T) {
bucket := "test-bucket"
object := "/test-multipart-object"
// Mock existing object to test conditional headers against
existingObject := &filer_pb.Entry{
Name: "test-multipart-object",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"existing123\""),
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC).Unix(),
FileSize: 2048,
},
Chunks: []*filer_pb.FileChunk{
{
FileId: "existing-file-id",
Offset: 0,
Size: 2048,
},
},
}
// Test CompleteMultipartUpload with If-None-Match: * (should fail when object exists)
t.Run("CompleteMultipartUpload_IfNoneMatchAsterisk_ObjectExists_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
// Create a mock CompleteMultipartUpload request with If-None-Match: *
req := &http.Request{
Method: "POST",
Header: make(http.Header),
URL: &url.URL{
RawQuery: "uploadId=test-upload-id",
},
}
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object exists with If-None-Match=*, got %v", errCode)
}
})
// Test CompleteMultipartUpload with If-None-Match: * (should succeed when object doesn't exist)
t.Run("CompleteMultipartUpload_IfNoneMatchAsterisk_ObjectNotExists_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No existing object
req := &http.Request{
Method: "POST",
Header: make(http.Header),
URL: &url.URL{
RawQuery: "uploadId=test-upload-id",
},
}
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object doesn't exist with If-None-Match=*, got %v", errCode)
}
})
// Test CompleteMultipartUpload with If-Match (should succeed when ETag matches)
t.Run("CompleteMultipartUpload_IfMatch_ETagMatches_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := &http.Request{
Method: "POST",
Header: make(http.Header),
URL: &url.URL{
RawQuery: "uploadId=test-upload-id",
},
}
req.Header.Set(s3_constants.IfMatch, "\"existing123\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when ETag matches, got %v", errCode)
}
})
// Test CompleteMultipartUpload with If-Match (should fail when object doesn't exist)
t.Run("CompleteMultipartUpload_IfMatch_ObjectNotExists_ShouldFail", func(t *testing.T) {
getter := createMockEntryGetter(nil) // No existing object
req := &http.Request{
Method: "POST",
Header: make(http.Header),
URL: &url.URL{
RawQuery: "uploadId=test-upload-id",
},
}
req.Header.Set(s3_constants.IfMatch, "\"any-etag\"")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Errorf("Expected ErrPreconditionFailed when object doesn't exist with If-Match, got %v", errCode)
}
})
// Test CompleteMultipartUpload with If-Match wildcard (should succeed when object exists)
t.Run("CompleteMultipartUpload_IfMatchWildcard_ObjectExists_ShouldSucceed", func(t *testing.T) {
getter := createMockEntryGetter(existingObject)
req := &http.Request{
Method: "POST",
Header: make(http.Header),
URL: &url.URL{
RawQuery: "uploadId=test-upload-id",
},
}
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Errorf("Expected ErrNone when object exists with If-Match=*, got %v", errCode)
}
})
}
func TestConditionalHeadersTreatDeleteMarkerAsMissing(t *testing.T) {
bucket := "test-bucket"
object := "/deleted-object"
deleteMarkerEntry := &filer_pb.Entry{
Name: "deleted-object",
Extended: map[string][]byte{
s3_constants.ExtDeleteMarkerKey: []byte("true"),
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).Unix(),
},
}
t.Run("WriteIfNoneMatchAsteriskSucceeds", func(t *testing.T) {
getter := createMockEntryGetter(deleteMarkerEntry)
req := createTestPutRequest(bucket, object, "new content")
req.Header.Set(s3_constants.IfNoneMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrNone {
t.Fatalf("expected ErrNone for delete marker with If-None-Match=*, got %v", errCode)
}
})
t.Run("WriteIfMatchAsteriskFails", func(t *testing.T) {
getter := createMockEntryGetter(deleteMarkerEntry)
req := createTestPutRequest(bucket, object, "new content")
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
errCode := s3a.checkConditionalHeadersWithGetter(getter, req, bucket, object)
if errCode != s3err.ErrPreconditionFailed {
t.Fatalf("expected ErrPreconditionFailed for delete marker with If-Match=*, got %v", errCode)
}
})
t.Run("ReadIfMatchAsteriskFails", func(t *testing.T) {
getter := createMockEntryGetter(deleteMarkerEntry)
req := &http.Request{Method: http.MethodGet, Header: make(http.Header)}
req.Header.Set(s3_constants.IfMatch, "*")
s3a := NewS3ApiServerForTest()
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrPreconditionFailed {
t.Fatalf("expected ErrPreconditionFailed for read against delete marker with If-Match=*, got %v", result.ErrorCode)
}
if result.Entry != nil {
t.Fatalf("expected no entry to be returned for delete marker, got %#v", result.Entry)
}
})
}

23
weed/s3api/s3api_copy_size_calculation.go
View File

@@ -4,7 +4,6 @@ import (
"net/http"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// CopySizeCalculator handles size calculations for different copy scenarios
@@ -78,12 +77,6 @@ func (calc *CopySizeCalculator) CalculateActualSize() int64 {
return calc.srcSize
}
// CalculateEncryptedSize calculates the encrypted size for the given encryption type
func (calc *CopySizeCalculator) CalculateEncryptedSize(encType EncryptionType) int64 {
// With IV in metadata, encrypted size equals actual size
return calc.CalculateActualSize()
}
// getSourceEncryptionType determines the encryption type of the source object
func getSourceEncryptionType(metadata map[string][]byte) (EncryptionType, bool) {
if IsSSECEncrypted(metadata) {
@@ -169,22 +162,6 @@ func (calc *CopySizeCalculator) GetSizeTransitionInfo() *SizeTransitionInfo {
return info
}
// String returns a string representation of the encryption type
func (e EncryptionType) String() string {
switch e {
case EncryptionTypeNone:
return "None"
case EncryptionTypeSSEC:
return s3_constants.SSETypeC
case EncryptionTypeSSEKMS:
return s3_constants.SSETypeKMS
case EncryptionTypeSSES3:
return s3_constants.SSETypeS3
default:
return "Unknown"
}
}
// OptimizedSizeCalculation provides size calculations optimized for different scenarios
type OptimizedSizeCalculation struct {
Strategy UnifiedCopyStrategy

167
weed/s3api/s3api_etag_quoting_test.go
View File

@@ -1,167 +0,0 @@
package s3api
import (
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// TestReproIfMatchMismatch tests specifically for the scenario where internal ETag
// is unquoted (common in SeaweedFS) but client sends quoted ETag in If-Match.
func TestReproIfMatchMismatch(t *testing.T) {
bucket := "test-bucket"
object := "/test-key"
etagValue := "37b51d194a7513e45b56f6524f2d51f2"
// Scenario 1: Internal ETag is UNQUOTED (stored in Extended), Client sends QUOTED If-Match
// This mirrors the behavior we enforced in filer_multipart.go
t.Run("UnquotedInternal_QuotedHeader", func(t *testing.T) {
entry := &filer_pb.Entry{
Name: "test-key",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte(etagValue), // Unquoted
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
},
}
getter := &MockEntryGetter{mockEntry: entry}
req := createTestGetRequest(bucket, object)
// Client sends quoted ETag
req.Header.Set(s3_constants.IfMatch, "\""+etagValue+"\"")
s3a := NewS3ApiServerForTest()
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrNone {
t.Errorf("Expected success (ErrNone) for unquoted internal ETag and quoted header, got %v. Internal ETag: %s", result.ErrorCode, string(entry.Extended[s3_constants.ExtETagKey]))
}
})
// Scenario 2: Internal ETag is QUOTED (stored in Extended), Client sends QUOTED If-Match
// This handles legacy or mixed content
t.Run("QuotedInternal_QuotedHeader", func(t *testing.T) {
entry := &filer_pb.Entry{
Name: "test-key",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"" + etagValue + "\""), // Quoted
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
},
}
getter := &MockEntryGetter{mockEntry: entry}
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "\""+etagValue+"\"")
s3a := NewS3ApiServerForTest()
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrNone {
t.Errorf("Expected success (ErrNone) for quoted internal ETag and quoted header, got %v", result.ErrorCode)
}
})
// Scenario 3: Internal ETag is from Md5 (QUOTED by getObjectETag), Client sends QUOTED If-Match
t.Run("Md5Internal_QuotedHeader", func(t *testing.T) {
// Mock Md5 attribute (16 bytes)
md5Bytes := make([]byte, 16)
copy(md5Bytes, []byte("1234567890123456")) // This doesn't match the hex string below, but getObjectETag formats it as hex
// Expected ETag from Md5 is hex string of bytes
expectedHex := fmt.Sprintf("%x", md5Bytes)
entry := &filer_pb.Entry{
Name: "test-key",
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
Md5: md5Bytes,
},
}
getter := &MockEntryGetter{mockEntry: entry}
req := createTestGetRequest(bucket, object)
req.Header.Set(s3_constants.IfMatch, "\""+expectedHex+"\"")
s3a := NewS3ApiServerForTest()
result := s3a.checkConditionalHeadersForReadsWithGetter(getter, req, bucket, object)
if result.ErrorCode != s3err.ErrNone {
t.Errorf("Expected success (ErrNone) for Md5 internal ETag and quoted header, got %v", result.ErrorCode)
}
})
// Test getObjectETag specifically ensuring it returns quoted strings
t.Run("getObjectETag_ShouldReturnQuoted", func(t *testing.T) {
entry := &filer_pb.Entry{
Name: "test-key",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("unquoted-etag"),
},
}
s3a := NewS3ApiServerForTest()
etag := s3a.getObjectETag(entry)
expected := "\"unquoted-etag\""
if etag != expected {
t.Errorf("Expected quoted ETag %s, got %s", expected, etag)
}
})
// Test getObjectETag fallback when Extended ETag is present but empty
t.Run("getObjectETag_EmptyExtended_ShouldFallback", func(t *testing.T) {
md5Bytes := []byte("1234567890123456")
expectedHex := fmt.Sprintf("\"%x\"", md5Bytes)
entry := &filer_pb.Entry{
Name: "test-key-fallback",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte(""), // Present but empty
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
Md5: md5Bytes,
},
}
s3a := NewS3ApiServerForTest()
etag := s3a.getObjectETag(entry)
if etag != expectedHex {
t.Errorf("Expected fallback ETag %s, got %s", expectedHex, etag)
}
})
// Test newListEntry ETag behavior
t.Run("newListEntry_ShouldReturnQuoted", func(t *testing.T) {
entry := &filer_pb.Entry{
Name: "test-key",
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("unquoted-etag"),
},
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
},
}
s3a := NewS3ApiServerForTest()
listEntry := newListEntry(s3a, entry, "", "bucket/dir", "test-key", "bucket/", false, false, false)
expected := "\"unquoted-etag\""
if listEntry.ETag != expected {
t.Errorf("Expected quoted ETag %s, got %s", expected, listEntry.ETag)
}
})
}

30
weed/s3api/s3api_key_rotation.go
View File

@@ -1,30 +0,0 @@
package s3api
import (
"net/http"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// IsSameObjectCopy determines if this is a same-object copy operation
func IsSameObjectCopy(r *http.Request, srcBucket, srcObject, dstBucket, dstObject string) bool {
return srcBucket == dstBucket && srcObject == dstObject
}
// NeedsKeyRotation determines if the copy operation requires key rotation
func NeedsKeyRotation(entry *filer_pb.Entry, r *http.Request) bool {
// Check for SSE-C key rotation
if IsSSECEncrypted(entry.Extended) && IsSSECRequest(r) {
return true // Assume different keys for safety
}
// Check for SSE-KMS key rotation
if IsSSEKMSEncrypted(entry.Extended) && IsSSEKMSRequest(r) {
srcKeyID, _ := GetSourceSSEKMSInfo(entry.Extended)
dstKeyID := r.Header.Get(s3_constants.AmzServerSideEncryptionAwsKmsKeyId)
return srcKeyID != dstKeyID
}
return false
}

62
weed/s3api/s3api_object_handlers.go
View File

@@ -268,15 +268,6 @@ func mimeDetect(r *http.Request, dataReader io.Reader) io.ReadCloser {
return io.NopCloser(dataReader)
}
func urlEscapeObject(object string) string {
normalized := s3_constants.NormalizeObjectKey(object)
// Ensure leading slash for filer paths
if normalized != "" && !strings.HasPrefix(normalized, "/") {
normalized = "/" + normalized
}
return urlPathEscape(normalized)
}
func entryUrlEncode(dir string, entry string, encodingTypeUrl bool) (dirName string, entryName string, prefix string) {
if !encodingTypeUrl {
return dir, entry, entry
@@ -2895,59 +2886,6 @@ func (m *MultipartSSEReader) Close() error {
return lastErr
}
// Read implements the io.Reader interface for SSERangeReader
func (r *SSERangeReader) Read(p []byte) (n int, err error) {
// Skip bytes iteratively (no recursion) until we reach the offset
for r.skipped < r.offset {
skipNeeded := r.offset - r.skipped
// Lazily allocate skip buffer on first use, reuse thereafter
if r.skipBuf == nil {
// Use a fixed 32KB buffer for skipping (avoids per-call allocation)
r.skipBuf = make([]byte, 32*1024)
}
// Determine how much to skip in this iteration
bufSize := int64(len(r.skipBuf))
if skipNeeded < bufSize {
bufSize = skipNeeded
}
skipRead, skipErr := r.reader.Read(r.skipBuf[:bufSize])
r.skipped += int64(skipRead)
if skipErr != nil {
return 0, skipErr
}
// Guard against infinite loop: io.Reader may return (0, nil)
// which is permitted by the interface contract for non-empty buffers.
// If we get zero bytes without an error, treat it as an unexpected EOF.
if skipRead == 0 {
return 0, io.ErrUnexpectedEOF
}
}
// If we have a remaining limit and it's reached
if r.remaining == 0 {
return 0, io.EOF
}
// Calculate how much to read
readSize := len(p)
if r.remaining > 0 && int64(readSize) > r.remaining {
readSize = int(r.remaining)
}
// Read the data
n, err = r.reader.Read(p[:readSize])
if r.remaining > 0 {
r.remaining -= int64(n)
}
return n, err
}
// PartBoundaryInfo holds information about a part's chunk boundaries
type PartBoundaryInfo struct {
PartNumber int `json:"part"`

61
weed/s3api/s3api_object_handlers_copy.go
View File

@@ -14,8 +14,6 @@ import (
"strings"
"time"
"modernc.org/strutil"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
@@ -797,58 +795,6 @@ func replaceDirective(reqHeader http.Header) (replaceMeta, replaceTagging bool)
return reqHeader.Get(s3_constants.AmzUserMetaDirective) == DirectiveReplace, reqHeader.Get(s3_constants.AmzObjectTaggingDirective) == DirectiveReplace
}
func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTagging bool, getTags func(parentDirectoryPath string, entryName string) (tags map[string]string, err error), dir, name string) (err error) {
if sc := reqHeader.Get(s3_constants.AmzStorageClass); len(sc) == 0 {
if sc := existing.Get(s3_constants.AmzStorageClass); len(sc) > 0 {
reqHeader.Set(s3_constants.AmzStorageClass, sc)
}
}
if !replaceMeta {
for header := range reqHeader {
if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) {
delete(reqHeader, header)
}
}
for k, v := range existing {
if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) {
reqHeader[k] = v
}
}
}
if !replaceTagging {
for header, _ := range reqHeader {
if strings.HasPrefix(header, s3_constants.AmzObjectTagging) {
delete(reqHeader, header)
}
}
found := false
for k, _ := range existing {
if strings.HasPrefix(k, s3_constants.AmzObjectTaggingPrefix) {
found = true
break
}
}
if found {
tags, err := getTags(dir, name)
if err != nil {
return err
}
var tagArr []string
for k, v := range tags {
tagArr = append(tagArr, fmt.Sprintf("%s=%s", k, v))
}
tagStr := strutil.JoinFields(tagArr, "&")
reqHeader.Set(s3_constants.AmzObjectTagging, tagStr)
}
}
return
}
func processMetadataBytes(reqHeader http.Header, existing map[string][]byte, replaceMeta, replaceTagging bool) (metadata map[string][]byte, err error) {
metadata = make(map[string][]byte)
@@ -2632,13 +2578,6 @@ func cleanupVersioningMetadata(metadata map[string][]byte) {
delete(metadata, s3_constants.ExtETagKey)
}
// shouldCreateVersionForCopy determines whether a version should be created during a copy operation
// based on the destination bucket's versioning state.
// Returns true only if versioning is explicitly "Enabled", not "Suspended" or unconfigured.
func shouldCreateVersionForCopy(versioningState string) bool {
return versioningState == s3_constants.VersioningEnabled
}
// isOrphanedSSES3Header checks if a header is an orphaned SSE-S3 encryption header.
// An orphaned header is one where the encryption indicator exists but the actual key is missing.
// This can happen when an object was previously encrypted but then copied without encryption,

760
weed/s3api/s3api_object_handlers_copy_test.go
View File

@@ -1,760 +0,0 @@
package s3api
import (
"fmt"
"net/http"
"reflect"
"sort"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type H map[string]string
func (h H) String() string {
pairs := make([]string, 0, len(h))
for k, v := range h {
pairs = append(pairs, fmt.Sprintf("%s : %s", k, v))
}
sort.Strings(pairs)
join := strings.Join(pairs, "\n")
return "\n" + join + "\n"
}
var processMetadataTestCases = []struct {
caseId int
request H
existing H
getTags H
want H
}{
{
201,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging": "A=B&a=b&type=existing",
},
},
{
202,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=existing",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
},
},
{
203,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
},
{
204,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
},
{
205,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{},
H{},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
},
{
206,
H{
"User-Agent": "firefox",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
},
{
207,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-Type": "existing",
},
H{
"A": "B",
"a": "b",
"type": "existing",
},
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
},
}
var processMetadataBytesTestCases = []struct {
caseId int
request H
existing H
want H
}{
{
101,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
},
{
102,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
H{
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
},
{
103,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "request",
},
},
{
104,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
H{
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "request",
},
},
{
105,
H{
"User-Agent": "firefox",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{
"X-Amz-Meta-My-Meta": "existing",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "existing",
},
H{},
},
{
107,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{},
H{
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging-A": "B",
"X-Amz-Tagging-a": "b",
"X-Amz-Tagging-type": "request",
},
},
{
108,
H{
"User-Agent": "firefox",
"X-Amz-Meta-My-Meta": "request",
"X-Amz-Tagging": "A=B&a=b&type=request*",
s3_constants.AmzUserMetaDirective: DirectiveReplace,
s3_constants.AmzObjectTaggingDirective: DirectiveReplace,
},
H{},
H{},
},
}
func TestProcessMetadata(t *testing.T) {
for _, tc := range processMetadataTestCases {
reqHeader := transferHToHeader(tc.request)
existing := transferHToHeader(tc.existing)
replaceMeta, replaceTagging := replaceDirective(reqHeader)
err := processMetadata(reqHeader, existing, replaceMeta, replaceTagging, func(_ string, _ string) (tags map[string]string, err error) {
return tc.getTags, nil
}, "", "")
if err != nil {
t.Error(err)
}
result := transferHeaderToH(reqHeader)
fmtTagging(result, tc.want)
if !reflect.DeepEqual(result, tc.want) {
t.Error(fmt.Errorf("\n### CaseID: %d ###"+
"\nRequest:%v"+
"\nExisting:%v"+
"\nGetTags:%v"+
"\nWant:%v"+
"\nActual:%v",
tc.caseId, tc.request, tc.existing, tc.getTags, tc.want, result))
}
}
}
func TestProcessMetadataBytes(t *testing.T) {
for _, tc := range processMetadataBytesTestCases {
reqHeader := transferHToHeader(tc.request)
existing := transferHToBytesArr(tc.existing)
replaceMeta, replaceTagging := replaceDirective(reqHeader)
extends, _ := processMetadataBytes(reqHeader, existing, replaceMeta, replaceTagging)
result := transferBytesArrToH(extends)
fmtTagging(result, tc.want)
if !reflect.DeepEqual(result, tc.want) {
t.Error(fmt.Errorf("\n### CaseID: %d ###"+
"\nRequest:%v"+
"\nExisting:%v"+
"\nWant:%v"+
"\nActual:%v",
tc.caseId, tc.request, tc.existing, tc.want, result))
}
}
}
func TestMergeCopyMetadataPreservesInternalFields(t *testing.T) {
existing := map[string][]byte{
s3_constants.SeaweedFSSSEKMSKey: []byte("kms-secret"),
s3_constants.SeaweedFSSSEIV: []byte("iv"),
"X-Amz-Meta-Old": []byte("old"),
"X-Amz-Tagging-Old": []byte("old-tag"),
s3_constants.AmzStorageClass: []byte("STANDARD"),
}
updated := map[string][]byte{
"X-Amz-Meta-New": []byte("new"),
"X-Amz-Tagging-New": []byte("new-tag"),
s3_constants.AmzStorageClass: []byte("GLACIER"),
}
merged := mergeCopyMetadata(existing, updated)
if got := string(merged[s3_constants.SeaweedFSSSEKMSKey]); got != "kms-secret" {
t.Fatalf("expected internal KMS key to be preserved, got %q", got)
}
if got := string(merged[s3_constants.SeaweedFSSSEIV]); got != "iv" {
t.Fatalf("expected internal IV to be preserved, got %q", got)
}
if _, ok := merged["X-Amz-Meta-Old"]; ok {
t.Fatalf("expected stale user metadata to be removed, got %#v", merged)
}
if _, ok := merged["X-Amz-Tagging-Old"]; ok {
t.Fatalf("expected stale tagging metadata to be removed, got %#v", merged)
}
if got := string(merged["X-Amz-Meta-New"]); got != "new" {
t.Fatalf("expected replacement user metadata to be applied, got %q", got)
}
if got := string(merged["X-Amz-Tagging-New"]); got != "new-tag" {
t.Fatalf("expected replacement tagging metadata to be applied, got %q", got)
}
if got := string(merged[s3_constants.AmzStorageClass]); got != "GLACIER" {
t.Fatalf("expected storage class to be updated, got %q", got)
}
}
func TestCopyEntryETagPrefersStoredETag(t *testing.T) {
entry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"stored-etag\""),
},
Attributes: &filer_pb.FuseAttributes{},
}
if got := copyEntryETag(util.FullPath("/buckets/test-bucket/object.txt"), entry); got != "\"stored-etag\"" {
t.Fatalf("copyEntryETag() = %q, want %q", got, "\"stored-etag\"")
}
}
func fmtTagging(maps ...map[string]string) {
for _, m := range maps {
if tagging := m[s3_constants.AmzObjectTagging]; len(tagging) > 0 {
split := strings.Split(tagging, "&")
sort.Strings(split)
m[s3_constants.AmzObjectTagging] = strings.Join(split, "&")
}
}
}
func transferHToHeader(data map[string]string) http.Header {
header := http.Header{}
for k, v := range data {
header.Add(k, v)
}
return header
}
func transferHToBytesArr(data map[string]string) map[string][]byte {
m := make(map[string][]byte, len(data))
for k, v := range data {
m[k] = []byte(v)
}
return m
}
func transferBytesArrToH(data map[string][]byte) H {
m := make(map[string]string, len(data))
for k, v := range data {
m[k] = string(v)
}
return m
}
func transferHeaderToH(data map[string][]string) H {
m := make(map[string]string, len(data))
for k, v := range data {
m[k] = v[len(v)-1]
}
return m
}
// TestShouldCreateVersionForCopy tests the production function that determines
// whether a version should be created during a copy operation.
// This addresses issue #7505 where copies were incorrectly creating versions for non-versioned buckets.
func TestShouldCreateVersionForCopy(t *testing.T) {
testCases := []struct {
name string
versioningState string
expectedResult bool
description string
}{
{
name: "VersioningEnabled",
versioningState: s3_constants.VersioningEnabled,
expectedResult: true,
description: "Should create versions in .versions/ directory when versioning is Enabled",
},
{
name: "VersioningSuspended",
versioningState: s3_constants.VersioningSuspended,
expectedResult: false,
description: "Should NOT create versions when versioning is Suspended",
},
{
name: "VersioningNotConfigured",
versioningState: "",
expectedResult: false,
description: "Should NOT create versions when versioning is not configured",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Call the actual production function
result := shouldCreateVersionForCopy(tc.versioningState)
if result != tc.expectedResult {
t.Errorf("Test case %s failed: %s\nExpected shouldCreateVersionForCopy(%q)=%v, got %v",
tc.name, tc.description, tc.versioningState, tc.expectedResult, result)
}
})
}
}
// TestCleanupVersioningMetadata tests the production function that removes versioning metadata.
// This ensures objects copied to non-versioned buckets don't carry invalid versioning metadata
// or stale ETag values from the source.
func TestCleanupVersioningMetadata(t *testing.T) {
testCases := []struct {
name string
sourceMetadata map[string][]byte
expectedKeys []string // Keys that should be present after cleanup
removedKeys []string // Keys that should be removed
}{
{
name: "RemovesAllVersioningMetadata",
sourceMetadata: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("version-123"),
s3_constants.ExtDeleteMarkerKey: []byte("false"),
s3_constants.ExtIsLatestKey: []byte("true"),
s3_constants.ExtETagKey: []byte("\"abc123\""),
"X-Amz-Meta-Custom": []byte("value"),
},
expectedKeys: []string{"X-Amz-Meta-Custom"},
removedKeys: []string{s3_constants.ExtVersionIdKey, s3_constants.ExtDeleteMarkerKey, s3_constants.ExtIsLatestKey, s3_constants.ExtETagKey},
},
{
name: "HandlesEmptyMetadata",
sourceMetadata: map[string][]byte{},
expectedKeys: []string{},
removedKeys: []string{s3_constants.ExtVersionIdKey, s3_constants.ExtDeleteMarkerKey, s3_constants.ExtIsLatestKey, s3_constants.ExtETagKey},
},
{
name: "PreservesNonVersioningMetadata",
sourceMetadata: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("version-456"),
s3_constants.ExtETagKey: []byte("\"def456\""),
"X-Amz-Meta-Custom": []byte("value1"),
"X-Amz-Meta-Another": []byte("value2"),
s3_constants.ExtIsLatestKey: []byte("true"),
},
expectedKeys: []string{"X-Amz-Meta-Custom", "X-Amz-Meta-Another"},
removedKeys: []string{s3_constants.ExtVersionIdKey, s3_constants.ExtETagKey, s3_constants.ExtIsLatestKey},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Create a copy of the source metadata
dstMetadata := make(map[string][]byte)
for k, v := range tc.sourceMetadata {
dstMetadata[k] = v
}
// Call the actual production function
cleanupVersioningMetadata(dstMetadata)
// Verify expected keys are present
for _, key := range tc.expectedKeys {
if _, exists := dstMetadata[key]; !exists {
t.Errorf("Expected key %s to be present in destination metadata", key)
}
}
// Verify removed keys are absent
for _, key := range tc.removedKeys {
if _, exists := dstMetadata[key]; exists {
t.Errorf("Expected key %s to be removed from destination metadata, but it's still present", key)
}
}
// Verify the count matches to ensure no extra keys are present
if len(dstMetadata) != len(tc.expectedKeys) {
t.Errorf("Expected %d metadata keys, but got %d. Extra keys might be present.", len(tc.expectedKeys), len(dstMetadata))
}
})
}
}
// TestCopyVersioningIntegration validates the metadata shaping that happens
// before copy finalization for each destination versioning mode.
func TestCopyVersioningIntegration(t *testing.T) {
testCases := []struct {
name string
versioningState string
sourceMetadata map[string][]byte
expectVersionPath bool
expectMetadataKeys []string
}{
{
name: "EnabledPreservesMetadata",
versioningState: s3_constants.VersioningEnabled,
sourceMetadata: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("v123"),
"X-Amz-Meta-Custom": []byte("value"),
},
expectVersionPath: true,
expectMetadataKeys: []string{
s3_constants.ExtVersionIdKey,
"X-Amz-Meta-Custom",
},
},
{
name: "SuspendedCleansVersionMetadataBeforeFinalize",
versioningState: s3_constants.VersioningSuspended,
sourceMetadata: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("v123"),
"X-Amz-Meta-Custom": []byte("value"),
},
expectVersionPath: false,
expectMetadataKeys: []string{
"X-Amz-Meta-Custom",
},
},
{
name: "NotConfiguredCleansMetadata",
versioningState: "",
sourceMetadata: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte("v123"),
s3_constants.ExtDeleteMarkerKey: []byte("false"),
"X-Amz-Meta-Custom": []byte("value"),
},
expectVersionPath: false,
expectMetadataKeys: []string{
"X-Amz-Meta-Custom",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Test version creation decision using production function
shouldCreateVersion := shouldCreateVersionForCopy(tc.versioningState)
if shouldCreateVersion != tc.expectVersionPath {
t.Errorf("shouldCreateVersionForCopy(%q) = %v, expected %v",
tc.versioningState, shouldCreateVersion, tc.expectVersionPath)
}
// Test metadata cleanup using production function
metadata := make(map[string][]byte)
for k, v := range tc.sourceMetadata {
metadata[k] = v
}
if !shouldCreateVersion {
cleanupVersioningMetadata(metadata)
}
// Verify only expected keys remain
for _, expectedKey := range tc.expectMetadataKeys {
if _, exists := metadata[expectedKey]; !exists {
t.Errorf("Expected key %q to be present in metadata", expectedKey)
}
}
// Verify the count matches (no extra keys)
if len(metadata) != len(tc.expectMetadataKeys) {
t.Errorf("Expected %d metadata keys, got %d", len(tc.expectMetadataKeys), len(metadata))
}
})
}
}
// TestIsOrphanedSSES3Header tests detection of orphaned SSE-S3 headers.
// This is a regression test for GitHub issue #7562 where copying from an
// encrypted bucket to an unencrypted bucket left behind the encryption header
// without the actual key, causing subsequent copy operations to fail.
func TestIsOrphanedSSES3Header(t *testing.T) {
testCases := []struct {
name string
headerKey string
metadata map[string][]byte
expected bool
}{
{
name: "Not an encryption header",
headerKey: "X-Amz-Meta-Custom",
metadata: map[string][]byte{
"X-Amz-Meta-Custom": []byte("value"),
},
expected: false,
},
{
name: "SSE-S3 header with key present (valid)",
headerKey: s3_constants.AmzServerSideEncryption,
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
s3_constants.SeaweedFSSSES3Key: []byte("key-data"),
},
expected: false,
},
{
name: "SSE-S3 header without key (orphaned - GitHub #7562)",
headerKey: s3_constants.AmzServerSideEncryption,
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
},
expected: true,
},
{
name: "SSE-KMS header (not SSE-S3)",
headerKey: s3_constants.AmzServerSideEncryption,
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("aws:kms"),
},
expected: false,
},
{
name: "Different header key entirely",
headerKey: s3_constants.SeaweedFSSSES3Key,
metadata: map[string][]byte{
s3_constants.AmzServerSideEncryption: []byte("AES256"),
},
expected: false,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := isOrphanedSSES3Header(tc.headerKey, tc.metadata)
if result != tc.expected {
t.Errorf("isOrphanedSSES3Header(%q, metadata) = %v, expected %v",
tc.headerKey, result, tc.expected)
}
})
}
}

119
weed/s3api/s3api_object_handlers_delete_test.go
View File

@@ -1,119 +0,0 @@
package s3api
import (
"encoding/xml"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
func TestValidateDeleteIfMatch(t *testing.T) {
s3a := NewS3ApiServerForTest()
existingEntry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtETagKey: []byte("\"abc123\""),
},
}
deleteMarkerEntry := &filer_pb.Entry{
Extended: map[string][]byte{
s3_constants.ExtDeleteMarkerKey: []byte("true"),
},
}
testCases := []struct {
name string
entry *filer_pb.Entry
ifMatch string
missingCode s3err.ErrorCode
expected s3err.ErrorCode
}{
{
name: "matching etag succeeds",
entry: existingEntry,
ifMatch: "\"abc123\"",
missingCode: s3err.ErrPreconditionFailed,
expected: s3err.ErrNone,
},
{
name: "wildcard succeeds for existing entry",
entry: existingEntry,
ifMatch: "*",
missingCode: s3err.ErrPreconditionFailed,
expected: s3err.ErrNone,
},
{
name: "mismatched etag fails",
entry: existingEntry,
ifMatch: "\"other\"",
missingCode: s3err.ErrPreconditionFailed,
expected: s3err.ErrPreconditionFailed,
},
{
name: "missing current object fails single delete",
entry: nil,
ifMatch: "*",
missingCode: s3err.ErrPreconditionFailed,
expected: s3err.ErrPreconditionFailed,
},
{
name: "missing current object returns no such key for batch delete",
entry: nil,
ifMatch: "*",
missingCode: s3err.ErrNoSuchKey,
expected: s3err.ErrNoSuchKey,
},
{
name: "current delete marker behaves like missing object",
entry: normalizeConditionalTargetEntry(deleteMarkerEntry),
ifMatch: "*",
missingCode: s3err.ErrPreconditionFailed,
expected: s3err.ErrPreconditionFailed,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if errCode := s3a.validateDeleteIfMatch(tc.entry, tc.ifMatch, tc.missingCode); errCode != tc.expected {
t.Fatalf("validateDeleteIfMatch() = %v, want %v", errCode, tc.expected)
}
})
}
}
func TestDeleteObjectsRequestUnmarshalConditionalETags(t *testing.T) {
var req DeleteObjectsRequest
body := []byte(`
<Delete>
<Quiet>true</Quiet>
<Object>
<Key>first.txt</Key>
<ETag>*</ETag>
</Object>
<Object>
<Key>second.txt</Key>
<VersionId>3HL4kqCxf3vjVBH40Nrjfkd</VersionId>
<ETag>"abc123"</ETag>
</Object>
</Delete>`)
if err := xml.Unmarshal(body, &req); err != nil {
t.Fatalf("xml.Unmarshal() error = %v", err)
}
if !req.Quiet {
t.Fatalf("expected Quiet=true")
}
if len(req.Objects) != 2 {
t.Fatalf("expected 2 objects, got %d", len(req.Objects))
}
if req.Objects[0].ETag != "*" {
t.Fatalf("expected first object ETag to be '*', got %q", req.Objects[0].ETag)
}
if req.Objects[1].ETag != "\"abc123\"" {
t.Fatalf("expected second object ETag to preserve quotes, got %q", req.Objects[1].ETag)
}
if req.Objects[1].VersionId != "3HL4kqCxf3vjVBH40Nrjfkd" {
t.Fatalf("expected second object VersionId to unmarshal, got %q", req.Objects[1].VersionId)
}
}

44
weed/s3api/s3api_object_handlers_put.go
View File

@@ -1859,28 +1859,6 @@ func (s3a *S3ApiServer) validateConditionalHeaders(r *http.Request, headers cond
return s3err.ErrNone
}
// checkConditionalHeadersWithGetter is a testable method that accepts a simple EntryGetter
// Uses the production getObjectETag and etagMatches methods to ensure testing of real logic
func (s3a *S3ApiServer) checkConditionalHeadersWithGetter(getter EntryGetter, r *http.Request, bucket, object string) s3err.ErrorCode {
headers, errCode := parseConditionalHeaders(r)
if errCode != s3err.ErrNone {
return errCode
}
// Get object entry for conditional checks.
bucketDir := "/buckets/" + bucket
entry, entryErr := getter.getEntry(bucketDir, object)
if entryErr != nil {
if errors.Is(entryErr, filer_pb.ErrNotFound) {
entry = nil
} else {
glog.Errorf("checkConditionalHeadersWithGetter: failed to get entry for %s/%s: %v", bucket, object, entryErr)
return s3err.ErrInternalError
}
}
return s3a.validateConditionalHeaders(r, headers, entry, bucket, object)
}
// checkConditionalHeaders is the production method that uses the S3ApiServer as EntryGetter
func (s3a *S3ApiServer) checkConditionalHeaders(r *http.Request, bucket, object string) s3err.ErrorCode {
// Fast path: if no conditional headers are present, skip object resolution entirely.
@@ -2002,28 +1980,6 @@ func (s3a *S3ApiServer) validateConditionalHeadersForReads(r *http.Request, head
return ConditionalHeaderResult{ErrorCode: s3err.ErrNone, Entry: entry}
}
// checkConditionalHeadersForReadsWithGetter is a testable method for read operations
// Uses the production getObjectETag and etagMatches methods to ensure testing of real logic
func (s3a *S3ApiServer) checkConditionalHeadersForReadsWithGetter(getter EntryGetter, r *http.Request, bucket, object string) ConditionalHeaderResult {
headers, errCode := parseConditionalHeaders(r)
if errCode != s3err.ErrNone {
return ConditionalHeaderResult{ErrorCode: errCode}
}
// Get object entry for conditional checks.
bucketDir := "/buckets/" + bucket
entry, entryErr := getter.getEntry(bucketDir, object)
if entryErr != nil {
if errors.Is(entryErr, filer_pb.ErrNotFound) {
entry = nil
} else {
glog.Errorf("checkConditionalHeadersForReadsWithGetter: failed to get entry for %s/%s: %v", bucket, object, entryErr)
return ConditionalHeaderResult{ErrorCode: s3err.ErrInternalError}
}
}
return s3a.validateConditionalHeadersForReads(r, headers, entry, bucket, object)
}
// checkConditionalHeadersForReads is the production method that uses the S3ApiServer as EntryGetter
func (s3a *S3ApiServer) checkConditionalHeadersForReads(r *http.Request, bucket, object string) ConditionalHeaderResult {
// Fast path: if no conditional headers are present, skip object resolution entirely.

341
weed/s3api/s3api_object_handlers_put_test.go
View File

@@ -1,341 +0,0 @@
package s3api
import (
"encoding/xml"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/util/constants"
)
func TestFilerErrorToS3Error(t *testing.T) {
tests := []struct {
name string
err error
expectedErr s3err.ErrorCode
}{
{
name: "nil error",
err: nil,
expectedErr: s3err.ErrNone,
},
{
name: "MD5 mismatch error",
err: errors.New(constants.ErrMsgBadDigest),
expectedErr: s3err.ErrBadDigest,
},
{
name: "Read only error (direct)",
err: weed_server.ErrReadOnly,
expectedErr: s3err.ErrAccessDenied,
},
{
name: "Read only error (wrapped)",
err: fmt.Errorf("create file /buckets/test/file.txt: %w", weed_server.ErrReadOnly),
expectedErr: s3err.ErrAccessDenied,
},
{
name: "Context canceled error",
err: errors.New("rpc error: code = Canceled desc = context canceled"),
expectedErr: s3err.ErrInvalidRequest,
},
{
name: "Context canceled error (simple)",
err: errors.New("context canceled"),
expectedErr: s3err.ErrInvalidRequest,
},
{
name: "Directory exists error (sentinel)",
err: fmt.Errorf("CreateEntry /path: %w", filer_pb.ErrExistingIsDirectory),
expectedErr: s3err.ErrExistingObjectIsDirectory,
},
{
name: "Parent is file error (sentinel)",
err: fmt.Errorf("CreateEntry /path: %w", filer_pb.ErrParentIsFile),
expectedErr: s3err.ErrExistingObjectIsFile,
},
{
name: "Existing is file error (sentinel)",
err: fmt.Errorf("CreateEntry /path: %w", filer_pb.ErrExistingIsFile),
expectedErr: s3err.ErrExistingObjectIsFile,
},
{
name: "Entry name too long (sentinel)",
err: fmt.Errorf("CreateEntry: %w", filer_pb.ErrEntryNameTooLong),
expectedErr: s3err.ErrKeyTooLongError,
},
{
name: "Entry name too long (bare sentinel)",
err: filer_pb.ErrEntryNameTooLong,
expectedErr: s3err.ErrKeyTooLongError,
},
{
name: "Unknown error",
err: errors.New("some random error"),
expectedErr: s3err.ErrInternalError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := filerErrorToS3Error(tt.err)
if result != tt.expectedErr {
t.Errorf("filerErrorToS3Error(%v) = %v, want %v", tt.err, result, tt.expectedErr)
}
})
}
}
// setupKeyLengthTestRouter creates a minimal router that maps requests directly
// to the given handler with {bucket} and {object} mux vars, bypassing auth.
func setupKeyLengthTestRouter(handler http.HandlerFunc) *mux.Router {
router := mux.NewRouter()
bucket := router.PathPrefix("/{bucket}").Subrouter()
bucket.Path("/{object:.+}").HandlerFunc(handler)
return router
}
func TestPutObjectHandler_KeyTooLong(t *testing.T) {
s3a := &S3ApiServer{}
router := setupKeyLengthTestRouter(s3a.PutObjectHandler)
longKey := strings.Repeat("a", s3_constants.MaxS3ObjectKeyLength+1)
req := httptest.NewRequest(http.MethodPut, "/bucket/"+longKey, nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusBadRequest {
t.Errorf("expected status %d, got %d", http.StatusBadRequest, rr.Code)
}
var errResp s3err.RESTErrorResponse
if err := xml.Unmarshal(rr.Body.Bytes(), &errResp); err != nil {
t.Fatalf("failed to parse error XML: %v", err)
}
if errResp.Code != "KeyTooLongError" {
t.Errorf("expected error code KeyTooLongError, got %s", errResp.Code)
}
}
func TestPutObjectHandler_KeyAtLimit(t *testing.T) {
s3a := &S3ApiServer{}
// Wrap handler to convert panics from uninitialized server state into 500
// responses. The key length check runs early and writes 400 KeyTooLongError
// before reaching any code that needs a fully initialized server. A panic
// means the handler accepted the key and continued past the check.
panicSafe := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if p := recover(); p != nil {
w.WriteHeader(http.StatusInternalServerError)
}
}()
s3a.PutObjectHandler(w, r)
})
router := setupKeyLengthTestRouter(panicSafe)
atLimitKey := strings.Repeat("a", s3_constants.MaxS3ObjectKeyLength)
req := httptest.NewRequest(http.MethodPut, "/bucket/"+atLimitKey, nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
// Must NOT be KeyTooLongError — any other response (including 500 from
// the minimal server hitting uninitialized state) proves the key passed.
var errResp s3err.RESTErrorResponse
if rr.Code == http.StatusBadRequest {
if err := xml.Unmarshal(rr.Body.Bytes(), &errResp); err == nil && errResp.Code == "KeyTooLongError" {
t.Errorf("key at exactly %d bytes should not be rejected as too long", s3_constants.MaxS3ObjectKeyLength)
}
}
}
func TestCopyObjectHandler_KeyTooLong(t *testing.T) {
s3a := &S3ApiServer{}
router := setupKeyLengthTestRouter(s3a.CopyObjectHandler)
longKey := strings.Repeat("a", s3_constants.MaxS3ObjectKeyLength+1)
req := httptest.NewRequest(http.MethodPut, "/bucket/"+longKey, nil)
req.Header.Set("X-Amz-Copy-Source", "/src-bucket/src-object")
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusBadRequest {
t.Errorf("expected status %d, got %d", http.StatusBadRequest, rr.Code)
}
var errResp s3err.RESTErrorResponse
if err := xml.Unmarshal(rr.Body.Bytes(), &errResp); err != nil {
t.Fatalf("failed to parse error XML: %v", err)
}
if errResp.Code != "KeyTooLongError" {
t.Errorf("expected error code KeyTooLongError, got %s", errResp.Code)
}
}
func TestNewMultipartUploadHandler_KeyTooLong(t *testing.T) {
s3a := &S3ApiServer{}
router := setupKeyLengthTestRouter(s3a.NewMultipartUploadHandler)
longKey := strings.Repeat("a", s3_constants.MaxS3ObjectKeyLength+1)
req := httptest.NewRequest(http.MethodPost, "/bucket/"+longKey+"?uploads", nil)
rr := httptest.NewRecorder()
router.ServeHTTP(rr, req)
if rr.Code != http.StatusBadRequest {
t.Errorf("expected status %d, got %d", http.StatusBadRequest, rr.Code)
}
var errResp s3err.RESTErrorResponse
if err := xml.Unmarshal(rr.Body.Bytes(), &errResp); err != nil {
t.Fatalf("failed to parse error XML: %v", err)
}
if errResp.Code != "KeyTooLongError" {
t.Errorf("expected error code KeyTooLongError, got %s", errResp.Code)
}
}
type testObjectWriteLockFactory struct {
mu sync.Mutex
locks map[string]*sync.Mutex
}
func (f *testObjectWriteLockFactory) newLock(bucket, object string) objectWriteLock {
key := bucket + "|" + object
f.mu.Lock()
lock, ok := f.locks[key]
if !ok {
lock = &sync.Mutex{}
f.locks[key] = lock
}
f.mu.Unlock()
lock.Lock()
return &testObjectWriteLock{unlock: lock.Unlock}
}
type testObjectWriteLock struct {
once sync.Once
unlock func()
}
func (l *testObjectWriteLock) StopShortLivedLock() error {
l.once.Do(l.unlock)
return nil
}
func TestWithObjectWriteLockSerializesConcurrentPreconditions(t *testing.T) {
s3a := NewS3ApiServerForTest()
lockFactory := &testObjectWriteLockFactory{
locks: make(map[string]*sync.Mutex),
}
s3a.newObjectWriteLock = lockFactory.newLock
const workers = 3
const bucket = "test-bucket"
const object = "/file.txt"
start := make(chan struct{})
results := make(chan s3err.ErrorCode, workers)
var wg sync.WaitGroup
var stateMu sync.Mutex
objectExists := false
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
<-start
errCode := s3a.withObjectWriteLock(bucket, object,
func() s3err.ErrorCode {
stateMu.Lock()
defer stateMu.Unlock()
if objectExists {
return s3err.ErrPreconditionFailed
}
return s3err.ErrNone
},
func() s3err.ErrorCode {
stateMu.Lock()
defer stateMu.Unlock()
objectExists = true
return s3err.ErrNone
},
)
results <- errCode
}()
}
close(start)
wg.Wait()
close(results)
var successCount int
var preconditionFailedCount int
for errCode := range results {
switch errCode {
case s3err.ErrNone:
successCount++
case s3err.ErrPreconditionFailed:
preconditionFailedCount++
default:
t.Fatalf("unexpected error code: %v", errCode)
}
}
if successCount != 1 {
t.Fatalf("expected exactly one successful writer, got %d", successCount)
}
if preconditionFailedCount != workers-1 {
t.Fatalf("expected %d precondition failures, got %d", workers-1, preconditionFailedCount)
}
}
func TestResolveFileMode(t *testing.T) {
tests := []struct {
name string
acl string
defaultFileMode uint32
expected uint32
}{
{"no acl, no default", "", 0, 0660},
{"no acl, with default", "", 0644, 0644},
{"private", s3_constants.CannedAclPrivate, 0, 0660},
{"private overrides default", s3_constants.CannedAclPrivate, 0644, 0660},
{"public-read", s3_constants.CannedAclPublicRead, 0, 0644},
{"public-read overrides default", s3_constants.CannedAclPublicRead, 0666, 0644},
{"public-read-write", s3_constants.CannedAclPublicReadWrite, 0, 0666},
{"authenticated-read", s3_constants.CannedAclAuthenticatedRead, 0, 0644},
{"bucket-owner-read", s3_constants.CannedAclBucketOwnerRead, 0, 0644},
{"bucket-owner-full-control", s3_constants.CannedAclBucketOwnerFullControl, 0, 0660},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3a := &S3ApiServer{
option: &S3ApiServerOption{
DefaultFileMode: tt.defaultFileMode,
},
}
req := httptest.NewRequest(http.MethodPut, "/bucket/object", nil)
if tt.acl != "" {
req.Header.Set(s3_constants.AmzCannedAcl, tt.acl)
}
got := s3a.resolveFileMode(req)
if got != tt.expected {
t.Errorf("resolveFileMode() = %04o, want %04o", got, tt.expected)
}
})
}
}

244
weed/s3api/s3api_object_handlers_test.go
View File

@@ -1,244 +0,0 @@
package s3api
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
)
func TestNewListEntryOwnerDisplayName(t *testing.T) {
// Create S3ApiServer with a properly initialized IAM
s3a := &S3ApiServer{
iam: &IdentityAccessManagement{
accounts: map[string]*Account{
"testid": {Id: "testid", DisplayName: "M. Tester"},
"userid123": {Id: "userid123", DisplayName: "John Doe"},
},
},
}
// Create test entry with owner metadata
entry := &filer_pb.Entry{
Name: "test-object",
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
FileSize: 1024,
},
Extended: map[string][]byte{
s3_constants.ExtAmzOwnerKey: []byte("testid"),
},
}
// Test that display name is correctly looked up from IAM
listEntry := newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
assert.NotNil(t, listEntry.Owner, "Owner should be set when fetchOwner is true")
assert.Equal(t, "testid", listEntry.Owner.ID, "Owner ID should match stored owner")
assert.Equal(t, "M. Tester", listEntry.Owner.DisplayName, "Display name should be looked up from IAM")
// Test with owner that doesn't exist in IAM (should fallback to ID)
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte("unknown-user")
listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
assert.Equal(t, "unknown-user", listEntry.Owner.ID, "Owner ID should match stored owner")
assert.Equal(t, "unknown-user", listEntry.Owner.DisplayName, "Display name should fallback to ID when not found in IAM")
// Test with no owner metadata (should use anonymous)
entry.Extended = make(map[string][]byte)
listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", true, false, false)
assert.Equal(t, s3_constants.AccountAnonymousId, listEntry.Owner.ID, "Should use anonymous ID when no owner metadata")
assert.Equal(t, "anonymous", listEntry.Owner.DisplayName, "Should use anonymous display name when no owner metadata")
// Test with fetchOwner false (should not set owner)
listEntry = newListEntry(s3a, entry, "", "dir", "test-object", "/buckets/test/", false, false, false)
assert.Nil(t, listEntry.Owner, "Owner should not be set when fetchOwner is false")
}
func TestRemoveDuplicateSlashes(t *testing.T) {
tests := []struct {
name string
path string
expectedResult string
}{
{
name: "empty",
path: "",
expectedResult: "",
},
{
name: "slash",
path: "/",
expectedResult: "/",
},
{
name: "object",
path: "object",
expectedResult: "object",
},
{
name: "correct path",
path: "/path/to/object",
expectedResult: "/path/to/object",
},
{
name: "path with duplicates",
path: "///path//to/object//",
expectedResult: "/path/to/object/",
},
}
for _, tst := range tests {
t.Run(tst.name, func(t *testing.T) {
obj := removeDuplicateSlashes(tst.path)
assert.Equal(t, tst.expectedResult, obj)
})
}
}
func TestS3ApiServer_toFilerPath(t *testing.T) {
tests := []struct {
name string
args string
want string
}{
{
"simple",
"/uploads/eaf10b3b-3b3a-4dcd-92a7-edf2a512276e/67b8b9bf-7cca-4cb6-9b34-22fcb4d6e27d/Bildschirmfoto 2022-09-19 um 21.38.37.png",
"/uploads/eaf10b3b-3b3a-4dcd-92a7-edf2a512276e/67b8b9bf-7cca-4cb6-9b34-22fcb4d6e27d/Bildschirmfoto%202022-09-19%20um%2021.38.37.png",
},
{
"double prefix",
"//uploads/t.png",
"/uploads/t.png",
},
{
"triple prefix",
"///uploads/t.png",
"/uploads/t.png",
},
{
"empty prefix",
"uploads/t.png",
"/uploads/t.png",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, urlEscapeObject(tt.args), "clean %v", tt.args)
})
}
}
func TestPartNumberWithRangeHeader(t *testing.T) {
tests := []struct {
name string
partStartOffset int64 // Part's start offset in the object
partEndOffset int64 // Part's end offset in the object
clientRangeHeader string
expectedStart int64 // Expected absolute start offset
expectedEnd int64 // Expected absolute end offset
expectError bool
}{
{
name: "No client range - full part",
partStartOffset: 1000,
partEndOffset: 1999,
clientRangeHeader: "",
expectedStart: 1000,
expectedEnd: 1999,
expectError: false,
},
{
name: "Range within part - start and end",
partStartOffset: 1000,
partEndOffset: 1999, // Part size: 1000 bytes
clientRangeHeader: "bytes=0-99",
expectedStart: 1000, // 1000 + 0
expectedEnd: 1099, // 1000 + 99
expectError: false,
},
{
name: "Range within part - start to end",
partStartOffset: 1000,
partEndOffset: 1999,
clientRangeHeader: "bytes=100-",
expectedStart: 1100, // 1000 + 100
expectedEnd: 1999, // 1000 + 999 (end of part)
expectError: false,
},
{
name: "Range suffix - last 100 bytes",
partStartOffset: 1000,
partEndOffset: 1999, // Part size: 1000 bytes
clientRangeHeader: "bytes=-100",
expectedStart: 1900, // 1000 + (1000 - 100)
expectedEnd: 1999, // 1000 + 999
expectError: false,
},
{
name: "Range suffix larger than part",
partStartOffset: 1000,
partEndOffset: 1999, // Part size: 1000 bytes
clientRangeHeader: "bytes=-2000",
expectedStart: 1000, // Start of part (clamped)
expectedEnd: 1999, // End of part
expectError: false,
},
{
name: "Range start beyond part size",
partStartOffset: 1000,
partEndOffset: 1999,
clientRangeHeader: "bytes=1000-1100",
expectedStart: 0,
expectedEnd: 0,
expectError: true,
},
{
name: "Range end clamped to part size",
partStartOffset: 1000,
partEndOffset: 1999,
clientRangeHeader: "bytes=0-2000",
expectedStart: 1000, // 1000 + 0
expectedEnd: 1999, // Clamped to end of part
expectError: false,
},
{
name: "Single byte range at start",
partStartOffset: 5000,
partEndOffset: 9999, // Part size: 5000 bytes
clientRangeHeader: "bytes=0-0",
expectedStart: 5000,
expectedEnd: 5000,
expectError: false,
},
{
name: "Single byte range in middle",
partStartOffset: 5000,
partEndOffset: 9999,
clientRangeHeader: "bytes=100-100",
expectedStart: 5100,
expectedEnd: 5100,
expectError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Test the actual range adjustment logic from GetObjectHandler
startOffset, endOffset, err := adjustRangeForPart(tt.partStartOffset, tt.partEndOffset, tt.clientRangeHeader)
if tt.expectError {
assert.Error(t, err, "Expected error for range %s", tt.clientRangeHeader)
} else {
assert.NoError(t, err, "Unexpected error for range %s: %v", tt.clientRangeHeader, err)
assert.Equal(t, tt.expectedStart, startOffset, "Start offset mismatch")
assert.Equal(t, tt.expectedEnd, endOffset, "End offset mismatch")
}
})
}
}

8
weed/s3api/s3api_sosapi.go
View File

@@ -14,7 +14,6 @@ import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -97,13 +96,6 @@ func isSOSAPIObject(object string) bool {
}
}
// isSOSAPIClient checks if the request comes from a SOSAPI-compatible client
// by examining the User-Agent header.
func isSOSAPIClient(r *http.Request) bool {
userAgent := r.Header.Get("User-Agent")
return strings.Contains(userAgent, sosAPIClientUserAgent)
}
// generateSystemXML creates the system.xml response containing storage system
// capabilities and recommendations.
func generateSystemXML() ([]byte, error) {

248
weed/s3api/s3api_sosapi_test.go
View File

@@ -1,248 +0,0 @@
package s3api
import (
"encoding/xml"
"net/http/httptest"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestIsSOSAPIObject(t *testing.T) {
tests := []struct {
name string
object string
expected bool
}{
{
name: "system.xml should be detected",
object: ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
expected: true,
},
{
name: "capacity.xml should be detected",
object: ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml",
expected: true,
},
{
name: "regular object should not be detected",
object: "myfile.txt",
expected: false,
},
{
name: "similar but different path should not be detected",
object: ".system-other-uuid/system.xml",
expected: false,
},
{
name: "nested path should not be detected",
object: "prefix/.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
expected: false,
},
{
name: "empty string should not be detected",
object: "",
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isSOSAPIObject(tt.object)
if result != tt.expected {
t.Errorf("isSOSAPIObject(%q) = %v, want %v", tt.object, result, tt.expected)
}
})
}
}
func TestIsSOSAPIClient(t *testing.T) {
tests := []struct {
name string
userAgent string
expected bool
}{
{
name: "Veeam backup client should be detected",
userAgent: "APN/1.0 Veeam/1.0 Backup/10.0",
expected: true,
},
{
name: "exact match should be detected",
userAgent: "APN/1.0 Veeam/1.0",
expected: true,
},
{
name: "AWS CLI should not be detected",
userAgent: "aws-cli/2.0.0 Python/3.8",
expected: false,
},
{
name: "empty user agent should not be detected",
userAgent: "",
expected: false,
},
{
name: "partial match should not be detected",
userAgent: "Veeam/1.0",
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := httptest.NewRequest("GET", "/bucket/object", nil)
req.Header.Set("User-Agent", tt.userAgent)
result := isSOSAPIClient(req)
if result != tt.expected {
t.Errorf("isSOSAPIClient() with User-Agent %q = %v, want %v", tt.userAgent, result, tt.expected)
}
})
}
}
func TestGenerateSystemXML(t *testing.T) {
xmlData, err := generateSystemXML()
if err != nil {
t.Fatalf("generateSystemXML() failed: %v", err)
}
// Verify it's valid XML
var si SystemInfo
if err := xml.Unmarshal(xmlData, &si); err != nil {
t.Fatalf("generated XML is invalid: %v", err)
}
// Verify required fields
if si.ProtocolVersion != sosAPIProtocolVersion {
t.Errorf("ProtocolVersion = %q, want %q", si.ProtocolVersion, sosAPIProtocolVersion)
}
if !strings.Contains(si.ModelName, "SeaweedFS") {
t.Errorf("ModelName = %q, should contain 'SeaweedFS'", si.ModelName)
}
if !si.ProtocolCapabilities.CapacityInfo {
t.Error("ProtocolCapabilities.CapacityInfo should be true")
}
if si.SystemRecommendations == nil {
t.Fatal("SystemRecommendations should not be nil")
}
if si.SystemRecommendations.KBBlockSize != sosAPIDefaultBlockSizeKB {
t.Errorf("KBBlockSize = %d, want %d", si.SystemRecommendations.KBBlockSize, sosAPIDefaultBlockSizeKB)
}
}
func TestSOSAPIObjectDetectionEdgeCases(t *testing.T) {
edgeCases := []struct {
object string
expected bool
}{
// With leading slash
{"/.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml", false},
// URL encoded
{".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c%2Fsystem.xml", false},
// Mixed case
{".System-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml", false},
// Extra slashes
{".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c//system.xml", false},
// Correct paths
{".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml", true},
{".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml", true},
}
for _, tc := range edgeCases {
result := isSOSAPIObject(tc.object)
if result != tc.expected {
t.Errorf("isSOSAPIObject(%q) = %v, want %v", tc.object, result, tc.expected)
}
}
}
func TestCollectBucketUsageFromTopology(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
{
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Size: 100, Collection: "bucket1"},
{Id: 2, Size: 200, Collection: "bucket2"},
{Id: 3, Size: 300, Collection: "bucket1"},
{Id: 1, Size: 100, Collection: "bucket1"}, // Duplicate (replica), should be ignored
},
},
},
},
},
},
},
},
},
}
usage := collectBucketUsageFromTopology(topo, "bucket1")
expected := int64(400) // 100 + 300
if usage != expected {
t.Errorf("collectBucketUsageFromTopology = %d, want %d", usage, expected)
}
usage2 := collectBucketUsageFromTopology(topo, "bucket2")
expected2 := int64(200)
if usage2 != expected2 {
t.Errorf("collectBucketUsageFromTopology = %d, want %d", usage2, expected2)
}
}
func TestCalculateClusterCapacity(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
{
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
MaxVolumeCount: 100,
FreeVolumeCount: 40,
},
},
},
{
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
MaxVolumeCount: 200,
FreeVolumeCount: 160,
},
},
},
},
},
},
},
},
}
volumeSizeLimitMb := uint64(1000) // 1GB
volumeSizeBytes := int64(1000) * 1024 * 1024
total, available := calculateClusterCapacity(topo, volumeSizeLimitMb)
expectedTotal := int64(300) * volumeSizeBytes
expectedAvailable := int64(200) * volumeSizeBytes
if total != expectedTotal {
t.Errorf("calculateClusterCapacity total = %d, want %d", total, expectedTotal)
}
if available != expectedAvailable {
t.Errorf("calculateClusterCapacity available = %d, want %d", available, expectedAvailable)
}
}

361
weed/s3api/s3api_sse_chunk_metadata_test.go
View File

@@ -1,361 +0,0 @@
package s3api
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/json"
"io"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// TestSSEKMSChunkMetadataAssignment tests that SSE-KMS creates per-chunk metadata
// with correct ChunkOffset values for each chunk (matching the fix in putToFiler)
func TestSSEKMSChunkMetadataAssignment(t *testing.T) {
kmsKey := SetupTestKMS(t)
defer kmsKey.Cleanup()
// Generate SSE-KMS key by encrypting test data (this gives us a real SSEKMSKey)
encryptionContext := BuildEncryptionContext("test-bucket", "test-object", false)
testData := "Test data for SSE-KMS chunk metadata validation"
encryptedReader, sseKMSKey, err := CreateSSEKMSEncryptedReader(bytes.NewReader([]byte(testData)), kmsKey.KeyID, encryptionContext)
if err != nil {
t.Fatalf("Failed to create encrypted reader: %v", err)
}
// Read to complete encryption setup
io.ReadAll(encryptedReader)
// Serialize the base metadata (what putToFiler receives before chunking)
baseMetadata, err := SerializeSSEKMSMetadata(sseKMSKey)
if err != nil {
t.Fatalf("Failed to serialize base SSE-KMS metadata: %v", err)
}
// Simulate multi-chunk upload scenario (what putToFiler does after UploadReaderInChunks)
simulatedChunks := []*filer_pb.FileChunk{
{FileId: "chunk1", Offset: 0, Size: 8 * 1024 * 1024}, // 8MB chunk at offset 0
{FileId: "chunk2", Offset: 8 * 1024 * 1024, Size: 8 * 1024 * 1024}, // 8MB chunk at offset 8MB
{FileId: "chunk3", Offset: 16 * 1024 * 1024, Size: 4 * 1024 * 1024}, // 4MB chunk at offset 16MB
}
// THIS IS THE CRITICAL FIX: Create per-chunk metadata (lines 421-443 in putToFiler)
for _, chunk := range simulatedChunks {
chunk.SseType = filer_pb.SSEType_SSE_KMS
// Create a copy of the SSE-KMS key with chunk-specific offset
chunkSSEKey := &SSEKMSKey{
KeyID: sseKMSKey.KeyID,
EncryptedDataKey: sseKMSKey.EncryptedDataKey,
EncryptionContext: sseKMSKey.EncryptionContext,
BucketKeyEnabled: sseKMSKey.BucketKeyEnabled,
IV: sseKMSKey.IV,
ChunkOffset: chunk.Offset, // Set chunk-specific offset
}
// Serialize per-chunk metadata
chunkMetadata, serErr := SerializeSSEKMSMetadata(chunkSSEKey)
if serErr != nil {
t.Fatalf("Failed to serialize SSE-KMS metadata for chunk at offset %d: %v", chunk.Offset, serErr)
}
chunk.SseMetadata = chunkMetadata
}
// VERIFICATION 1: Each chunk should have different metadata (due to different ChunkOffset)
metadataSet := make(map[string]bool)
for i, chunk := range simulatedChunks {
metadataStr := string(chunk.SseMetadata)
if metadataSet[metadataStr] {
t.Errorf("Chunk %d has duplicate metadata (should be unique per chunk)", i)
}
metadataSet[metadataStr] = true
// Deserialize and verify ChunkOffset
var metadata SSEKMSMetadata
if err := json.Unmarshal(chunk.SseMetadata, &metadata); err != nil {
t.Fatalf("Failed to deserialize chunk %d metadata: %v", i, err)
}
expectedOffset := chunk.Offset
if metadata.PartOffset != expectedOffset {
t.Errorf("Chunk %d: expected PartOffset=%d, got %d", i, expectedOffset, metadata.PartOffset)
}
t.Logf("✓ Chunk %d: PartOffset=%d (correct)", i, metadata.PartOffset)
}
// VERIFICATION 2: Verify metadata can be deserialized and has correct ChunkOffset
for i, chunk := range simulatedChunks {
// Deserialize chunk metadata
deserializedKey, err := DeserializeSSEKMSMetadata(chunk.SseMetadata)
if err != nil {
t.Fatalf("Failed to deserialize chunk %d metadata: %v", i, err)
}
// Verify the deserialized key has correct ChunkOffset
if deserializedKey.ChunkOffset != chunk.Offset {
t.Errorf("Chunk %d: deserialized ChunkOffset=%d, expected %d",
i, deserializedKey.ChunkOffset, chunk.Offset)
}
// Verify IV is set (should be inherited from base)
if len(deserializedKey.IV) != aes.BlockSize {
t.Errorf("Chunk %d: invalid IV length: %d", i, len(deserializedKey.IV))
}
// Verify KeyID matches
if deserializedKey.KeyID != sseKMSKey.KeyID {
t.Errorf("Chunk %d: KeyID mismatch", i)
}
t.Logf("✓ Chunk %d: metadata deserialized successfully (ChunkOffset=%d, KeyID=%s)",
i, deserializedKey.ChunkOffset, deserializedKey.KeyID)
}
// VERIFICATION 3: Ensure base metadata is NOT reused (the bug we're preventing)
var baseMetadataStruct SSEKMSMetadata
if err := json.Unmarshal(baseMetadata, &baseMetadataStruct); err != nil {
t.Fatalf("Failed to deserialize base metadata: %v", err)
}
// Base metadata should have ChunkOffset=0
if baseMetadataStruct.PartOffset != 0 {
t.Errorf("Base metadata should have PartOffset=0, got %d", baseMetadataStruct.PartOffset)
}
// Chunks 2 and 3 should NOT have the same metadata as base (proving we're not reusing)
for i := 1; i < len(simulatedChunks); i++ {
if bytes.Equal(simulatedChunks[i].SseMetadata, baseMetadata) {
t.Errorf("CRITICAL BUG: Chunk %d reuses base metadata (should have per-chunk metadata)", i)
}
}
t.Log("✓ All chunks have unique per-chunk metadata (bug prevented)")
}
// TestSSES3ChunkMetadataAssignment tests that SSE-S3 creates per-chunk metadata
// with offset-adjusted IVs for each chunk (matching the fix in putToFiler)
func TestSSES3ChunkMetadataAssignment(t *testing.T) {
// Initialize global SSE-S3 key manager
globalSSES3KeyManager = NewSSES3KeyManager()
defer func() {
globalSSES3KeyManager = NewSSES3KeyManager()
}()
keyManager := GetSSES3KeyManager()
keyManager.superKey = make([]byte, 32)
rand.Read(keyManager.superKey)
// Generate SSE-S3 key
sseS3Key, err := GenerateSSES3Key()
if err != nil {
t.Fatalf("Failed to generate SSE-S3 key: %v", err)
}
// Generate base IV
baseIV := make([]byte, aes.BlockSize)
rand.Read(baseIV)
sseS3Key.IV = baseIV
// Serialize base metadata (what putToFiler receives)
baseMetadata, err := SerializeSSES3Metadata(sseS3Key)
if err != nil {
t.Fatalf("Failed to serialize base SSE-S3 metadata: %v", err)
}
// Simulate multi-chunk upload scenario (what putToFiler does after UploadReaderInChunks)
simulatedChunks := []*filer_pb.FileChunk{
{FileId: "chunk1", Offset: 0, Size: 8 * 1024 * 1024}, // 8MB chunk at offset 0
{FileId: "chunk2", Offset: 8 * 1024 * 1024, Size: 8 * 1024 * 1024}, // 8MB chunk at offset 8MB
{FileId: "chunk3", Offset: 16 * 1024 * 1024, Size: 4 * 1024 * 1024}, // 4MB chunk at offset 16MB
}
// THIS IS THE CRITICAL FIX: Create per-chunk metadata (lines 444-468 in putToFiler)
for _, chunk := range simulatedChunks {
chunk.SseType = filer_pb.SSEType_SSE_S3
// Calculate chunk-specific IV using base IV and chunk offset
chunkIV, _ := calculateIVWithOffset(sseS3Key.IV, chunk.Offset)
// Create a copy of the SSE-S3 key with chunk-specific IV
chunkSSEKey := &SSES3Key{
Key: sseS3Key.Key,
KeyID: sseS3Key.KeyID,
Algorithm: sseS3Key.Algorithm,
IV: chunkIV, // Use chunk-specific IV
}
// Serialize per-chunk metadata
chunkMetadata, serErr := SerializeSSES3Metadata(chunkSSEKey)
if serErr != nil {
t.Fatalf("Failed to serialize SSE-S3 metadata for chunk at offset %d: %v", chunk.Offset, serErr)
}
chunk.SseMetadata = chunkMetadata
}
// VERIFICATION 1: Each chunk should have different metadata (due to different IVs)
metadataSet := make(map[string]bool)
for i, chunk := range simulatedChunks {
metadataStr := string(chunk.SseMetadata)
if metadataSet[metadataStr] {
t.Errorf("Chunk %d has duplicate metadata (should be unique per chunk)", i)
}
metadataSet[metadataStr] = true
// Deserialize and verify IV
deserializedKey, err := DeserializeSSES3Metadata(chunk.SseMetadata, keyManager)
if err != nil {
t.Fatalf("Failed to deserialize chunk %d metadata: %v", i, err)
}
// Calculate expected IV for this chunk
expectedIV, _ := calculateIVWithOffset(baseIV, chunk.Offset)
if !bytes.Equal(deserializedKey.IV, expectedIV) {
t.Errorf("Chunk %d: IV mismatch\nExpected: %x\nGot: %x",
i, expectedIV[:8], deserializedKey.IV[:8])
}
t.Logf("✓ Chunk %d: IV correctly adjusted for offset=%d", i, chunk.Offset)
}
// VERIFICATION 2: Verify decryption works with per-chunk IVs
for i, chunk := range simulatedChunks {
// Deserialize chunk metadata
deserializedKey, err := DeserializeSSES3Metadata(chunk.SseMetadata, keyManager)
if err != nil {
t.Fatalf("Failed to deserialize chunk %d metadata: %v", i, err)
}
// Simulate encryption/decryption with the chunk's IV
testData := []byte("Test data for SSE-S3 chunk decryption verification")
block, err := aes.NewCipher(deserializedKey.Key)
if err != nil {
t.Fatalf("Failed to create cipher: %v", err)
}
// Encrypt with chunk's IV
ciphertext := make([]byte, len(testData))
stream := cipher.NewCTR(block, deserializedKey.IV)
stream.XORKeyStream(ciphertext, testData)
// Decrypt with chunk's IV
plaintext := make([]byte, len(ciphertext))
block2, _ := aes.NewCipher(deserializedKey.Key)
stream2 := cipher.NewCTR(block2, deserializedKey.IV)
stream2.XORKeyStream(plaintext, ciphertext)
if !bytes.Equal(plaintext, testData) {
t.Errorf("Chunk %d: decryption failed", i)
}
t.Logf("✓ Chunk %d: encryption/decryption successful with chunk-specific IV", i)
}
// VERIFICATION 3: Ensure base IV is NOT reused for non-zero offset chunks (the bug we're preventing)
for i := 1; i < len(simulatedChunks); i++ {
if bytes.Equal(simulatedChunks[i].SseMetadata, baseMetadata) {
t.Errorf("CRITICAL BUG: Chunk %d reuses base metadata (should have per-chunk metadata)", i)
}
// Verify chunk metadata has different IV than base IV
deserializedKey, _ := DeserializeSSES3Metadata(simulatedChunks[i].SseMetadata, keyManager)
if bytes.Equal(deserializedKey.IV, baseIV) {
t.Errorf("CRITICAL BUG: Chunk %d uses base IV (should use offset-adjusted IV)", i)
}
}
t.Log("✓ All chunks have unique per-chunk IVs (bug prevented)")
}
// TestSSEChunkMetadataComparison tests that the bug (reusing same metadata for all chunks)
// would cause decryption failures, while the fix (per-chunk metadata) works correctly
func TestSSEChunkMetadataComparison(t *testing.T) {
// Generate test key and IV
key := make([]byte, 32)
rand.Read(key)
baseIV := make([]byte, aes.BlockSize)
rand.Read(baseIV)
// Create test data for 3 chunks
chunk0Data := []byte("Chunk 0 data at offset 0")
chunk1Data := []byte("Chunk 1 data at offset 8MB")
chunk2Data := []byte("Chunk 2 data at offset 16MB")
chunkOffsets := []int64{0, 8 * 1024 * 1024, 16 * 1024 * 1024}
chunkDataList := [][]byte{chunk0Data, chunk1Data, chunk2Data}
// Scenario 1: BUG - Using same IV for all chunks (what the old code did)
t.Run("Bug: Reusing base IV causes decryption failures", func(t *testing.T) {
var encryptedChunks [][]byte
// Encrypt each chunk with offset-adjusted IV (what encryption does)
for i, offset := range chunkOffsets {
adjustedIV, _ := calculateIVWithOffset(baseIV, offset)
block, _ := aes.NewCipher(key)
stream := cipher.NewCTR(block, adjustedIV)
ciphertext := make([]byte, len(chunkDataList[i]))
stream.XORKeyStream(ciphertext, chunkDataList[i])
encryptedChunks = append(encryptedChunks, ciphertext)
}
// Try to decrypt with base IV (THE BUG)
for i := range encryptedChunks {
block, _ := aes.NewCipher(key)
stream := cipher.NewCTR(block, baseIV) // BUG: Always using base IV
plaintext := make([]byte, len(encryptedChunks[i]))
stream.XORKeyStream(plaintext, encryptedChunks[i])
if i == 0 {
// Chunk 0 should work (offset 0 means base IV = adjusted IV)
if !bytes.Equal(plaintext, chunkDataList[i]) {
t.Errorf("Chunk 0 decryption failed (unexpected)")
}
} else {
// Chunks 1 and 2 should FAIL (wrong IV)
if bytes.Equal(plaintext, chunkDataList[i]) {
t.Errorf("BUG NOT REPRODUCED: Chunk %d decrypted correctly with base IV (should fail)", i)
} else {
t.Logf("✓ Chunk %d: Correctly failed to decrypt with base IV (bug reproduced)", i)
}
}
}
})
// Scenario 2: FIX - Using per-chunk offset-adjusted IVs (what the new code does)
t.Run("Fix: Per-chunk IVs enable correct decryption", func(t *testing.T) {
var encryptedChunks [][]byte
var chunkIVs [][]byte
// Encrypt each chunk with offset-adjusted IV
for i, offset := range chunkOffsets {
adjustedIV, _ := calculateIVWithOffset(baseIV, offset)
chunkIVs = append(chunkIVs, adjustedIV)
block, _ := aes.NewCipher(key)
stream := cipher.NewCTR(block, adjustedIV)
ciphertext := make([]byte, len(chunkDataList[i]))
stream.XORKeyStream(ciphertext, chunkDataList[i])
encryptedChunks = append(encryptedChunks, ciphertext)
}
// Decrypt with per-chunk IVs (THE FIX)
for i := range encryptedChunks {
block, _ := aes.NewCipher(key)
stream := cipher.NewCTR(block, chunkIVs[i]) // FIX: Using per-chunk IV
plaintext := make([]byte, len(encryptedChunks[i]))
stream.XORKeyStream(plaintext, encryptedChunks[i])
if !bytes.Equal(plaintext, chunkDataList[i]) {
t.Errorf("Chunk %d decryption failed with per-chunk IV (unexpected)", i)
} else {
t.Logf("✓ Chunk %d: Successfully decrypted with per-chunk IV", i)
}
}
})
}

601
weed/s3api/s3api_streaming_copy.go
View File

@@ -1,601 +0,0 @@
package s3api
import (
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"io"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// StreamingCopySpec defines the specification for streaming copy operations
type StreamingCopySpec struct {
SourceReader io.Reader
TargetSize int64
EncryptionSpec *EncryptionSpec
CompressionSpec *CompressionSpec
HashCalculation bool
BufferSize int
}
// EncryptionSpec defines encryption parameters for streaming
type EncryptionSpec struct {
NeedsDecryption bool
NeedsEncryption bool
SourceKey interface{} // SSECustomerKey or SSEKMSKey
DestinationKey interface{} // SSECustomerKey or SSEKMSKey
SourceType EncryptionType
DestinationType EncryptionType
SourceMetadata map[string][]byte // Source metadata for IV extraction
DestinationIV []byte // Generated IV for destination
}
// CompressionSpec defines compression parameters for streaming
type CompressionSpec struct {
IsCompressed bool
CompressionType string
NeedsDecompression bool
NeedsCompression bool
}
// StreamingCopyManager handles streaming copy operations
type StreamingCopyManager struct {
s3a *S3ApiServer
bufferSize int
}
// NewStreamingCopyManager creates a new streaming copy manager
func NewStreamingCopyManager(s3a *S3ApiServer) *StreamingCopyManager {
return &StreamingCopyManager{
s3a: s3a,
bufferSize: 64 * 1024, // 64KB default buffer
}
}
// ExecuteStreamingCopy performs a streaming copy operation and returns the encryption spec
// The encryption spec is needed for SSE-S3 to properly set destination metadata (fixes GitHub #7562)
func (scm *StreamingCopyManager) ExecuteStreamingCopy(ctx context.Context, entry *filer_pb.Entry, r *http.Request, dstPath string, state *EncryptionState) ([]*filer_pb.FileChunk, *EncryptionSpec, error) {
// Create streaming copy specification
spec, err := scm.createStreamingSpec(entry, r, state)
if err != nil {
return nil, nil, fmt.Errorf("create streaming spec: %w", err)
}
// Create source reader from entry
sourceReader, err := scm.createSourceReader(entry)
if err != nil {
return nil, nil, fmt.Errorf("create source reader: %w", err)
}
defer sourceReader.Close()
spec.SourceReader = sourceReader
// Create processing pipeline
processedReader, err := scm.createProcessingPipeline(spec)
if err != nil {
return nil, nil, fmt.Errorf("create processing pipeline: %w", err)
}
// Stream to destination
chunks, err := scm.streamToDestination(ctx, processedReader, spec, dstPath)
if err != nil {
return nil, nil, err
}
return chunks, spec.EncryptionSpec, nil
}
// createStreamingSpec creates a streaming specification based on copy parameters
func (scm *StreamingCopyManager) createStreamingSpec(entry *filer_pb.Entry, r *http.Request, state *EncryptionState) (*StreamingCopySpec, error) {
spec := &StreamingCopySpec{
BufferSize: scm.bufferSize,
HashCalculation: true,
}
// Calculate target size
sizeCalc := NewCopySizeCalculator(entry, r)
spec.TargetSize = sizeCalc.CalculateTargetSize()
// Create encryption specification
encSpec, err := scm.createEncryptionSpec(entry, r, state)
if err != nil {
return nil, err
}
spec.EncryptionSpec = encSpec
// Create compression specification
spec.CompressionSpec = scm.createCompressionSpec(entry, r)
return spec, nil
}
// createEncryptionSpec creates encryption specification for streaming
func (scm *StreamingCopyManager) createEncryptionSpec(entry *filer_pb.Entry, r *http.Request, state *EncryptionState) (*EncryptionSpec, error) {
spec := &EncryptionSpec{
NeedsDecryption: state.IsSourceEncrypted(),
NeedsEncryption: state.IsTargetEncrypted(),
SourceMetadata: entry.Extended, // Pass source metadata for IV extraction
}
// Set source encryption details
if state.SrcSSEC {
spec.SourceType = EncryptionTypeSSEC
sourceKey, err := ParseSSECCopySourceHeaders(r)
if err != nil {
return nil, fmt.Errorf("parse SSE-C copy source headers: %w", err)
}
spec.SourceKey = sourceKey
} else if state.SrcSSEKMS {
spec.SourceType = EncryptionTypeSSEKMS
// Extract SSE-KMS key from metadata
if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSEKMSKey]; exists {
sseKey, err := DeserializeSSEKMSMetadata(keyData)
if err != nil {
return nil, fmt.Errorf("deserialize SSE-KMS metadata: %w", err)
}
spec.SourceKey = sseKey
}
} else if state.SrcSSES3 {
spec.SourceType = EncryptionTypeSSES3
// Extract SSE-S3 key from metadata
if keyData, exists := entry.Extended[s3_constants.SeaweedFSSSES3Key]; exists {
keyManager := GetSSES3KeyManager()
sseKey, err := DeserializeSSES3Metadata(keyData, keyManager)
if err != nil {
return nil, fmt.Errorf("deserialize SSE-S3 metadata: %w", err)
}
spec.SourceKey = sseKey
}
}
// Set destination encryption details
if state.DstSSEC {
spec.DestinationType = EncryptionTypeSSEC
destKey, err := ParseSSECHeaders(r)
if err != nil {
return nil, fmt.Errorf("parse SSE-C headers: %w", err)
}
spec.DestinationKey = destKey
} else if state.DstSSEKMS {
spec.DestinationType = EncryptionTypeSSEKMS
// Parse KMS parameters
keyID, encryptionContext, bucketKeyEnabled, err := ParseSSEKMSCopyHeaders(r)
if err != nil {
return nil, fmt.Errorf("parse SSE-KMS copy headers: %w", err)
}
// Create SSE-KMS key for destination
sseKey := &SSEKMSKey{
KeyID: keyID,
EncryptionContext: encryptionContext,
BucketKeyEnabled: bucketKeyEnabled,
}
spec.DestinationKey = sseKey
} else if state.DstSSES3 {
spec.DestinationType = EncryptionTypeSSES3
// Generate or retrieve SSE-S3 key
keyManager := GetSSES3KeyManager()
sseKey, err := keyManager.GetOrCreateKey("")
if err != nil {
return nil, fmt.Errorf("get SSE-S3 key: %w", err)
}
spec.DestinationKey = sseKey
}
return spec, nil
}
// createCompressionSpec creates compression specification for streaming
func (scm *StreamingCopyManager) createCompressionSpec(entry *filer_pb.Entry, r *http.Request) *CompressionSpec {
return &CompressionSpec{
IsCompressed: isCompressedEntry(entry),
// For now, we don't change compression during copy
NeedsDecompression: false,
NeedsCompression: false,
}
}
// createSourceReader creates a reader for the source entry
func (scm *StreamingCopyManager) createSourceReader(entry *filer_pb.Entry) (io.ReadCloser, error) {
// Create a multi-chunk reader that streams from all chunks
return scm.s3a.createMultiChunkReader(entry)
}
// createProcessingPipeline creates a processing pipeline for the copy operation
func (scm *StreamingCopyManager) createProcessingPipeline(spec *StreamingCopySpec) (io.Reader, error) {
reader := spec.SourceReader
// Add decryption if needed
if spec.EncryptionSpec.NeedsDecryption {
decryptedReader, err := scm.createDecryptionReader(reader, spec.EncryptionSpec)
if err != nil {
return nil, fmt.Errorf("create decryption reader: %w", err)
}
reader = decryptedReader
}
// Add decompression if needed
if spec.CompressionSpec.NeedsDecompression {
decompressedReader, err := scm.createDecompressionReader(reader, spec.CompressionSpec)
if err != nil {
return nil, fmt.Errorf("create decompression reader: %w", err)
}
reader = decompressedReader
}
// Add compression if needed
if spec.CompressionSpec.NeedsCompression {
compressedReader, err := scm.createCompressionReader(reader, spec.CompressionSpec)
if err != nil {
return nil, fmt.Errorf("create compression reader: %w", err)
}
reader = compressedReader
}
// Add encryption if needed
if spec.EncryptionSpec.NeedsEncryption {
encryptedReader, err := scm.createEncryptionReader(reader, spec.EncryptionSpec)
if err != nil {
return nil, fmt.Errorf("create encryption reader: %w", err)
}
reader = encryptedReader
}
// Add hash calculation if needed
if spec.HashCalculation {
reader = scm.createHashReader(reader)
}
return reader, nil
}
// createDecryptionReader creates a decryption reader based on encryption type
func (scm *StreamingCopyManager) createDecryptionReader(reader io.Reader, encSpec *EncryptionSpec) (io.Reader, error) {
switch encSpec.SourceType {
case EncryptionTypeSSEC:
if sourceKey, ok := encSpec.SourceKey.(*SSECustomerKey); ok {
// Get IV from metadata
iv, err := GetSSECIVFromMetadata(encSpec.SourceMetadata)
if err != nil {
return nil, fmt.Errorf("get IV from metadata: %w", err)
}
return CreateSSECDecryptedReader(reader, sourceKey, iv)
}
return nil, fmt.Errorf("invalid SSE-C source key type")
case EncryptionTypeSSEKMS:
if sseKey, ok := encSpec.SourceKey.(*SSEKMSKey); ok {
return CreateSSEKMSDecryptedReader(reader, sseKey)
}
return nil, fmt.Errorf("invalid SSE-KMS source key type")
case EncryptionTypeSSES3:
if sseKey, ok := encSpec.SourceKey.(*SSES3Key); ok {
// For SSE-S3, the IV is stored within the SSES3Key metadata, not as separate metadata
iv := sseKey.IV
if len(iv) == 0 {
return nil, fmt.Errorf("SSE-S3 key is missing IV for streaming copy")
}
return CreateSSES3DecryptedReader(reader, sseKey, iv)
}
return nil, fmt.Errorf("invalid SSE-S3 source key type")
default:
return reader, nil
}
}
// createEncryptionReader creates an encryption reader based on encryption type
func (scm *StreamingCopyManager) createEncryptionReader(reader io.Reader, encSpec *EncryptionSpec) (io.Reader, error) {
switch encSpec.DestinationType {
case EncryptionTypeSSEC:
if destKey, ok := encSpec.DestinationKey.(*SSECustomerKey); ok {
encryptedReader, iv, err := CreateSSECEncryptedReader(reader, destKey)
if err != nil {
return nil, err
}
// Store IV in destination metadata (this would need to be handled by caller)
encSpec.DestinationIV = iv
return encryptedReader, nil
}
return nil, fmt.Errorf("invalid SSE-C destination key type")
case EncryptionTypeSSEKMS:
if sseKey, ok := encSpec.DestinationKey.(*SSEKMSKey); ok {
encryptedReader, updatedKey, err := CreateSSEKMSEncryptedReaderWithBucketKey(reader, sseKey.KeyID, sseKey.EncryptionContext, sseKey.BucketKeyEnabled)
if err != nil {
return nil, err
}
// Store IV from the updated key
encSpec.DestinationIV = updatedKey.IV
return encryptedReader, nil
}
return nil, fmt.Errorf("invalid SSE-KMS destination key type")
case EncryptionTypeSSES3:
if sseKey, ok := encSpec.DestinationKey.(*SSES3Key); ok {
encryptedReader, iv, err := CreateSSES3EncryptedReader(reader, sseKey)
if err != nil {
return nil, err
}
// Store IV for metadata
encSpec.DestinationIV = iv
return encryptedReader, nil
}
return nil, fmt.Errorf("invalid SSE-S3 destination key type")
default:
return reader, nil
}
}
// createDecompressionReader creates a decompression reader
func (scm *StreamingCopyManager) createDecompressionReader(reader io.Reader, compSpec *CompressionSpec) (io.Reader, error) {
if !compSpec.NeedsDecompression {
return reader, nil
}
switch compSpec.CompressionType {
case "gzip":
// Use SeaweedFS's streaming gzip decompression
pr, pw := io.Pipe()
go func() {
defer pw.Close()
_, err := util.GunzipStream(pw, reader)
if err != nil {
pw.CloseWithError(fmt.Errorf("gzip decompression failed: %v", err))
}
}()
return pr, nil
default:
// Unknown compression type, return as-is
return reader, nil
}
}
// createCompressionReader creates a compression reader
func (scm *StreamingCopyManager) createCompressionReader(reader io.Reader, compSpec *CompressionSpec) (io.Reader, error) {
if !compSpec.NeedsCompression {
return reader, nil
}
switch compSpec.CompressionType {
case "gzip":
// Use SeaweedFS's streaming gzip compression
pr, pw := io.Pipe()
go func() {
defer pw.Close()
_, err := util.GzipStream(pw, reader)
if err != nil {
pw.CloseWithError(fmt.Errorf("gzip compression failed: %v", err))
}
}()
return pr, nil
default:
// Unknown compression type, return as-is
return reader, nil
}
}
// HashReader wraps an io.Reader to calculate MD5 and SHA256 hashes
type HashReader struct {
reader io.Reader
md5Hash hash.Hash
sha256Hash hash.Hash
}
// NewHashReader creates a new hash calculating reader
func NewHashReader(reader io.Reader) *HashReader {
return &HashReader{
reader: reader,
md5Hash: md5.New(),
sha256Hash: sha256.New(),
}
}
// Read implements io.Reader and calculates hashes as data flows through
func (hr *HashReader) Read(p []byte) (n int, err error) {
n, err = hr.reader.Read(p)
if n > 0 {
// Update both hashes with the data read
hr.md5Hash.Write(p[:n])
hr.sha256Hash.Write(p[:n])
}
return n, err
}
// MD5Sum returns the current MD5 hash
func (hr *HashReader) MD5Sum() []byte {
return hr.md5Hash.Sum(nil)
}
// SHA256Sum returns the current SHA256 hash
func (hr *HashReader) SHA256Sum() []byte {
return hr.sha256Hash.Sum(nil)
}
// MD5Hex returns the MD5 hash as a hex string
func (hr *HashReader) MD5Hex() string {
return hex.EncodeToString(hr.MD5Sum())
}
// SHA256Hex returns the SHA256 hash as a hex string
func (hr *HashReader) SHA256Hex() string {
return hex.EncodeToString(hr.SHA256Sum())
}
// createHashReader creates a hash calculation reader
func (scm *StreamingCopyManager) createHashReader(reader io.Reader) io.Reader {
return NewHashReader(reader)
}
// streamToDestination streams the processed data to the destination
func (scm *StreamingCopyManager) streamToDestination(ctx context.Context, reader io.Reader, spec *StreamingCopySpec, dstPath string) ([]*filer_pb.FileChunk, error) {
// For now, we'll use the existing chunk-based approach
// In a full implementation, this would stream directly to the destination
// without creating intermediate chunks
// This is a placeholder that converts back to chunk-based approach
// A full streaming implementation would write directly to the destination
return scm.streamToChunks(ctx, reader, spec, dstPath)
}
// streamToChunks converts streaming data back to chunks (temporary implementation)
func (scm *StreamingCopyManager) streamToChunks(ctx context.Context, reader io.Reader, spec *StreamingCopySpec, dstPath string) ([]*filer_pb.FileChunk, error) {
// This is a simplified implementation that reads the stream and creates chunks
// A full implementation would be more sophisticated
var chunks []*filer_pb.FileChunk
buffer := make([]byte, spec.BufferSize)
offset := int64(0)
for {
n, err := reader.Read(buffer)
if n > 0 {
// Create chunk for this data, setting SSE type and per-chunk metadata (including chunk-specific IVs for SSE-S3)
chunk, chunkErr := scm.createChunkFromData(buffer[:n], offset, dstPath, spec.EncryptionSpec)
if chunkErr != nil {
return nil, fmt.Errorf("create chunk from data: %w", chunkErr)
}
chunks = append(chunks, chunk)
offset += int64(n)
}
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read stream: %w", err)
}
}
return chunks, nil
}
// createChunkFromData creates a chunk from streaming data
func (scm *StreamingCopyManager) createChunkFromData(data []byte, offset int64, dstPath string, encSpec *EncryptionSpec) (*filer_pb.FileChunk, error) {
// Assign new volume
assignResult, err := scm.s3a.assignNewVolume(dstPath)
if err != nil {
return nil, fmt.Errorf("assign volume: %w", err)
}
// Create chunk
chunk := &filer_pb.FileChunk{
Offset: offset,
Size: uint64(len(data)),
}
// Set SSE type and metadata on chunk if destination is encrypted
// This is critical for GetObject to know to decrypt the data - fixes GitHub #7562
if encSpec != nil && encSpec.NeedsEncryption {
switch encSpec.DestinationType {
case EncryptionTypeSSEC:
chunk.SseType = filer_pb.SSEType_SSE_C
// SSE-C metadata is handled at object level, not per-chunk for streaming copy
case EncryptionTypeSSEKMS:
chunk.SseType = filer_pb.SSEType_SSE_KMS
// SSE-KMS metadata is handled at object level, not per-chunk for streaming copy
case EncryptionTypeSSES3:
chunk.SseType = filer_pb.SSEType_SSE_S3
// Create per-chunk SSE-S3 metadata with chunk-specific IV
if sseKey, ok := encSpec.DestinationKey.(*SSES3Key); ok {
// Calculate chunk-specific IV using base IV and chunk offset
baseIV := encSpec.DestinationIV
if len(baseIV) == 0 {
return nil, fmt.Errorf("SSE-S3 encryption requires DestinationIV to be set for chunk at offset %d", offset)
}
chunkIV, _ := calculateIVWithOffset(baseIV, offset)
// Create chunk key with the chunk-specific IV
chunkSSEKey := &SSES3Key{
Key: sseKey.Key,
KeyID: sseKey.KeyID,
Algorithm: sseKey.Algorithm,
IV: chunkIV,
}
chunkMetadata, serErr := SerializeSSES3Metadata(chunkSSEKey)
if serErr != nil {
return nil, fmt.Errorf("failed to serialize chunk SSE-S3 metadata: %w", serErr)
}
chunk.SseMetadata = chunkMetadata
}
}
}
// Set file ID
if err := scm.s3a.setChunkFileId(chunk, assignResult); err != nil {
return nil, err
}
// Upload data
if err := scm.s3a.uploadChunkData(data, assignResult, false); err != nil {
return nil, fmt.Errorf("upload chunk data: %w", err)
}
return chunk, nil
}
// createMultiChunkReader creates a reader that streams from multiple chunks
func (s3a *S3ApiServer) createMultiChunkReader(entry *filer_pb.Entry) (io.ReadCloser, error) {
// Create a multi-reader that combines all chunks
var readers []io.Reader
for _, chunk := range entry.GetChunks() {
chunkReader, err := s3a.createChunkReader(chunk)
if err != nil {
return nil, fmt.Errorf("create chunk reader: %w", err)
}
readers = append(readers, chunkReader)
}
multiReader := io.MultiReader(readers...)
return &multiReadCloser{reader: multiReader}, nil
}
// createChunkReader creates a reader for a single chunk
func (s3a *S3ApiServer) createChunkReader(chunk *filer_pb.FileChunk) (io.Reader, error) {
// Get chunk URL
srcUrl, err := s3a.lookupVolumeUrl(chunk.GetFileIdString())
if err != nil {
return nil, fmt.Errorf("lookup volume URL: %w", err)
}
// Create HTTP request for chunk data
req, err := http.NewRequest("GET", srcUrl, nil)
if err != nil {
return nil, fmt.Errorf("create HTTP request: %w", err)
}
// Execute request
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("execute HTTP request: %w", err)
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("HTTP request failed: %d", resp.StatusCode)
}
return resp.Body, nil
}
// multiReadCloser wraps a multi-reader with a close method
type multiReadCloser struct {
reader io.Reader
}
func (mrc *multiReadCloser) Read(p []byte) (int, error) {
return mrc.reader.Read(p)
}
func (mrc *multiReadCloser) Close() error {
return nil
}

7
weed/s3api/s3err/audit_fluent.go
View File

@@ -128,13 +128,6 @@ func getOperation(object string, r *http.Request) string {
return operation
}
func GetAccessHttpLog(r *http.Request, statusCode int, s3errCode ErrorCode) AccessLogHTTP {
return AccessLogHTTP{
RequestURI: r.RequestURI,
Referer: r.Header.Get("Referer"),
}
}
func GetAccessLog(r *http.Request, HTTPStatusCode int, s3errCode ErrorCode) *AccessLog {
bucket, key := s3_constants.GetBucketAndObject(r)
var errorCode string

127
weed/s3api/s3lifecycle/evaluator.go
View File

@@ -1,127 +0,0 @@
package s3lifecycle
import "time"
// Evaluate checks the given lifecycle rules against an object and returns
// the highest-priority action that applies. The evaluation follows S3's
// action priority:
// 1. ExpiredObjectDeleteMarker (delete marker is sole version)
// 2. NoncurrentVersionExpiration (non-current version age/count)
// 3. Current version Expiration (Days or Date)
//
// AbortIncompleteMultipartUpload is evaluated separately since it applies
// to uploads, not objects. Use EvaluateMPUAbort for that.
func Evaluate(rules []Rule, obj ObjectInfo, now time.Time) EvalResult {
// Phase 1: ExpiredObjectDeleteMarker
if obj.IsDeleteMarker && obj.IsLatest && obj.NumVersions == 1 {
for _, rule := range rules {
if rule.Status != "Enabled" {
continue
}
if !MatchesFilter(rule, obj) {
continue
}
if rule.ExpiredObjectDeleteMarker {
return EvalResult{Action: ActionExpireDeleteMarker, RuleID: rule.ID}
}
}
}
// Phase 2: NoncurrentVersionExpiration
if !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
for _, rule := range rules {
if ShouldExpireNoncurrentVersion(rule, obj, obj.NoncurrentIndex, now) {
return EvalResult{Action: ActionDeleteVersion, RuleID: rule.ID}
}
}
}
// Phase 3: Current version Expiration
if obj.IsLatest && !obj.IsDeleteMarker {
for _, rule := range rules {
if rule.Status != "Enabled" {
continue
}
if !MatchesFilter(rule, obj) {
continue
}
// Date-based expiration
if !rule.ExpirationDate.IsZero() && !now.Before(rule.ExpirationDate) {
return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID}
}
// Days-based expiration
if rule.ExpirationDays > 0 {
expiryTime := expectedExpiryTime(obj.ModTime, rule.ExpirationDays)
if !now.Before(expiryTime) {
return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID}
}
}
}
}
return EvalResult{Action: ActionNone}
}
// ShouldExpireNoncurrentVersion checks whether a non-current version should
// be expired considering both NoncurrentDays and NewerNoncurrentVersions.
// noncurrentIndex is the 0-based position among non-current versions sorted
// newest-first (0 = newest non-current version).
func ShouldExpireNoncurrentVersion(rule Rule, obj ObjectInfo, noncurrentIndex int, now time.Time) bool {
if rule.Status != "Enabled" {
return false
}
if rule.NoncurrentVersionExpirationDays <= 0 {
return false
}
if obj.IsLatest || obj.SuccessorModTime.IsZero() {
return false
}
if !MatchesFilter(rule, obj) {
return false
}
// Check age threshold.
expiryTime := expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpirationDays)
if now.Before(expiryTime) {
return false
}
// Check NewerNoncurrentVersions count threshold.
if rule.NewerNoncurrentVersions > 0 && noncurrentIndex < rule.NewerNoncurrentVersions {
return false
}
return true
}
// EvaluateMPUAbort finds the applicable AbortIncompleteMultipartUpload rule
// for a multipart upload with the given key prefix and creation time.
func EvaluateMPUAbort(rules []Rule, uploadKey string, createdAt time.Time, now time.Time) EvalResult {
for _, rule := range rules {
if rule.Status != "Enabled" {
continue
}
if rule.AbortMPUDaysAfterInitiation <= 0 {
continue
}
if !matchesPrefix(rule.Prefix, uploadKey) {
continue
}
cutoff := expectedExpiryTime(createdAt, rule.AbortMPUDaysAfterInitiation)
if !now.Before(cutoff) {
return EvalResult{Action: ActionAbortMultipartUpload, RuleID: rule.ID}
}
}
return EvalResult{Action: ActionNone}
}
// expectedExpiryTime computes the expiration time given a reference time and
// a number of days. Following S3 semantics, expiration happens at midnight UTC
// of the day after the specified number of days.
func expectedExpiryTime(refTime time.Time, days int) time.Time {
if days == 0 {
return refTime
}
t := refTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
return t.Truncate(24 * time.Hour)
}

495
weed/s3api/s3lifecycle/evaluator_test.go
View File

@@ -1,495 +0,0 @@
package s3lifecycle
import (
"testing"
"time"
)
var now = time.Date(2026, 3, 27, 12, 0, 0, 0, time.UTC)
func TestEvaluate_ExpirationDays(t *testing.T) {
rules := []Rule{{
ID: "expire-30d", Status: "Enabled",
ExpirationDays: 30,
}}
t.Run("object_older_than_days_is_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/file.txt", IsLatest: true,
ModTime: now.Add(-31 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
assertEqual(t, "expire-30d", result.RuleID)
})
t.Run("object_younger_than_days_is_not_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/file.txt", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("non_latest_version_not_affected_by_expiration_days", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/file.txt", IsLatest: false,
ModTime: now.Add(-60 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("delete_marker_not_affected_by_expiration_days", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/file.txt", IsLatest: true, IsDeleteMarker: true,
ModTime: now.Add(-60 * 24 * time.Hour), NumVersions: 3,
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_ExpirationDate(t *testing.T) {
expirationDate := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)
rules := []Rule{{
ID: "expire-date", Status: "Enabled",
ExpirationDate: expirationDate,
}}
t.Run("object_expired_after_date", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-60 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
})
t.Run("object_not_expired_before_date", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-1 * time.Hour),
}
beforeDate := time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)
result := Evaluate(rules, obj, beforeDate)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_ExpiredObjectDeleteMarker(t *testing.T) {
rules := []Rule{{
ID: "cleanup-markers", Status: "Enabled",
ExpiredObjectDeleteMarker: true,
}}
t.Run("sole_delete_marker_is_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
NumVersions: 1,
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionExpireDeleteMarker, result.Action)
})
t.Run("delete_marker_with_other_versions_not_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
NumVersions: 3,
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("non_latest_delete_marker_not_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false, IsDeleteMarker: true,
NumVersions: 1,
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("non_delete_marker_not_affected", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true, IsDeleteMarker: false,
NumVersions: 1,
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_NoncurrentVersionExpiration(t *testing.T) {
rules := []Rule{{
ID: "expire-noncurrent", Status: "Enabled",
NoncurrentVersionExpirationDays: 30,
}}
t.Run("old_noncurrent_version_is_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteVersion, result.Action)
})
t.Run("recent_noncurrent_version_is_not_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-10 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("latest_version_not_affected", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-60 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestShouldExpireNoncurrentVersion(t *testing.T) {
rule := Rule{
ID: "noncurrent-rule", Status: "Enabled",
NoncurrentVersionExpirationDays: 30,
NewerNoncurrentVersions: 2,
}
t.Run("old_version_beyond_count_is_expired", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
}
// noncurrentIndex=2 means this is the 3rd noncurrent version (0-indexed)
// With NewerNoncurrentVersions=2, indices 0 and 1 are kept.
if !ShouldExpireNoncurrentVersion(rule, obj, 2, now) {
t.Error("expected version at index 2 to be expired")
}
})
t.Run("old_version_within_count_is_kept", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
}
// noncurrentIndex=1 is within the keep threshold (NewerNoncurrentVersions=2).
if ShouldExpireNoncurrentVersion(rule, obj, 1, now) {
t.Error("expected version at index 1 to be kept")
}
})
t.Run("recent_version_beyond_count_is_kept", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-5 * 24 * time.Hour),
}
// Even at index 5 (beyond count), if too young, it's kept.
if ShouldExpireNoncurrentVersion(rule, obj, 5, now) {
t.Error("expected recent version to be kept regardless of index")
}
})
t.Run("disabled_rule_never_expires", func(t *testing.T) {
disabled := Rule{
ID: "disabled", Status: "Disabled",
NoncurrentVersionExpirationDays: 1,
}
obj := ObjectInfo{
Key: "file.txt", IsLatest: false,
SuccessorModTime: now.Add(-365 * 24 * time.Hour),
}
if ShouldExpireNoncurrentVersion(disabled, obj, 10, now) {
t.Error("disabled rule should never expire")
}
})
}
func TestEvaluate_PrefixFilter(t *testing.T) {
rules := []Rule{{
ID: "logs-only", Status: "Enabled",
Prefix: "logs/",
ExpirationDays: 7,
}}
t.Run("matching_prefix", func(t *testing.T) {
obj := ObjectInfo{
Key: "logs/app.log", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
})
t.Run("non_matching_prefix", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/file.txt", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_TagFilter(t *testing.T) {
rules := []Rule{{
ID: "temp-only", Status: "Enabled",
ExpirationDays: 1,
FilterTags: map[string]string{"env": "temp"},
}}
t.Run("matching_tags", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-5 * 24 * time.Hour),
Tags: map[string]string{"env": "temp", "project": "foo"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
})
t.Run("missing_tag", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-5 * 24 * time.Hour),
Tags: map[string]string{"project": "foo"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("wrong_tag_value", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-5 * 24 * time.Hour),
Tags: map[string]string{"env": "prod"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("nil_object_tags", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-5 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_SizeFilter(t *testing.T) {
rules := []Rule{{
ID: "large-files", Status: "Enabled",
ExpirationDays: 7,
FilterSizeGreaterThan: 1024 * 1024, // > 1 MB
FilterSizeLessThan: 100 * 1024 * 1024, // < 100 MB
}}
t.Run("matching_size", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.bin", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 10 * 1024 * 1024, // 10 MB
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
})
t.Run("too_small", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.bin", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 512, // 512 bytes
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("too_large", func(t *testing.T) {
obj := ObjectInfo{
Key: "file.bin", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 200 * 1024 * 1024, // 200 MB
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_CombinedFilters(t *testing.T) {
rules := []Rule{{
ID: "combined", Status: "Enabled",
Prefix: "logs/",
ExpirationDays: 7,
FilterTags: map[string]string{"env": "dev"},
FilterSizeGreaterThan: 100,
}}
t.Run("all_filters_match", func(t *testing.T) {
obj := ObjectInfo{
Key: "logs/app.log", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 1024,
Tags: map[string]string{"env": "dev"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
})
t.Run("prefix_doesnt_match", func(t *testing.T) {
obj := ObjectInfo{
Key: "data/app.log", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 1024,
Tags: map[string]string{"env": "dev"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("tag_doesnt_match", func(t *testing.T) {
obj := ObjectInfo{
Key: "logs/app.log", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 1024,
Tags: map[string]string{"env": "prod"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
t.Run("size_doesnt_match", func(t *testing.T) {
obj := ObjectInfo{
Key: "logs/app.log", IsLatest: true,
ModTime: now.Add(-10 * 24 * time.Hour),
Size: 50, // too small
Tags: map[string]string{"env": "dev"},
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
})
}
func TestEvaluate_DisabledRule(t *testing.T) {
rules := []Rule{{
ID: "disabled", Status: "Disabled",
ExpirationDays: 1,
}}
obj := ObjectInfo{
Key: "file.txt", IsLatest: true,
ModTime: now.Add(-365 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionNone, result.Action)
}
func TestEvaluate_MultipleRules_Priority(t *testing.T) {
t.Run("delete_marker_takes_priority_over_expiration", func(t *testing.T) {
rules := []Rule{
{ID: "expire", Status: "Enabled", ExpirationDays: 1},
{ID: "marker", Status: "Enabled", ExpiredObjectDeleteMarker: true},
}
obj := ObjectInfo{
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
NumVersions: 1, ModTime: now.Add(-10 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionExpireDeleteMarker, result.Action)
assertEqual(t, "marker", result.RuleID)
})
t.Run("first_matching_expiration_rule_wins", func(t *testing.T) {
rules := []Rule{
{ID: "rule1", Status: "Enabled", ExpirationDays: 30, Prefix: "logs/"},
{ID: "rule2", Status: "Enabled", ExpirationDays: 7},
}
obj := ObjectInfo{
Key: "logs/app.log", IsLatest: true,
ModTime: now.Add(-31 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
assertEqual(t, "rule1", result.RuleID)
})
}
func TestEvaluate_EmptyPrefix(t *testing.T) {
rules := []Rule{{
ID: "all", Status: "Enabled",
ExpirationDays: 30,
}}
obj := ObjectInfo{
Key: "any/path/file.txt", IsLatest: true,
ModTime: now.Add(-31 * 24 * time.Hour),
}
result := Evaluate(rules, obj, now)
assertAction(t, ActionDeleteObject, result.Action)
}
func TestEvaluateMPUAbort(t *testing.T) {
rules := []Rule{{
ID: "abort-mpu", Status: "Enabled",
AbortMPUDaysAfterInitiation: 7,
}}
t.Run("old_upload_is_aborted", func(t *testing.T) {
result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-10*24*time.Hour), now)
assertAction(t, ActionAbortMultipartUpload, result.Action)
})
t.Run("recent_upload_is_not_aborted", func(t *testing.T) {
result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-3*24*time.Hour), now)
assertAction(t, ActionNone, result.Action)
})
t.Run("prefix_scoped_abort", func(t *testing.T) {
prefixRules := []Rule{{
ID: "abort-logs", Status: "Enabled",
Prefix: "logs/",
AbortMPUDaysAfterInitiation: 1,
}}
result := EvaluateMPUAbort(prefixRules, "data/file.bin", now.Add(-5*24*time.Hour), now)
assertAction(t, ActionNone, result.Action)
})
}
func TestExpectedExpiryTime(t *testing.T) {
ref := time.Date(2026, 3, 1, 15, 30, 0, 0, time.UTC)
t.Run("30_days", func(t *testing.T) {
// S3 spec: expires at midnight UTC of day 32 (ref + 31 days, truncated).
expiry := expectedExpiryTime(ref, 30)
expected := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
if !expiry.Equal(expected) {
t.Errorf("expected %v, got %v", expected, expiry)
}
})
t.Run("zero_days_returns_ref", func(t *testing.T) {
expiry := expectedExpiryTime(ref, 0)
if !expiry.Equal(ref) {
t.Errorf("expected %v, got %v", ref, expiry)
}
})
}
func assertAction(t *testing.T, expected, actual Action) {
t.Helper()
if expected != actual {
t.Errorf("expected action %d, got %d", expected, actual)
}
}
func assertEqual(t *testing.T, expected, actual string) {
t.Helper()
if expected != actual {
t.Errorf("expected %q, got %q", expected, actual)
}
}

56
weed/s3api/s3lifecycle/filter.go
View File

@@ -1,56 +0,0 @@
package s3lifecycle
import "strings"
// MatchesFilter checks if an object matches the rule's filter criteria
// (prefix, tags, and size constraints).
func MatchesFilter(rule Rule, obj ObjectInfo) bool {
if !matchesPrefix(rule.Prefix, obj.Key) {
return false
}
if !matchesTags(rule.FilterTags, obj.Tags) {
return false
}
if !matchesSize(rule.FilterSizeGreaterThan, rule.FilterSizeLessThan, obj.Size) {
return false
}
return true
}
// matchesPrefix returns true if the object key starts with the given prefix.
// An empty prefix matches all keys.
func matchesPrefix(prefix, key string) bool {
if prefix == "" {
return true
}
return strings.HasPrefix(key, prefix)
}
// matchesTags returns true if all rule tags are present in the object's tags
// with matching values. An empty or nil rule tag set matches all objects.
func matchesTags(ruleTags, objTags map[string]string) bool {
if len(ruleTags) == 0 {
return true
}
if len(objTags) == 0 {
return false
}
for k, v := range ruleTags {
if objVal, ok := objTags[k]; !ok || objVal != v {
return false
}
}
return true
}
// matchesSize returns true if the object's size falls within the specified
// bounds. Zero values mean no constraint on that side.
func matchesSize(greaterThan, lessThan, objSize int64) bool {
if greaterThan > 0 && objSize <= greaterThan {
return false
}
if lessThan > 0 && objSize >= lessThan {
return false
}
return true
}

79
weed/s3api/s3lifecycle/filter_test.go
View File

@@ -1,79 +0,0 @@
package s3lifecycle
import "testing"
func TestMatchesPrefix(t *testing.T) {
tests := []struct {
name string
prefix string
key string
want bool
}{
{"empty_prefix_matches_all", "", "any/key.txt", true},
{"exact_prefix_match", "logs/", "logs/app.log", true},
{"prefix_mismatch", "logs/", "data/file.txt", false},
{"key_shorter_than_prefix", "very/long/prefix/", "short", false},
{"prefix_equals_key", "exact", "exact", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchesPrefix(tt.prefix, tt.key); got != tt.want {
t.Errorf("matchesPrefix(%q, %q) = %v, want %v", tt.prefix, tt.key, got, tt.want)
}
})
}
}
func TestMatchesTags(t *testing.T) {
tests := []struct {
name string
ruleTags map[string]string
objTags map[string]string
want bool
}{
{"nil_rule_tags_match_all", nil, map[string]string{"a": "1"}, true},
{"empty_rule_tags_match_all", map[string]string{}, map[string]string{"a": "1"}, true},
{"nil_obj_tags_no_match", map[string]string{"a": "1"}, nil, false},
{"single_tag_match", map[string]string{"env": "dev"}, map[string]string{"env": "dev", "foo": "bar"}, true},
{"single_tag_value_mismatch", map[string]string{"env": "dev"}, map[string]string{"env": "prod"}, false},
{"single_tag_key_missing", map[string]string{"env": "dev"}, map[string]string{"foo": "bar"}, false},
{"multi_tag_all_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev", "tier": "hot", "extra": "x"}, true},
{"multi_tag_partial_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev"}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchesTags(tt.ruleTags, tt.objTags); got != tt.want {
t.Errorf("matchesTags() = %v, want %v", got, tt.want)
}
})
}
}
func TestMatchesSize(t *testing.T) {
tests := []struct {
name string
greaterThan int64
lessThan int64
objSize int64
want bool
}{
{"no_constraints", 0, 0, 1000, true},
{"only_greater_than_pass", 100, 0, 200, true},
{"only_greater_than_fail", 100, 0, 50, false},
{"only_greater_than_equal_fail", 100, 0, 100, false},
{"only_less_than_pass", 0, 1000, 500, true},
{"only_less_than_fail", 0, 1000, 2000, false},
{"only_less_than_equal_fail", 0, 1000, 1000, false},
{"both_constraints_pass", 100, 1000, 500, true},
{"both_constraints_too_small", 100, 1000, 50, false},
{"both_constraints_too_large", 100, 1000, 2000, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchesSize(tt.greaterThan, tt.lessThan, tt.objSize); got != tt.want {
t.Errorf("matchesSize(%d, %d, %d) = %v, want %v",
tt.greaterThan, tt.lessThan, tt.objSize, got, tt.want)
}
})
}
}

31
weed/s3api/s3lifecycle/tags.go
View File

@@ -1,34 +1,3 @@
package s3lifecycle
import "strings"
const tagPrefix = "X-Amz-Tagging-"
// ExtractTags extracts S3 object tags from a filer entry's Extended metadata.
// Tags are stored with the key prefix "X-Amz-Tagging-" followed by the tag key.
func ExtractTags(extended map[string][]byte) map[string]string {
if len(extended) == 0 {
return nil
}
var tags map[string]string
for k, v := range extended {
if strings.HasPrefix(k, tagPrefix) {
if tags == nil {
tags = make(map[string]string)
}
tags[k[len(tagPrefix):]] = string(v)
}
}
return tags
}
// HasTagRules returns true if any enabled rule in the set uses tag-based filtering.
// This is used as an optimization to skip tag extraction when no rules need it.
func HasTagRules(rules []Rule) bool {
for _, r := range rules {
if r.Status == "Enabled" && len(r.FilterTags) > 0 {
return true
}
}
return false
}

89
weed/s3api/s3lifecycle/tags_test.go
View File

@@ -1,89 +0,0 @@
package s3lifecycle
import "testing"
func TestExtractTags(t *testing.T) {
t.Run("extracts_tags_with_prefix", func(t *testing.T) {
extended := map[string][]byte{
"X-Amz-Tagging-env": []byte("prod"),
"X-Amz-Tagging-project": []byte("foo"),
"Content-Type": []byte("text/plain"),
"X-Amz-Meta-Custom": []byte("value"),
}
tags := ExtractTags(extended)
if len(tags) != 2 {
t.Fatalf("expected 2 tags, got %d", len(tags))
}
if tags["env"] != "prod" {
t.Errorf("expected env=prod, got %q", tags["env"])
}
if tags["project"] != "foo" {
t.Errorf("expected project=foo, got %q", tags["project"])
}
})
t.Run("nil_extended_returns_nil", func(t *testing.T) {
tags := ExtractTags(nil)
if tags != nil {
t.Errorf("expected nil, got %v", tags)
}
})
t.Run("no_tags_returns_nil", func(t *testing.T) {
extended := map[string][]byte{
"Content-Type": []byte("text/plain"),
}
tags := ExtractTags(extended)
if tags != nil {
t.Errorf("expected nil, got %v", tags)
}
})
t.Run("empty_tag_value", func(t *testing.T) {
extended := map[string][]byte{
"X-Amz-Tagging-empty": []byte(""),
}
tags := ExtractTags(extended)
if len(tags) != 1 {
t.Fatalf("expected 1 tag, got %d", len(tags))
}
if tags["empty"] != "" {
t.Errorf("expected empty value, got %q", tags["empty"])
}
})
}
func TestHasTagRules(t *testing.T) {
t.Run("has_tag_rules", func(t *testing.T) {
rules := []Rule{
{Status: "Enabled", FilterTags: map[string]string{"env": "dev"}},
}
if !HasTagRules(rules) {
t.Error("expected true")
}
})
t.Run("no_tag_rules", func(t *testing.T) {
rules := []Rule{
{Status: "Enabled", ExpirationDays: 30},
}
if HasTagRules(rules) {
t.Error("expected false")
}
})
t.Run("disabled_tag_rule", func(t *testing.T) {
rules := []Rule{
{Status: "Disabled", FilterTags: map[string]string{"env": "dev"}},
}
if HasTagRules(rules) {
t.Error("expected false for disabled rule")
}
})
t.Run("empty_rules", func(t *testing.T) {
if HasTagRules(nil) {
t.Error("expected false for nil rules")
}
})
}

93
weed/s3api/s3lifecycle/version_time.go
View File

@@ -1,99 +1,6 @@
package s3lifecycle
import (
"math"
"strconv"
"time"
)
// versionIdFormatThreshold distinguishes old vs new format version IDs.
// New format (inverted timestamps) produces values above this threshold;
// old format (raw timestamps) produces values below it.
const versionIdFormatThreshold = 0x4000000000000000
// GetVersionTimestamp extracts the actual timestamp from a SeaweedFS version ID,
// handling both old (raw nanosecond) and new (inverted nanosecond) formats.
// Returns zero time if the version ID is invalid or "null".
func GetVersionTimestamp(versionId string) time.Time {
ns := getVersionTimestampNanos(versionId)
if ns == 0 {
return time.Time{}
}
return time.Unix(0, ns)
}
// getVersionTimestampNanos extracts the raw nanosecond timestamp from a version ID.
func getVersionTimestampNanos(versionId string) int64 {
if len(versionId) < 16 || versionId == "null" {
return 0
}
timestampPart, err := strconv.ParseUint(versionId[:16], 16, 64)
if err != nil {
return 0
}
if timestampPart > math.MaxInt64 {
return 0
}
if timestampPart > versionIdFormatThreshold {
// New format: inverted timestamp, convert back.
return int64(math.MaxInt64 - timestampPart)
}
return int64(timestampPart)
}
// isNewFormatVersionId returns true if the version ID uses inverted timestamps.
func isNewFormatVersionId(versionId string) bool {
if len(versionId) < 16 || versionId == "null" {
return false
}
timestampPart, err := strconv.ParseUint(versionId[:16], 16, 64)
if err != nil {
return false
}
return timestampPart > versionIdFormatThreshold && timestampPart <= math.MaxInt64
}
// CompareVersionIds compares two version IDs for sorting (newest first).
// Returns negative if a is newer, positive if b is newer, 0 if equal.
// Handles both old and new format version IDs and uses full lexicographic
// comparison (not just timestamps) to break ties from the random suffix.
func CompareVersionIds(a, b string) int {
if a == b {
return 0
}
if a == "null" {
return 1
}
if b == "null" {
return -1
}
aIsNew := isNewFormatVersionId(a)
bIsNew := isNewFormatVersionId(b)
if aIsNew == bIsNew {
if aIsNew {
// New format: smaller hex = newer (inverted timestamps).
if a < b {
return -1
}
return 1
}
// Old format: smaller hex = older.
if a < b {
return 1
}
return -1
}
// Mixed formats: compare by actual timestamp.
aTime := getVersionTimestampNanos(a)
bTime := getVersionTimestampNanos(b)
if aTime > bTime {
return -1
}
if aTime < bTime {
return 1
}
return 0
}

74
weed/s3api/s3lifecycle/version_time_test.go
View File

@@ -1,74 +0,0 @@
package s3lifecycle
import (
"fmt"
"math"
"testing"
"time"
)
func TestGetVersionTimestamp(t *testing.T) {
t.Run("new_format_inverted_timestamp", func(t *testing.T) {
// Simulate a new-format version ID (inverted timestamp above threshold).
now := time.Now()
inverted := math.MaxInt64 - now.UnixNano()
versionId := fmt.Sprintf("%016x", inverted) + "0000000000000000"
got := GetVersionTimestamp(versionId)
// Should recover the original timestamp within 1 second.
diff := got.Sub(now)
if diff < -time.Second || diff > time.Second {
t.Errorf("timestamp diff too large: %v (got %v, want ~%v)", diff, got, now)
}
})
t.Run("old_format_raw_timestamp", func(t *testing.T) {
// Simulate an old-format version ID (raw nanosecond timestamp below threshold).
// Use a timestamp from 2023 which would be below threshold.
ts := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)
versionId := fmt.Sprintf("%016x", ts.UnixNano()) + "abcdef0123456789"
got := GetVersionTimestamp(versionId)
if !got.Equal(ts) {
t.Errorf("expected %v, got %v", ts, got)
}
})
t.Run("null_version_id", func(t *testing.T) {
got := GetVersionTimestamp("null")
if !got.IsZero() {
t.Errorf("expected zero time for null version, got %v", got)
}
})
t.Run("empty_version_id", func(t *testing.T) {
got := GetVersionTimestamp("")
if !got.IsZero() {
t.Errorf("expected zero time for empty version, got %v", got)
}
})
t.Run("short_version_id", func(t *testing.T) {
got := GetVersionTimestamp("abc")
if !got.IsZero() {
t.Errorf("expected zero time for short version, got %v", got)
}
})
t.Run("high_bit_overflow_returns_zero", func(t *testing.T) {
// Version ID with first 16 hex chars > math.MaxInt64 should return zero,
// not a wrapped negative timestamp.
versionId := "80000000000000000000000000000000"
got := GetVersionTimestamp(versionId)
if !got.IsZero() {
t.Errorf("expected zero time for overflow version ID, got %v", got)
}
})
t.Run("invalid_hex", func(t *testing.T) {
got := GetVersionTimestamp("zzzzzzzzzzzzzzzz0000000000000000")
if !got.IsZero() {
t.Errorf("expected zero time for invalid hex, got %v", got)
}
})
}

40
weed/s3api/s3tables/filer_ops.go
View File

@@ -50,46 +50,6 @@ func (h *S3TablesHandler) ensureDirectory(ctx context.Context, client filer_pb.S
return err
}
// upsertFile creates or updates a small file with the given content
func (h *S3TablesHandler) upsertFile(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, data []byte) error {
dir, name := splitPath(path)
now := time.Now().Unix()
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
if !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: &filer_pb.Entry{
Name: name,
Content: data,
Attributes: &filer_pb.FuseAttributes{
Mtime: now,
Crtime: now,
FileMode: uint32(0644),
FileSize: uint64(len(data)),
},
},
})
}
entry := resp.Entry
if entry.Attributes == nil {
entry.Attributes = &filer_pb.FuseAttributes{}
}
entry.Attributes.Mtime = now
entry.Attributes.FileSize = uint64(len(data))
entry.Content = data
return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
Directory: dir,
Entry: entry,
})
}
// deleteEntryIfExists removes an entry if it exists, ignoring missing errors
func (h *S3TablesHandler) deleteEntryIfExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)

132
weed/s3api/s3tables/iceberg_layout.go
View File

@@ -1,14 +1,9 @@
package s3tables
import (
"context"
"encoding/json"
"errors"
pathpkg "path"
"regexp"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// Iceberg file layout validation
@@ -307,130 +302,3 @@ func (v *TableBucketFileValidator) ValidateTableBucketUpload(fullPath string) er
return v.layoutValidator.ValidateFilePath(tableRelativePath)
}
// IsTableBucketPath checks if a path is under the table buckets directory
func IsTableBucketPath(fullPath string) bool {
return strings.HasPrefix(fullPath, TablesPath+"/")
}
// GetTableInfoFromPath extracts bucket, namespace, and table names from a table bucket path
// Returns empty strings if the path doesn't contain enough components
func GetTableInfoFromPath(fullPath string) (bucket, namespace, table string) {
if !strings.HasPrefix(fullPath, TablesPath+"/") {
return "", "", ""
}
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
parts := strings.SplitN(relativePath, "/", 4)
if len(parts) >= 1 {
bucket = parts[0]
}
if len(parts) >= 2 {
namespace = parts[1]
}
if len(parts) >= 3 {
table = parts[2]
}
return
}
// ValidateTableBucketUploadWithClient validates upload and checks that the table exists and is ICEBERG format
func (v *TableBucketFileValidator) ValidateTableBucketUploadWithClient(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
fullPath string,
) error {
// If not a table bucket path, nothing more to check
if !IsTableBucketPath(fullPath) {
return nil
}
// Get table info and verify it exists
bucket, namespace, table := GetTableInfoFromPath(fullPath)
if bucket == "" || namespace == "" || table == "" {
return nil // Not deep enough to need validation
}
if strings.HasPrefix(bucket, ".") {
return nil
}
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: TablesPath,
Name: bucket,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to verify table bucket: " + err.Error(),
}
}
if resp == nil || !IsTableBucketEntry(resp.Entry) {
return nil
}
// Now check basic layout once we know this is a table bucket path.
if err := v.ValidateTableBucketUpload(fullPath); err != nil {
return err
}
// Verify the table exists and has ICEBERG format by checking its metadata
tablePath := GetTablePath(bucket, namespace, table)
dir, name := splitPath(tablePath)
resp, err = filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
// Distinguish between "not found" and other errors
if errors.Is(err, filer_pb.ErrNotFound) {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table does not exist",
}
}
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to verify table existence: " + err.Error(),
}
}
// Check if table has metadata indicating ICEBERG format
if resp.Entry == nil || resp.Entry.Extended == nil {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not a valid ICEBERG table (missing metadata)",
}
}
metadataBytes, ok := resp.Entry.Extended[ExtendedKeyMetadata]
if !ok {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not in ICEBERG format (missing format metadata)",
}
}
var metadata tableMetadataInternal
if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to parse table metadata: " + err.Error(),
}
}
const TableFormatIceberg = "ICEBERG"
if metadata.Format != TableFormatIceberg {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not in " + TableFormatIceberg + " format",
}
}
return nil
}

186
weed/s3api/s3tables/iceberg_layout_test.go
View File

@@ -1,186 +0,0 @@
package s3tables
import (
"testing"
)
func TestIcebergLayoutValidator_ValidateFilePath(t *testing.T) {
v := NewIcebergLayoutValidator()
tests := []struct {
name string
path string
wantErr bool
}{
// Valid metadata files
{"valid metadata v1", "metadata/v1.metadata.json", false},
{"valid metadata v123", "metadata/v123.metadata.json", false},
{"valid snapshot manifest", "metadata/snap-123-1-abc12345-1234-5678-9abc-def012345678.avro", false},
{"valid manifest file", "metadata/abc12345-1234-5678-9abc-def012345678-m0.avro", false},
{"valid general manifest", "metadata/abc12345-1234-5678-9abc-def012345678.avro", false},
{"valid version hint", "metadata/version-hint.text", false},
{"valid uuid metadata", "metadata/abc12345-1234-5678-9abc-def012345678.metadata.json", false},
{"valid trino stats", "metadata/20260208_212535_00007_bn4hb-d3599c32-1709-4b94-b6b2-1957b6d6db04.stats", false},
// Valid data files
{"valid parquet file", "data/file.parquet", false},
{"valid orc file", "data/file.orc", false},
{"valid avro data file", "data/file.avro", false},
{"valid parquet with path", "data/00000-0-abc12345.parquet", false},
// Valid partitioned data
{"valid partitioned parquet", "data/year=2024/file.parquet", false},
{"valid multi-partition", "data/year=2024/month=01/file.parquet", false},
{"valid bucket subdirectory", "data/bucket0/file.parquet", false},
// Directories only
{"metadata directory bare", "metadata", true},
{"data directory bare", "data", true},
{"metadata directory with slash", "metadata/", false},
{"data directory with slash", "data/", false},
// Invalid paths
{"empty path", "", true},
{"invalid top dir", "invalid/file.parquet", true},
{"root file", "file.parquet", true},
{"invalid metadata file", "metadata/random.txt", true},
{"nested metadata directory", "metadata/nested/v1.metadata.json", true},
{"nested metadata directory no file", "metadata/nested/", true},
{"metadata subdir no slash", "metadata/nested", true},
{"invalid data file", "data/file.csv", true},
{"invalid data file json", "data/file.json", true},
// Partition/subdirectory without trailing slashes
{"partition directory no slash", "data/year=2024", false},
{"data subdirectory no slash", "data/my_subdir", false},
{"multi-level partition", "data/event_date=2025-01-01/hour=00/file.parquet", false},
{"multi-level partition directory", "data/event_date=2025-01-01/hour=00/", false},
{"multi-level partition directory no slash", "data/event_date=2025-01-01/hour=00", false},
// Double slashes
{"data double slash", "data//file.parquet", true},
{"data redundant slash", "data/year=2024//file.parquet", true},
{"metadata redundant slash", "metadata//v1.metadata.json", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := v.ValidateFilePath(tt.path)
if (err != nil) != tt.wantErr {
t.Errorf("ValidateFilePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
}
})
}
}
func TestIcebergLayoutValidator_PartitionPaths(t *testing.T) {
v := NewIcebergLayoutValidator()
validPaths := []string{
"data/year=2024/file.parquet",
"data/date=2024-01-15/file.parquet",
"data/category=electronics/file.parquet",
"data/user_id=12345/file.parquet",
"data/region=us-east-1/file.parquet",
"data/year=2024/month=01/day=15/file.parquet",
}
for _, path := range validPaths {
if err := v.ValidateFilePath(path); err != nil {
t.Errorf("ValidateFilePath(%q) should be valid, got error: %v", path, err)
}
}
}
func TestTableBucketFileValidator_ValidateTableBucketUpload(t *testing.T) {
v := NewTableBucketFileValidator()
tests := []struct {
name string
path string
wantErr bool
}{
// Non-table bucket paths should pass (no validation)
{"regular bucket path", "/buckets/mybucket/file.txt", false},
{"filer path", "/home/user/file.txt", false},
// Table bucket structure paths (creating directories)
{"table bucket root", "/buckets/mybucket", false},
{"namespace dir", "/buckets/mybucket/myns", false},
{"table dir", "/buckets/mybucket/myns/mytable", false},
{"table dir trailing slash", "/buckets/mybucket/myns/mytable/", false},
// Valid table bucket file uploads
{"valid parquet upload", "/buckets/mybucket/myns/mytable/data/file.parquet", false},
{"valid metadata upload", "/buckets/mybucket/myns/mytable/metadata/v1.metadata.json", false},
{"valid trino stats upload", "/buckets/mybucket/myns/mytable/metadata/20260208_212535_00007_bn4hb-d3599c32-1709-4b94-b6b2-1957b6d6db04.stats", false},
{"valid partitioned data", "/buckets/mybucket/myns/mytable/data/year=2024/file.parquet", false},
// Invalid table bucket file uploads
{"invalid file type", "/buckets/mybucket/myns/mytable/data/file.csv", true},
{"invalid top-level dir", "/buckets/mybucket/myns/mytable/invalid/file.parquet", true},
{"root file in table", "/buckets/mybucket/myns/mytable/file.parquet", true},
// Empty segment cases
{"empty bucket", "/buckets//myns/mytable/data/file.parquet", true},
{"empty namespace", "/buckets/mybucket//mytable/data/file.parquet", true},
{"empty table", "/buckets/mybucket/myns//data/file.parquet", true},
{"empty bucket dir", "/buckets//", true},
{"empty namespace dir", "/buckets/mybucket//", true},
{"table double slash bypass", "/buckets/mybucket/myns/mytable//data/file.parquet", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := v.ValidateTableBucketUpload(tt.path)
if (err != nil) != tt.wantErr {
t.Errorf("ValidateTableBucketUpload(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
}
})
}
}
func TestIsTableBucketPath(t *testing.T) {
tests := []struct {
path string
want bool
}{
{"/buckets/mybucket", true},
{"/buckets/mybucket/ns/table/data/file.parquet", true},
{"/home/user/file.txt", false},
{"buckets/mybucket", false}, // missing leading slash
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
if got := IsTableBucketPath(tt.path); got != tt.want {
t.Errorf("IsTableBucketPath(%q) = %v, want %v", tt.path, got, tt.want)
}
})
}
}
func TestGetTableInfoFromPath(t *testing.T) {
tests := []struct {
path string
wantBucket string
wantNamespace string
wantTable string
}{
{"/buckets/mybucket/myns/mytable/data/file.parquet", "mybucket", "myns", "mytable"},
{"/buckets/mybucket/myns/mytable", "mybucket", "myns", "mytable"},
{"/buckets/mybucket/myns", "mybucket", "myns", ""},
{"/buckets/mybucket", "mybucket", "", ""},
{"/home/user/file.txt", "", "", ""},
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
bucket, namespace, table := GetTableInfoFromPath(tt.path)
if bucket != tt.wantBucket || namespace != tt.wantNamespace || table != tt.wantTable {
t.Errorf("GetTableInfoFromPath(%q) = (%q, %q, %q), want (%q, %q, %q)",
tt.path, bucket, namespace, table, tt.wantBucket, tt.wantNamespace, tt.wantTable)
}
})
}
}

118
weed/s3api/s3tables/permissions.go
View File

@@ -90,17 +90,6 @@ type PolicyContext struct {
DefaultAllow bool
}
// CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
func CheckPermissionWithResource(operation, principal, owner, resourcePolicy, resourceARN string) bool {
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN, nil)
}
// CheckPermission checks if a principal has permission to perform an operation
// (without resource-specific validation - for backward compatibility)
func CheckPermission(operation, principal, owner, resourcePolicy string) bool {
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, "", nil)
}
// CheckPermissionWithContext checks permission with optional resource and condition context.
func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
// Deny access if identities are empty
@@ -415,113 +404,6 @@ func matchesResourcePattern(pattern, resourceARN string) bool {
return wildcard.MatchesWildcard(pattern, resourceARN)
}
// Helper functions for specific permissions
// CanCreateTableBucket checks if principal can create table buckets
func CanCreateTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateTableBucket", principal, owner, resourcePolicy)
}
// CanGetTableBucket checks if principal can get table bucket details
func CanGetTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTableBucket", principal, owner, resourcePolicy)
}
// CanListTableBuckets checks if principal can list table buckets
func CanListTableBuckets(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListTableBuckets", principal, owner, resourcePolicy)
}
// CanDeleteTableBucket checks if principal can delete table buckets
func CanDeleteTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTableBucket", principal, owner, resourcePolicy)
}
// CanPutTableBucketPolicy checks if principal can put table bucket policies
func CanPutTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("PutTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanGetTableBucketPolicy checks if principal can get table bucket policies
func CanGetTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanDeleteTableBucketPolicy checks if principal can delete table bucket policies
func CanDeleteTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanCreateNamespace checks if principal can create namespaces
func CanCreateNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateNamespace", principal, owner, resourcePolicy)
}
// CanGetNamespace checks if principal can get namespace details
func CanGetNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetNamespace", principal, owner, resourcePolicy)
}
// CanListNamespaces checks if principal can list namespaces
func CanListNamespaces(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListNamespaces", principal, owner, resourcePolicy)
}
// CanDeleteNamespace checks if principal can delete namespaces
func CanDeleteNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteNamespace", principal, owner, resourcePolicy)
}
// CanCreateTable checks if principal can create tables
func CanCreateTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateTable", principal, owner, resourcePolicy)
}
// CanGetTable checks if principal can get table details
func CanGetTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTable", principal, owner, resourcePolicy)
}
// CanListTables checks if principal can list tables
func CanListTables(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListTables", principal, owner, resourcePolicy)
}
// CanDeleteTable checks if principal can delete tables
func CanDeleteTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTable", principal, owner, resourcePolicy)
}
// CanPutTablePolicy checks if principal can put table policies
func CanPutTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("PutTablePolicy", principal, owner, resourcePolicy)
}
// CanGetTablePolicy checks if principal can get table policies
func CanGetTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTablePolicy", principal, owner, resourcePolicy)
}
// CanDeleteTablePolicy checks if principal can delete table policies
func CanDeleteTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTablePolicy", principal, owner, resourcePolicy)
}
// CanTagResource checks if principal can tag a resource
func CanTagResource(principal, owner, resourcePolicy string) bool {
return CheckPermission("TagResource", principal, owner, resourcePolicy)
}
// CanUntagResource checks if principal can untag a resource
func CanUntagResource(principal, owner, resourcePolicy string) bool {
return CheckPermission("UntagResource", principal, owner, resourcePolicy)
}
// CanManageTags checks if principal can manage tags (tag or untag)
func CanManageTags(principal, owner, resourcePolicy string) bool {
return CanTagResource(principal, owner, resourcePolicy) || CanUntagResource(principal, owner, resourcePolicy)
}
// AuthError represents an authorization error
type AuthError struct {
Operation string

10
weed/s3api/s3tables/utils.go
View File

@@ -200,11 +200,6 @@ func validateBucketName(name string) error {
return nil
}
// ValidateBucketName validates bucket name and returns an error if invalid.
func ValidateBucketName(name string) error {
return validateBucketName(name)
}
// BuildBucketARN builds a bucket ARN with the provided region and account ID.
// If region is empty, the ARN will omit the region field.
func BuildBucketARN(region, accountID, bucketName string) (string, error) {
@@ -367,11 +362,6 @@ func validateNamespace(namespace []string) (string, error) {
return flattenNamespace(parts), nil
}
// ValidateNamespace is a wrapper to validate namespace for other packages.
func ValidateNamespace(namespace []string) (string, error) {
return validateNamespace(namespace)
}
// ParseNamespace parses a namespace string into namespace parts.
func ParseNamespace(namespace string) ([]string, error) {
return normalizeNamespace([]string{namespace})