9984ce7dcb
* fix(s3): omit NotResource:null from bucket policy JSON response (#8657) Change NotResource from value type to pointer (*StringOrStringSlice) so that omitempty properly omits it when unset, matching the existing Principal field pattern. This prevents IaC tools (Terraform, Ansible) from detecting false configuration drift. Add bucket policy round-trip idempotency integration tests. * simplify JSON comparison in bucket policy idempotency test Use require.JSONEq directly on the raw JSON strings instead of round-tripping through unmarshal/marshal, since JSONEq already handles normalization internally. * fix bucket policy test cases that locked out the admin user The Deny+NotResource test cases used Action:"s3:*" which denied the admin's own GetBucketPolicy call. Scope deny to s3:GetObject only, and add an Allow+NotResource variant instead. * fix(s3): also make Resource a pointer to fix empty string in JSON Apply the same omitempty pointer fix to the Resource field, which was emitting "Resource":"" when only NotResource was set. Add NewStringOrStringSlicePtr helper, make Strings() nil-safe, and handle *StringOrStringSlice in normalizeToStringSliceWithError. * improve bucket policy integration tests per review feedback - Replace time.Sleep with waitForClusterReady using ListBuckets - Use structural hasKey check instead of brittle substring NotContains - Assert specific NoSuchBucketPolicy error code after delete - Handle single-statement policies in hasKey helper
321 lines
8.9 KiB
Go
321 lines
8.9 KiB
Go
package policy
|
|
|
|
// Tests for S3 bucket policy JSON round-trip idempotency.
|
|
// These validate behavior that IaC tools (Terraform, Ansible) depend on:
|
|
// PUT a policy, GET it back, and verify the JSON matches exactly.
|
|
// See https://github.com/seaweedfs/seaweedfs/issues/8657
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
|
"github.com/aws/aws-sdk-go/aws/session"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func waitForClusterReady(t *testing.T, s3Client *s3.S3) {
|
|
t.Helper()
|
|
// ListBuckets is a lightweight call that confirms the S3 API is serving.
|
|
_, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
|
|
require.NoError(t, err, "S3 endpoint not ready")
|
|
}
|
|
|
|
func newS3ClientForCluster(t *testing.T, cluster *TestCluster) *s3.S3 {
|
|
t.Helper()
|
|
sess, err := session.NewSession(&aws.Config{
|
|
Region: aws.String("us-east-1"),
|
|
Endpoint: aws.String(cluster.s3Endpoint),
|
|
DisableSSL: aws.Bool(true),
|
|
S3ForcePathStyle: aws.Bool(true),
|
|
Credentials: credentials.NewStaticCredentials("admin", "admin", ""),
|
|
})
|
|
require.NoError(t, err)
|
|
return s3.New(sess)
|
|
}
|
|
|
|
// TestBucketPolicyRoundTrip verifies that GetBucketPolicy returns exactly what
|
|
// was submitted via PutBucketPolicy, without adding spurious fields like
|
|
// "NotResource": null. This is the core issue from #8657.
|
|
func TestBucketPolicyRoundTrip(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
cluster, err := startMiniCluster(t)
|
|
require.NoError(t, err)
|
|
defer cluster.Stop()
|
|
|
|
s3Client := newS3ClientForCluster(t, cluster)
|
|
waitForClusterReady(t, s3Client)
|
|
bucket := uniqueName("policy-rt")
|
|
|
|
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
policy map[string]interface{}
|
|
}{
|
|
{
|
|
name: "simple allow without NotResource",
|
|
policy: map[string]interface{}{
|
|
"Version": "2012-10-17",
|
|
"Statement": []interface{}{
|
|
map[string]interface{}{
|
|
"Sid": "AllowPublicRead",
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "multiple actions without NotResource",
|
|
policy: map[string]interface{}{
|
|
"Version": "2012-10-17",
|
|
"Statement": []interface{}{
|
|
map[string]interface{}{
|
|
"Sid": "ReadWrite",
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": []interface{}{"s3:GetObject", "s3:PutObject"},
|
|
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "allow with NotResource",
|
|
policy: map[string]interface{}{
|
|
"Version": "2012-10-17",
|
|
"Statement": []interface{}{
|
|
map[string]interface{}{
|
|
"Sid": "AllowOutsidePublic",
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"NotResource": "arn:aws:s3:::" + bucket + "/private/*",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "multiple statements with NotResource",
|
|
policy: map[string]interface{}{
|
|
"Version": "2012-10-17",
|
|
"Statement": []interface{}{
|
|
map[string]interface{}{
|
|
"Sid": "AllowRead",
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
|
},
|
|
map[string]interface{}{
|
|
"Sid": "DenyPrivateObjects",
|
|
"Effect": "Deny",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"NotResource": "arn:aws:s3:::" + bucket + "/public/*",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
policyJSON, err := json.Marshal(tc.policy)
|
|
require.NoError(t, err)
|
|
|
|
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
Policy: aws.String(string(policyJSON)),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
getOut, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, getOut.Policy)
|
|
|
|
// The returned policy must not contain fields that were not submitted.
|
|
// This is the exact issue from #8657: NotResource:null was being added.
|
|
returnedJSON := *getOut.Policy
|
|
var returnedPolicy map[string]interface{}
|
|
require.NoError(t, json.Unmarshal([]byte(returnedJSON), &returnedPolicy))
|
|
if !hasKey(tc.policy, "NotResource") {
|
|
require.False(t, hasKey(returnedPolicy, "NotResource"),
|
|
"returned policy must not contain NotResource when it was not submitted")
|
|
}
|
|
|
|
// Semantic comparison of all submitted fields
|
|
require.JSONEq(t, string(policyJSON), returnedJSON,
|
|
"GET should return semantically identical policy to what was PUT")
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestBucketPolicyIdempotentPut verifies that putting the same policy twice
|
|
// does not change the returned value — the behavior IaC tools rely on.
|
|
func TestBucketPolicyIdempotentPut(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
cluster, err := startMiniCluster(t)
|
|
require.NoError(t, err)
|
|
defer cluster.Stop()
|
|
|
|
s3Client := newS3ClientForCluster(t, cluster)
|
|
waitForClusterReady(t, s3Client)
|
|
bucket := uniqueName("policy-idem")
|
|
|
|
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
|
|
policyJSON := `{
|
|
"Version": "2012-10-17",
|
|
"Statement": [{
|
|
"Sid": "AllowRead",
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"Resource": "arn:aws:s3:::` + bucket + `/*"
|
|
}]
|
|
}`
|
|
|
|
// PUT, then GET, then PUT the returned value, then GET again.
|
|
// Both GETs should return the same result.
|
|
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
Policy: aws.String(policyJSON),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
getOut1, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Re-PUT the policy that was returned by GET (what Terraform does on update)
|
|
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
Policy: getOut1.Policy,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
getOut2, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
require.JSONEq(t, *getOut1.Policy, *getOut2.Policy,
|
|
"re-PUTting the GET result must produce identical output (idempotency)")
|
|
}
|
|
|
|
// TestBucketPolicyDeleteAndRecreate verifies clean lifecycle.
|
|
func TestBucketPolicyDeleteAndRecreate(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
cluster, err := startMiniCluster(t)
|
|
require.NoError(t, err)
|
|
defer cluster.Stop()
|
|
|
|
s3Client := newS3ClientForCluster(t, cluster)
|
|
waitForClusterReady(t, s3Client)
|
|
bucket := uniqueName("policy-del")
|
|
|
|
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
|
|
policyJSON := `{
|
|
"Version": "2012-10-17",
|
|
"Statement": [{
|
|
"Effect": "Allow",
|
|
"Principal": "*",
|
|
"Action": "s3:GetObject",
|
|
"Resource": "arn:aws:s3:::` + bucket + `/*"
|
|
}]
|
|
}`
|
|
|
|
// PUT
|
|
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
Policy: aws.String(policyJSON),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// DELETE
|
|
_, err = s3Client.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// GET should fail with NoSuchBucketPolicy
|
|
_, err = s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.Error(t, err)
|
|
awsErr, ok := err.(awserr.Error)
|
|
require.True(t, ok, "expected AWS error, got %T: %v", err, err)
|
|
require.Equal(t, "NoSuchBucketPolicy", awsErr.Code())
|
|
|
|
// Re-PUT same policy
|
|
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
Policy: aws.String(policyJSON),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// GET should succeed and be clean
|
|
getOut, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
|
Bucket: aws.String(bucket),
|
|
})
|
|
require.NoError(t, err)
|
|
var recreatedPolicy map[string]interface{}
|
|
require.NoError(t, json.Unmarshal([]byte(*getOut.Policy), &recreatedPolicy))
|
|
require.False(t, hasKey(recreatedPolicy, "NotResource"),
|
|
"recreated policy must not contain spurious NotResource")
|
|
}
|
|
|
|
// hasKey checks whether any Statement in the policy map contains the given key.
|
|
// Handles both single-statement objects and arrays of statements.
|
|
func hasKey(policy map[string]interface{}, key string) bool {
|
|
stmtsRaw, ok := policy["Statement"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
// Single statement object
|
|
if stmt, ok := stmtsRaw.(map[string]interface{}); ok {
|
|
_, exists := stmt[key]
|
|
return exists
|
|
}
|
|
|
|
// Array of statements
|
|
if stmts, ok := stmtsRaw.([]interface{}); ok {
|
|
for _, s := range stmts {
|
|
stmt, ok := s.(map[string]interface{})
|
|
if !ok {
|
|
continue
|
|
}
|
|
if _, exists := stmt[key]; exists {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|