fix(s3): omit NotResource:null from bucket policy JSON response (#8658)
* fix(s3): omit NotResource:null from bucket policy JSON response (#8657) Change NotResource from value type to pointer (*StringOrStringSlice) so that omitempty properly omits it when unset, matching the existing Principal field pattern. This prevents IaC tools (Terraform, Ansible) from detecting false configuration drift. Add bucket policy round-trip idempotency integration tests. * simplify JSON comparison in bucket policy idempotency test Use require.JSONEq directly on the raw JSON strings instead of round-tripping through unmarshal/marshal, since JSONEq already handles normalization internally. * fix bucket policy test cases that locked out the admin user The Deny+NotResource test cases used Action:"s3:*" which denied the admin's own GetBucketPolicy call. Scope deny to s3:GetObject only, and add an Allow+NotResource variant instead. * fix(s3): also make Resource a pointer to fix empty string in JSON Apply the same omitempty pointer fix to the Resource field, which was emitting "Resource":"" when only NotResource was set. Add NewStringOrStringSlicePtr helper, make Strings() nil-safe, and handle *StringOrStringSlice in normalizeToStringSliceWithError. * improve bucket policy integration tests per review feedback - Replace time.Sleep with waitForClusterReady using ListBuckets - Use structural hasKey check instead of brittle substring NotContains - Assert specific NoSuchBucketPolicy error code after delete - Handle single-statement policies in hasKey helper
This commit is contained in:
Chris Lu
320
test/s3/policy/bucket_policy_idempotency_test.go
Normal file
320
test/s3/policy/bucket_policy_idempotency_test.go
Normal file
@@ -0,0 +1,320 @@
|
||||
package policy
|
||||
|
||||
// Tests for S3 bucket policy JSON round-trip idempotency.
|
||||
// These validate behavior that IaC tools (Terraform, Ansible) depend on:
|
||||
// PUT a policy, GET it back, and verify the JSON matches exactly.
|
||||
// See https://github.com/seaweedfs/seaweedfs/issues/8657
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func waitForClusterReady(t *testing.T, s3Client *s3.S3) {
|
||||
t.Helper()
|
||||
// ListBuckets is a lightweight call that confirms the S3 API is serving.
|
||||
_, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
|
||||
require.NoError(t, err, "S3 endpoint not ready")
|
||||
}
|
||||
|
||||
func newS3ClientForCluster(t *testing.T, cluster *TestCluster) *s3.S3 {
|
||||
t.Helper()
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(cluster.s3Endpoint),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Credentials: credentials.NewStaticCredentials("admin", "admin", ""),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s3.New(sess)
|
||||
}
|
||||
|
||||
// TestBucketPolicyRoundTrip verifies that GetBucketPolicy returns exactly what
|
||||
// was submitted via PutBucketPolicy, without adding spurious fields like
|
||||
// "NotResource": null. This is the core issue from #8657.
|
||||
func TestBucketPolicyRoundTrip(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
s3Client := newS3ClientForCluster(t, cluster)
|
||||
waitForClusterReady(t, s3Client)
|
||||
bucket := uniqueName("policy-rt")
|
||||
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
policy map[string]interface{}
|
||||
}{
|
||||
{
|
||||
name: "simple allow without NotResource",
|
||||
policy: map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []interface{}{
|
||||
map[string]interface{}{
|
||||
"Sid": "AllowPublicRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple actions without NotResource",
|
||||
policy: map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []interface{}{
|
||||
map[string]interface{}{
|
||||
"Sid": "ReadWrite",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": []interface{}{"s3:GetObject", "s3:PutObject"},
|
||||
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allow with NotResource",
|
||||
policy: map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []interface{}{
|
||||
map[string]interface{}{
|
||||
"Sid": "AllowOutsidePublic",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"NotResource": "arn:aws:s3:::" + bucket + "/private/*",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple statements with NotResource",
|
||||
policy: map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []interface{}{
|
||||
map[string]interface{}{
|
||||
"Sid": "AllowRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::" + bucket + "/*",
|
||||
},
|
||||
map[string]interface{}{
|
||||
"Sid": "DenyPrivateObjects",
|
||||
"Effect": "Deny",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"NotResource": "arn:aws:s3:::" + bucket + "/public/*",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
policyJSON, err := json.Marshal(tc.policy)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Policy: aws.String(string(policyJSON)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
getOut, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, getOut.Policy)
|
||||
|
||||
// The returned policy must not contain fields that were not submitted.
|
||||
// This is the exact issue from #8657: NotResource:null was being added.
|
||||
returnedJSON := *getOut.Policy
|
||||
var returnedPolicy map[string]interface{}
|
||||
require.NoError(t, json.Unmarshal([]byte(returnedJSON), &returnedPolicy))
|
||||
if !hasKey(tc.policy, "NotResource") {
|
||||
require.False(t, hasKey(returnedPolicy, "NotResource"),
|
||||
"returned policy must not contain NotResource when it was not submitted")
|
||||
}
|
||||
|
||||
// Semantic comparison of all submitted fields
|
||||
require.JSONEq(t, string(policyJSON), returnedJSON,
|
||||
"GET should return semantically identical policy to what was PUT")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketPolicyIdempotentPut verifies that putting the same policy twice
|
||||
// does not change the returned value — the behavior IaC tools rely on.
|
||||
func TestBucketPolicyIdempotentPut(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
s3Client := newS3ClientForCluster(t, cluster)
|
||||
waitForClusterReady(t, s3Client)
|
||||
bucket := uniqueName("policy-idem")
|
||||
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Sid": "AllowRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::` + bucket + `/*"
|
||||
}]
|
||||
}`
|
||||
|
||||
// PUT, then GET, then PUT the returned value, then GET again.
|
||||
// Both GETs should return the same result.
|
||||
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Policy: aws.String(policyJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
getOut1, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Re-PUT the policy that was returned by GET (what Terraform does on update)
|
||||
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Policy: getOut1.Policy,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
getOut2, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.JSONEq(t, *getOut1.Policy, *getOut2.Policy,
|
||||
"re-PUTting the GET result must produce identical output (idempotency)")
|
||||
}
|
||||
|
||||
// TestBucketPolicyDeleteAndRecreate verifies clean lifecycle.
|
||||
func TestBucketPolicyDeleteAndRecreate(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
s3Client := newS3ClientForCluster(t, cluster)
|
||||
waitForClusterReady(t, s3Client)
|
||||
bucket := uniqueName("policy-del")
|
||||
|
||||
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
|
||||
policyJSON := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::` + bucket + `/*"
|
||||
}]
|
||||
}`
|
||||
|
||||
// PUT
|
||||
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Policy: aws.String(policyJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// DELETE
|
||||
_, err = s3Client.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// GET should fail with NoSuchBucketPolicy
|
||||
_, err = s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.Error(t, err)
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok, "expected AWS error, got %T: %v", err, err)
|
||||
require.Equal(t, "NoSuchBucketPolicy", awsErr.Code())
|
||||
|
||||
// Re-PUT same policy
|
||||
_, err = s3Client.PutBucketPolicy(&s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Policy: aws.String(policyJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// GET should succeed and be clean
|
||||
getOut, err := s3Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var recreatedPolicy map[string]interface{}
|
||||
require.NoError(t, json.Unmarshal([]byte(*getOut.Policy), &recreatedPolicy))
|
||||
require.False(t, hasKey(recreatedPolicy, "NotResource"),
|
||||
"recreated policy must not contain spurious NotResource")
|
||||
}
|
||||
|
||||
// hasKey checks whether any Statement in the policy map contains the given key.
|
||||
// Handles both single-statement objects and arrays of statements.
|
||||
func hasKey(policy map[string]interface{}, key string) bool {
|
||||
stmtsRaw, ok := policy["Statement"]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// Single statement object
|
||||
if stmt, ok := stmtsRaw.(map[string]interface{}); ok {
|
||||
_, exists := stmt[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
// Array of statements
|
||||
if stmts, ok := stmtsRaw.([]interface{}); ok {
|
||||
for _, s := range stmts {
|
||||
stmt, ok := s.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := stmt[key]; exists {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user