S3: Implement IAM defaults and STS signing key fallback (#8348)

* S3: Implement IAM defaults and STS signing key fallback logic

* S3: Refactor startup order to init SSE-S3 key manager before IAM

* S3: Derive STS signing key from KEK using HKDF for security isolation

* S3: Document STS signing key fallback in security.toml

* fix(s3api): refine anonymous access logic and secure-by-default behavior

- Initialize anonymous identity by default in `NewIdentityAccessManagement` to prevent nil pointer exceptions.
- Ensure `ReplaceS3ApiConfiguration` preserves the anonymous identity if not present in the new configuration.
- Update `NewIdentityAccessManagement` signature to accept `filerClient`.
- In legacy mode (no policy engine), anonymous defaults to Deny (no actions), preserving secure-by-default behavior.
- Use specific `LookupAnonymous` method instead of generic map lookup.
- Update tests to accommodate signature changes and verify improved anonymous handling.

* feat(s3api): make IAM configuration optional

- Start S3 API server without a configuration file if `EnableIam` option is set.
- Default to `Allow` effect for policy engine when no configuration is provided (Zero-Config mode).
- Handle empty configuration path gracefully in `loadIAMManagerFromConfig`.
- Add integration test `iam_optional_test.go` to verify empty config behavior.

* fix(iamapi): fix signature mismatch in NewIdentityAccessManagementWithStore

* fix(iamapi): properly initialize FilerClient instead of passing nil

* fix(iamapi): properly initialize filer client for IAM management

- Instead of passing `nil`, construct a `wdclient.FilerClient` using the provided `Filers` addresses.
- Ensure `NewIdentityAccessManagementWithStore` receives a valid `filerClient` to avoid potential nil pointer dereferences or limited functionality.

* clean: remove dead code in s3api_server.go

* refactor(s3api): improve IAM initialization, safety and anonymous access security

* fix(s3api): ensure IAM config loads from filer after client init

* fix(s3): resolve test failures in integration, CORS, and tagging tests

- Fix CORS tests by providing explicit anonymous permissions config
- Fix S3 integration tests by setting admin credentials in init
- Align tagging test credentials in CI with IAM defaults
- Added goroutine to retry IAM config load in iamapi server

* fix(s3): allow anonymous access to health targets and S3 Tables when identities are present

* fix(ci): use /healthz for Caddy health check in awscli tests

* iam, s3api: expose DefaultAllow from IAM and Policy Engine

This allows checking the global "Open by Default" configuration from
other components like S3 Tables.

* s3api/s3tables: support DefaultAllow in permission logic and handler

Updated CheckPermissionWithContext to respect the DefaultAllow flag
in PolicyContext. This enables "Open by Default" behavior for
unauthenticated access in zero-config environments. Added a targeted
unit test to verify the logic.

* s3api/s3tables: propagate DefaultAllow through handlers

Propagated the DefaultAllow flag to individual handlers for
namespaces, buckets, tables, policies, and tagging. This ensures
consistent "Open by Default" behavior across all S3 Tables API
endpoints.

* s3api: wire up DefaultAllow for S3 Tables API initialization

Updated registerS3TablesRoutes to query the global IAM configuration
and set the DefaultAllow flag on the S3 Tables API server. This
completes the end-to-end propagation required for anonymous access in
zero-config environments. Added a SetDefaultAllow method to
S3TablesApiServer to facilitate this.

* s3api: fix tests by adding DefaultAllow to mock IAM integrations

The IAMIntegration interface was updated to include DefaultAllow(),
breaking several mock implementations in tests. This commit fixes
the build errors by adding the missing method to the mocks.

* env

* ensure ports

* env

* env

* fix default allow

* add one more test using non-anonymous user

* debug

* add more debug

* less logs

weed/s3api/s3tables/handler.go

@@ -44,15 +44,17 @@ const (
 
 // S3TablesHandler handles S3 Tables API requests
 type S3TablesHandler struct {
-	region    string
-	accountID string
+	region       string
+	accountID    string
+	defaultAllow bool // Whether to allow access by default (for zero-config IAM)
 }
 
 // NewS3TablesHandler creates a new S3 Tables handler
 func NewS3TablesHandler() *S3TablesHandler {
 	return &S3TablesHandler{
-		region:    DefaultRegion,
-		accountID: DefaultAccountID,
+		region:       DefaultRegion,
+		accountID:    DefaultAccountID,
+		defaultAllow: false,
 	}
 }
 
@@ -70,6 +72,11 @@ func (h *S3TablesHandler) SetAccountID(accountID string) {
 	}
 }
 
+// SetDefaultAllow sets whether to allow access by default
+func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
+	h.defaultAllow = allow
+}
+
 // FilerClient interface for filer operations
 type FilerClient interface {
 	WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error

weed/s3api/s3tables/handler_bucket_create.go

@@ -16,7 +16,9 @@ import (
 func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
 	// Check permission
 	principal := h.getAccountID(r)
-	if !CanCreateTableBucket(principal, principal, "") {
+	if !CheckPermissionWithContext("CreateTableBucket", principal, principal, "", "", &PolicyContext{
+		DefaultAllow: h.defaultAllow,
+	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
 		return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
 	}

weed/s3api/s3tables/handler_bucket_get_list_delete.go

@@ -72,6 +72,7 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
 	if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
 		return ErrAccessDenied
@@ -101,6 +102,7 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
 	identityActions := getIdentityActions(r)
 	if !CheckPermissionWithContext("ListTableBuckets", principal, accountID, "", "", &PolicyContext{
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to list table buckets")
 		return NewAuthError("ListTableBuckets", principal, "not authorized to list table buckets")
@@ -198,6 +200,7 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
 				if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 					TableBucketName: entry.Entry.Name,
 					IdentityActions: identityActions,
+					DefaultAllow:    h.defaultAllow,
 				}) {
 					continue
 				}
@@ -300,6 +303,7 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http
 		if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 			TableBucketName: bucketName,
 			IdentityActions: identityActions,
+			DefaultAllow:    h.defaultAllow,
 		}) {
 			return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
 		}

weed/s3api/s3tables/handler_namespace.go

@@ -118,6 +118,7 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
 		Namespace:       namespaceName,
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
@@ -258,6 +259,7 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
 		Namespace:       namespaceName,
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
 		return ErrAccessDenied
@@ -344,6 +346,7 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re
 		TableBucketName: bucketName,
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
 		return ErrAccessDenied
@@ -528,6 +531,7 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R
 		Namespace:       namespaceName,
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
 		return ErrAccessDenied

weed/s3api/s3tables/handler_policy.go

@@ -94,6 +94,7 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h
 	if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
 		return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
@@ -171,6 +172,7 @@ func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *h
 	if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
 		return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
@@ -246,6 +248,7 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r
 	if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
 		return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
@@ -346,6 +349,7 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re
 		Namespace:       namespaceName,
 		TableName:       tableName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
 		return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
@@ -453,6 +457,7 @@ func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Re
 		Namespace:       namespaceName,
 		TableName:       tableName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
 		return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
@@ -542,6 +547,7 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http
 		Namespace:       namespaceName,
 		TableName:       tableName,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	}) {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
 		return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
@@ -640,6 +646,7 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque
 			TagKeys:         requestTagKeys,
 			ResourceTags:    existingTags,
 			IdentityActions: identityActions,
+			DefaultAllow:    h.defaultAllow,
 		}) {
 			return NewAuthError("TagResource", principal, "not authorized to tag resource")
 		}
@@ -757,6 +764,7 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht
 			TableBucketTags: bucketTags,
 			ResourceTags:    tags,
 			IdentityActions: identityActions,
+			DefaultAllow:    h.defaultAllow,
 		}) {
 			return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
 		}
@@ -864,6 +872,7 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
 			TagKeys:         req.TagKeys,
 			ResourceTags:    tags,
 			IdentityActions: identityActions,
+			DefaultAllow:    h.defaultAllow,
 		}) {
 			return NewAuthError("UntagResource", principal, "not authorized to untag resource")
 		}

weed/s3api/s3tables/handler_table.go

@@ -145,6 +145,7 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
 		TagKeys:         mapKeys(req.Tags),
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 	bucketAllowed := CheckPermissionWithContext("CreateTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
@@ -154,6 +155,7 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
 		TagKeys:         mapKeys(req.Tags),
 		TableBucketTags: bucketTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 
 	if !nsAllowed && !bucketAllowed {
@@ -390,6 +392,7 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 	bucketAllowed := CheckPermissionWithContext("GetTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
@@ -398,6 +401,7 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 
 	if !tableAllowed && !bucketAllowed {
@@ -527,12 +531,14 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
 				Namespace:       namespaceName,
 				TableBucketTags: bucketTags,
 				IdentityActions: identityActions,
+				DefaultAllow:    h.defaultAllow,
 			})
 			bucketAllowed := CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 				TableBucketName: bucketName,
 				Namespace:       namespaceName,
 				TableBucketTags: bucketTags,
 				IdentityActions: identityActions,
+				DefaultAllow:    h.defaultAllow,
 			})
 			if !nsAllowed && !bucketAllowed {
 				return ErrAccessDenied
@@ -574,6 +580,7 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
 				TableBucketName: bucketName,
 				TableBucketTags: bucketTags,
 				IdentityActions: identityActions,
+				DefaultAllow:    h.defaultAllow,
 			}) {
 				return ErrAccessDenied
 			}
@@ -910,6 +917,7 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 	bucketAllowed := CheckPermissionWithContext("DeleteTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
@@ -918,6 +926,7 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 	if !tableAllowed && !bucketAllowed {
 		h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table")
@@ -1053,6 +1062,7 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 	bucketAllowed := CheckPermissionWithContext("UpdateTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
 		TableBucketName: bucketName,
@@ -1061,6 +1071,7 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
 		TableBucketTags: bucketTags,
 		ResourceTags:    tableTags,
 		IdentityActions: identityActions,
+		DefaultAllow:    h.defaultAllow,
 	})
 
 	if !tableAllowed && !bucketAllowed {

weed/s3api/s3tables/manager.go

@@ -20,7 +20,10 @@ type Manager struct {
 
 // NewManager creates a new Manager.
 func NewManager() *Manager {
-	return &Manager{handler: NewS3TablesHandler()}
+	m := &Manager{handler: NewS3TablesHandler()}
+	// Default to allowing access when IAM is not configured
+	m.handler.SetDefaultAllow(true)
+	return m
 }
 
 // SetRegion sets the AWS region for ARN generation.
@@ -33,6 +36,11 @@ func (m *Manager) SetAccountID(accountID string) {
 	m.handler.SetAccountID(accountID)
 }
 
+// SetDefaultAllow sets whether to allow access by default.
+func (m *Manager) SetDefaultAllow(allow bool) {
+	m.handler.SetDefaultAllow(allow)
+}
+
 // Execute runs an S3 Tables operation and decodes the response into resp (if provided).
 func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
 	body, err := json.Marshal(req)

weed/s3api/s3tables/permissions.go

@@ -86,6 +86,7 @@ type PolicyContext struct {
 	SSEAlgorithm    string
 	KMSKeyArn       string
 	StorageClass    string
+	DefaultAllow    bool
 }
 
 // CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
@@ -117,17 +118,30 @@ func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, res
 }
 
 func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
+	fmt.Printf("DEBUG: checkPermission op=%s princ=%s owner=%s policyLen=%d defaultAllow=%v\n",
+		operation, principal, owner, len(resourcePolicy), ctx != nil && ctx.DefaultAllow)
+	if resourcePolicy != "" {
+		fmt.Printf("DEBUG: policy content: %s\n", resourcePolicy)
+	}
+
 	// Owner always has permission
 	if principal == owner {
+		fmt.Printf("DEBUG: Allowed by Owner check\n")
 		return true
 	}
 
 	if hasIdentityPermission(operation, ctx) {
+		fmt.Printf("DEBUG: Allowed by Identity check\n")
 		return true
 	}
 
-	// If no policy is provided, deny access (default deny)
+	// If no policy is provided, use default allow if enabled
 	if resourcePolicy == "" {
+		if ctx != nil && ctx.DefaultAllow {
+			fmt.Printf("DEBUG: Allowed by DefaultAllow\n")
+			return true
+		}
+		fmt.Printf("DEBUG: Denied by DefaultAllow=false (no policy)\n")
 		return false
 	}
 
@@ -177,7 +191,16 @@ func checkPermission(operation, principal, owner, resourcePolicy, resourceARN st
 		}
 	}
 
-	return hasAllow
+	if hasAllow {
+		return true
+	}
+
+	// If no statement matched, use default allow if enabled
+	if ctx != nil && ctx.DefaultAllow {
+		return true
+	}
+
+	return false
 }
 
 func hasIdentityPermission(operation string, ctx *PolicyContext) bool {

weed/s3api/s3tables/permissions_test.go

@@ -206,3 +206,48 @@ func TestEvaluatePolicyWithConditions(t *testing.T) {
 		})
 	}
 }
+func TestCheckPermissionWithDefaultAllow(t *testing.T) {
+	tests := []struct {
+		name         string
+		defaultAllow bool
+		policy       string
+		expected     bool
+	}{
+		{
+			"default deny (no policy, DefaultAllow=false)",
+			false,
+			"",
+			false,
+		},
+		{
+			"default allow (no policy, DefaultAllow=true)",
+			true,
+			"",
+			true,
+		},
+		{
+			"explicit deny overrides DefaultAllow=true",
+			true,
+			`{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3tables:GetTable"}]}`,
+			false,
+		},
+		{
+			"explicit allow works with DefaultAllow=false",
+			false,
+			`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:GetTable"}]}`,
+			true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := &PolicyContext{
+				DefaultAllow: tt.defaultAllow,
+			}
+			result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", tt.policy, "", ctx)
+			if result != tt.expected {
+				t.Errorf("CheckPermissionWithContext() = %v, want %v (DefaultAllow=%v, Policy=%s)", result, tt.expected, tt.defaultAllow, tt.policy)
+			}
+		})
+	}
+}