S3: Implement IAM defaults and STS signing key fallback (#8348)

* S3: Implement IAM defaults and STS signing key fallback logic

* S3: Refactor startup order to init SSE-S3 key manager before IAM

* S3: Derive STS signing key from KEK using HKDF for security isolation

* S3: Document STS signing key fallback in security.toml

* fix(s3api): refine anonymous access logic and secure-by-default behavior

- Initialize anonymous identity by default in `NewIdentityAccessManagement` to prevent nil pointer exceptions.
- Ensure `ReplaceS3ApiConfiguration` preserves the anonymous identity if not present in the new configuration.
- Update `NewIdentityAccessManagement` signature to accept `filerClient`.
- In legacy mode (no policy engine), anonymous defaults to Deny (no actions), preserving secure-by-default behavior.
- Use specific `LookupAnonymous` method instead of generic map lookup.
- Update tests to accommodate signature changes and verify improved anonymous handling.

* feat(s3api): make IAM configuration optional

- Start S3 API server without a configuration file if `EnableIam` option is set.
- Default to `Allow` effect for policy engine when no configuration is provided (Zero-Config mode).
- Handle empty configuration path gracefully in `loadIAMManagerFromConfig`.
- Add integration test `iam_optional_test.go` to verify empty config behavior.

* fix(iamapi): fix signature mismatch in NewIdentityAccessManagementWithStore

* fix(iamapi): properly initialize FilerClient instead of passing nil

* fix(iamapi): properly initialize filer client for IAM management

- Instead of passing `nil`, construct a `wdclient.FilerClient` using the provided `Filers` addresses.
- Ensure `NewIdentityAccessManagementWithStore` receives a valid `filerClient` to avoid potential nil pointer dereferences or limited functionality.

* clean: remove dead code in s3api_server.go

* refactor(s3api): improve IAM initialization, safety and anonymous access security

* fix(s3api): ensure IAM config loads from filer after client init

* fix(s3): resolve test failures in integration, CORS, and tagging tests

- Fix CORS tests by providing explicit anonymous permissions config
- Fix S3 integration tests by setting admin credentials in init
- Align tagging test credentials in CI with IAM defaults
- Added goroutine to retry IAM config load in iamapi server

* fix(s3): allow anonymous access to health targets and S3 Tables when identities are present

* fix(ci): use /healthz for Caddy health check in awscli tests

* iam, s3api: expose DefaultAllow from IAM and Policy Engine

This allows checking the global "Open by Default" configuration from
other components like S3 Tables.

* s3api/s3tables: support DefaultAllow in permission logic and handler

Updated CheckPermissionWithContext to respect the DefaultAllow flag
in PolicyContext. This enables "Open by Default" behavior for
unauthenticated access in zero-config environments. Added a targeted
unit test to verify the logic.

* s3api/s3tables: propagate DefaultAllow through handlers

Propagated the DefaultAllow flag to individual handlers for
namespaces, buckets, tables, policies, and tagging. This ensures
consistent "Open by Default" behavior across all S3 Tables API
endpoints.

* s3api: wire up DefaultAllow for S3 Tables API initialization

Updated registerS3TablesRoutes to query the global IAM configuration
and set the DefaultAllow flag on the S3 Tables API server. This
completes the end-to-end propagation required for anonymous access in
zero-config environments. Added a SetDefaultAllow method to
S3TablesApiServer to facilitate this.

* s3api: fix tests by adding DefaultAllow to mock IAM integrations

The IAMIntegration interface was updated to include DefaultAllow(),
breaking several mock implementations in tests. This commit fixes
the build errors by adding the missing method to the mocks.

* env

* ensure ports

* env

* env

* fix default allow

* add one more test using non-anonymous user

* debug

* add more debug

* less logs

weed/s3api/auth_credentials.go

@@ -22,6 +22,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
 
 	// Import KMS providers to register them
 	_ "github.com/seaweedfs/seaweedfs/weed/kms/aws"
@@ -54,7 +55,7 @@ type IdentityAccessManagement struct {
 	domain            string
 	isAuthEnabled     bool
 	credentialManager *credential.CredentialManager
-	filerClient       filer_pb.SeaweedFilerClient
+	filerClient       *wdclient.FilerClient
 	grpcDialOption    grpc.DialOption
 
 	// IAM Integration for advanced features
@@ -132,15 +133,37 @@ func (c *Credential) isCredentialExpired() bool {
 	return c.Expiration > 0 && c.Expiration < time.Now().Unix()
 }
 
-func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
-	return NewIdentityAccessManagementWithStore(option, "")
+// NewIdentityAccessManagement creates a new IAM manager
+// SetFilerClient updates the filer client and its associated credential store
+func (iam *IdentityAccessManagement) SetFilerClient(filerClient *wdclient.FilerClient) {
+	iam.m.Lock()
+	iam.filerClient = filerClient
+	iam.m.Unlock()
+
+	if iam.credentialManager == nil || filerClient == nil {
+		return
+	}
+
+	// Update credential store to use FilerClient's current filer for HA
+	if store := iam.credentialManager.GetStore(); store != nil {
+		if filerFuncSetter, ok := store.(interface {
+			SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
+		}); ok {
+			filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, iam.grpcDialOption)
+		}
+	}
 }
 
-func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitStore string) *IdentityAccessManagement {
+func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
+	return NewIdentityAccessManagementWithStore(option, filerClient, "")
+}
+
+func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient *wdclient.FilerClient, explicitStore string) *IdentityAccessManagement {
 	iam := &IdentityAccessManagement{
 		domain:       option.DomainName,
 		hashes:       make(map[string]*sync.Pool),
 		hashCounters: make(map[string]*int32),
+		filerClient:  filerClient,
 	}
 
 	// Always initialize credential manager with fallback to defaults
@@ -172,6 +195,25 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto
 
 	iam.credentialManager = credentialManager
 	iam.stopChan = make(chan struct{})
+	iam.grpcDialOption = option.GrpcDialOption
+
+	// Initialize default anonymous identity
+	// This ensures consistent behavior for anonymous access:
+	// 1. In simple auth mode (no IAM integration):
+	//    - lookupAnonymous returns this identity
+	//    - VerifyActionPermission checks actions (which are empty) -> Denies access
+	//    - This preserves the secure-by-default behavior for simple auth
+	// 2. In advanced IAM mode (with Policy Engine):
+	//    - lookupAnonymous returns this identity
+	//    - VerifyActionPermission proceeds to Policy Engine
+	//    - Policy Engine evaluates against policies (DefaultEffect=Allow if no config)
+	//    - This enables the flexible "Open by Default" for zero-config startup
+	iam.identityAnonymous = &Identity{
+		Name:     "anonymous",
+		Account:  &AccountAnonymous,
+		Actions:  []Action{},
+		IsStatic: true,
+	}
 
 	// First, try to load configurations from file or filer
 	startConfigFile := option.Config
@@ -552,6 +594,16 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
 		}
 	}
 
+	// Ensure anonymous identity exists
+	if identityAnonymous == nil {
+		identityAnonymous = &Identity{
+			Name:     "anonymous",
+			Account:  accounts[AccountAnonymous.Id],
+			Actions:  []Action{},
+			IsStatic: true,
+		}
+	}
+
 	// atomically switch
 	iam.identities = identities
 	iam.identityAnonymous = identityAnonymous
@@ -572,6 +624,9 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
 			}
 		}
 		if !exists {
+			if len(envIdent.Credentials) == 0 {
+				continue
+			}
 			iam.identities = append(iam.identities, envIdent)
 			iam.accessKeyIdent[envIdent.Credentials[0].AccessKey] = envIdent
 			iam.nameToIdentity[envIdent.Name] = envIdent
@@ -992,7 +1047,8 @@ func (iam *IdentityAccessManagement) LookupByAccessKey(accessKey string) (identi
 	return iam.lookupByAccessKey(accessKey)
 }
 
-func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
+// LookupAnonymous returns the anonymous identity if it exists
+func (iam *IdentityAccessManagement) LookupAnonymous() (identity *Identity, found bool) {
 	iam.m.RLock()
 	defer iam.m.RUnlock()
 	if iam.identityAnonymous != nil {
@@ -1112,6 +1168,9 @@ func (iam *IdentityAccessManagement) handleAuthResult(w http.ResponseWriter, r *
 // Wrapper to maintain backward compatibility
 func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
 	identity, err, _ := iam.authRequestWithAuthType(r, action)
+	if err != s3err.ErrNone {
+		return nil, err
+	}
 	return identity, err
 }
 
@@ -1173,7 +1232,7 @@ func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request
 		}
 	case authTypeAnonymous:
 		amzAuthType = "Anonymous"
-		if identity, found = iam.lookupAnonymous(); !found {
+		if identity, found = iam.LookupAnonymous(); !found {
 			r.Header.Set(s3_constants.AmzAuthType, amzAuthType)
 			return identity, s3err.ErrAccessDenied, reqAuthType
 		}
@@ -1212,8 +1271,8 @@ func (iam *IdentityAccessManagement) authRequestWithAuthType(r *http.Request, ac
 	// through buckets and checking permissions for each. Skip the global check here.
 	policyAllows := false
 
-	if action == s3_constants.ACTION_LIST && bucket == "" {
-		// ListBuckets operation - authorization handled per-bucket in the handler
+	if action == s3_constants.ACTION_LIST && bucket == "" && identity.Name != s3_constants.AccountAnonymousId {
+		// ListBuckets operation for authenticated users - authorization handled per-bucket in the handler
 	} else {
 		// First check bucket policy if one exists
 		// Bucket policies can grant or deny access to specific users/principals
@@ -1307,8 +1366,8 @@ func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identi
 			return identity, s3err.ErrNotImplemented
 		}
 	case authTypeAnonymous:
-		// Anonymous users cannot use IAM API
-		return identity, s3err.ErrAccessDenied
+		// Anonymous users can be authenticated, but authorization is handled separately
+		return iam.identityAnonymous, s3err.ErrNone
 	default:
 		return identity, s3err.ErrNotImplemented
 	}