Implement IAM propagation to S3 servers (#8130)

* Implement IAM propagation to S3 servers

- Add PropagatingCredentialStore to propagate IAM changes to S3 servers via gRPC
- Add Policy management RPCs to S3 proto and S3ApiServer
- Update CredentialManager to use PropagatingCredentialStore when MasterClient is available
- Wire FilerServer to enable propagation

* Implement parallel IAM propagation and fix S3 cluster registration

- Parallelized IAM change propagation with 10s timeout.
- Refined context usage in PropagatingCredentialStore.
- Added S3Type support to cluster node management.
- Enabled S3 servers to register with gRPC address to the master.
- Ensured IAM configuration reload after policy updates via gRPC.

* Optimize IAM propagation with direct in-memory cache updates

* Secure IAM propagation: Use metadata to skip persistence only on propagation

* pb: refactor IAM and S3 services for unidirectional IAM propagation

- Move SeaweedS3IamCache service from iam.proto to s3.proto.
- Remove legacy IAM management RPCs and empty SeaweedS3 service from s3.proto.
- Enforce that S3 servers only use the synchronization interface.

* pb: regenerate Go code for IAM and S3 services

Updated generated code following the proto refactoring of IAM synchronization services.

* s3api: implement read-only mode for Embedded IAM API

- Add readOnly flag to EmbeddedIamApi to reject write operations via HTTP.
- Enable read-only mode by default in S3ApiServer.
- Handle AccessDenied error in writeIamErrorResponse.
- Embed SeaweedS3IamCacheServer in S3ApiServer.

* credential: refactor PropagatingCredentialStore for unidirectional IAM flow

- Update to use s3_pb.SeaweedS3IamCacheClient for propagation to S3 servers.
- Propagate full Identity object via PutIdentity for consistency.
- Remove redundant propagation of specific user/account/policy management RPCs.
- Add timeout context for propagation calls.

* s3api: implement SeaweedS3IamCacheServer for unidirectional sync

- Update S3ApiServer to implement the cache synchronization gRPC interface.
- Methods (PutIdentity, RemoveIdentity, etc.) now perform direct in-memory cache updates.
- Register SeaweedS3IamCacheServer in command/s3.go.
- Remove registration for the legacy and now empty SeaweedS3 service.

* s3api: update tests for read-only IAM and propagation

- Added TestEmbeddedIamReadOnly to verify rejection of write operations in read-only mode.
- Update test setup to pass readOnly=false to NewEmbeddedIamApi in routing tests.
- Updated EmbeddedIamApiForTest helper with read-only checks matching production behavior.

* s3api: add back temporary debug logs for IAM updates

Log IAM updates received via:
- gRPC propagation (PutIdentity, PutPolicy, etc.)
- Metadata configuration reloads (LoadS3ApiConfigurationFromCredentialManager)
- Core identity management (UpsertIdentity, RemoveIdentity)

* IAM: finalize propagation fix with reduced logging and clarified architecture

* Allow configuring IAM read-only mode for S3 server integration tests

* s3api: add defensive validation to UpsertIdentity

* s3api: fix log message to reference correct IAM read-only flag

* test/s3/iam: ensure WaitForS3Service checks for IAM write permissions

* test: enable writable IAM in Makefile for integration tests

* IAM: add GetPolicy/ListPolicies RPCs to s3.proto

* S3: add GetBucketPolicy and ListBucketPolicies helpers

* S3: support storing generic IAM policies in IdentityAccessManagement

* S3: implement IAM policy RPCs using IdentityAccessManagement

* IAM: fix stale user identity on rename propagation

weed/s3api/auth_credentials.go

@@ -44,6 +44,7 @@ type IdentityAccessManagement struct {
 	identities        []*Identity
 	accessKeyIdent    map[string]*Identity
 	nameToIdentity    map[string]*Identity // O(1) lookup by identity name
+	policies          map[string]*iam_pb.Policy
 	accounts          map[string]*Account
 	emailAccount      map[string]*Account
 	hashes            map[string]*sync.Pool
@@ -83,6 +84,7 @@ type Identity struct {
 	PrincipalArn string                 // ARN for IAM authorization (e.g., "arn:aws:iam::account-id:user/username")
 	Disabled     bool                   // User status: false = enabled (default), true = disabled
 	Claims       map[string]interface{} // JWT claims for policy substitution
+	IsStatic     bool                   // Whether identity was loaded from static config (immutable)
 }
 
 // Account represents a system user, a system user can
@@ -187,6 +189,7 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto
 		iam.staticIdentityNames = make(map[string]bool)
 		for _, identity := range iam.identities {
 			iam.staticIdentityNames[identity.Name] = true
+			identity.IsStatic = true
 		}
 		iam.m.Unlock()
 	}
@@ -294,6 +297,7 @@ func (iam *IdentityAccessManagement) loadEnvironmentVariableCredentials() {
 		Actions: []Action{
 			s3_constants.ACTION_ADMIN,
 		},
+		IsStatic: true,
 	}
 
 	iam.m.Lock()
@@ -407,19 +411,20 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 
 	if hasStaticConfig {
 		// Merge mode: preserve static identities, add/update dynamic ones
-		return iam.mergeS3ApiConfiguration(config)
+		return iam.MergeS3ApiConfiguration(config)
 	}
 
 	// Normal mode: completely replace configuration
-	return iam.replaceS3ApiConfiguration(config)
+	return iam.ReplaceS3ApiConfiguration(config)
 }
 
-// replaceS3ApiConfiguration completely replaces the current configuration (used when no static config)
-func (iam *IdentityAccessManagement) replaceS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
+// ReplaceS3ApiConfiguration completely replaces the current configuration (used when no static config)
+func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
 	var identities []*Identity
 	var identityAnonymous *Identity
 	accessKeyIdent := make(map[string]*Identity)
 	nameToIdentity := make(map[string]*Identity)
+	policies := make(map[string]*iam_pb.Policy)
 	accounts := make(map[string]*Account)
 	emailAccount := make(map[string]*Account)
 	foundAccountAdmin := false
@@ -458,6 +463,9 @@ func (iam *IdentityAccessManagement) replaceS3ApiConfiguration(config *iam_pb.S3
 		}
 		emailAccount[AccountAnonymous.EmailAddress] = accounts[AccountAnonymous.Id]
 	}
+	for _, policy := range config.Policies {
+		policies[policy.Name] = policy
+	}
 	for _, ident := range config.Identities {
 		glog.V(3).Infof("loading identity %s (disabled=%v)", ident.Name, ident.Disabled)
 		t := &Identity{
@@ -537,6 +545,7 @@ func (iam *IdentityAccessManagement) replaceS3ApiConfiguration(config *iam_pb.S3
 	iam.emailAccount = emailAccount
 	iam.accessKeyIdent = accessKeyIdent
 	iam.nameToIdentity = nameToIdentity
+	iam.policies = policies
 	// Update authentication state based on whether identities exist
 	// Once enabled, keep it enabled (one-way toggle)
 	authJustEnabled := iam.updateAuthenticationState(len(identities))
@@ -566,10 +575,10 @@ func (iam *IdentityAccessManagement) replaceS3ApiConfiguration(config *iam_pb.S3
 	return nil
 }
 
-// mergeS3ApiConfiguration merges dynamic configuration with existing static configuration
+// MergeS3ApiConfiguration merges dynamic configuration with existing static configuration
 // Static identities (from file) are preserved and cannot be updated
 // Dynamic identities (from filer/admin) can be added or updated
-func (iam *IdentityAccessManagement) mergeS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
+func (iam *IdentityAccessManagement) MergeS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
 	// Start with current configuration (which includes static identities)
 	iam.m.RLock()
 	identities := make([]*Identity, len(iam.identities))
@@ -583,6 +592,10 @@ func (iam *IdentityAccessManagement) mergeS3ApiConfiguration(config *iam_pb.S3Ap
 	for k, v := range iam.nameToIdentity {
 		nameToIdentity[k] = v
 	}
+	policies := make(map[string]*iam_pb.Policy)
+	for k, v := range iam.policies {
+		policies[k] = v
+	}
 	accounts := make(map[string]*Account)
 	for k, v := range iam.accounts {
 		accounts[k] = v
@@ -755,6 +768,10 @@ func (iam *IdentityAccessManagement) mergeS3ApiConfiguration(config *iam_pb.S3Ap
 		glog.V(3).Infof("Loaded service account %s for dynamic parent %s (expiration: %d)", sa.Id, sa.ParentUser, sa.Expiration)
 	}
 
+	for _, policy := range config.Policies {
+		policies[policy.Name] = policy
+	}
+
 	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
@@ -763,6 +780,7 @@ func (iam *IdentityAccessManagement) mergeS3ApiConfiguration(config *iam_pb.S3Ap
 	iam.emailAccount = emailAccount
 	iam.accessKeyIdent = accessKeyIdent
 	iam.nameToIdentity = nameToIdentity
+	iam.policies = policies
 	// Update authentication state based on whether identities exist
 	// Once enabled, keep it enabled (one-way toggle)
 	authJustEnabled := iam.updateAuthenticationState(len(identities))
@@ -792,6 +810,56 @@ func (iam *IdentityAccessManagement) mergeS3ApiConfiguration(config *iam_pb.S3Ap
 	return nil
 }
 
+func (iam *IdentityAccessManagement) RemoveIdentity(name string) {
+	glog.V(1).Infof("IAM: remove identity %s", name)
+	iam.m.Lock()
+	defer iam.m.Unlock()
+
+	identity, ok := iam.nameToIdentity[name]
+	if !ok {
+		return
+	}
+
+	if identity.IsStatic {
+		glog.V(1).Infof("IAM: skipping removal of static identity %s (immutable)", name)
+		return
+	}
+
+	// Remove from identities slice
+	for i, ident := range iam.identities {
+		if ident.Name == name {
+			iam.identities = append(iam.identities[:i], iam.identities[i+1:]...)
+			break
+		}
+	}
+
+	// Remove from maps
+	delete(iam.nameToIdentity, name)
+	for _, cred := range identity.Credentials {
+		if iam.accessKeyIdent[cred.AccessKey] == identity {
+			delete(iam.accessKeyIdent, cred.AccessKey)
+		}
+	}
+
+	if identity == iam.identityAnonymous {
+		iam.identityAnonymous = nil
+	}
+}
+
+func (iam *IdentityAccessManagement) UpsertIdentity(ident *iam_pb.Identity) error {
+	if ident == nil {
+		return fmt.Errorf("upsert identity failed: nil identity")
+
+	}
+	if ident.Name == "" {
+		return fmt.Errorf("upsert identity failed: empty identity name")
+	}
+	glog.V(1).Infof("IAM: upsert identity %s", ident.Name)
+	return iam.MergeS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{ident},
+	})
+}
+
 // isEnabled reports whether S3 auth should be enforced for this server.
 //
 // Auth is considered enabled if either:
@@ -1316,6 +1384,7 @@ func (iam *IdentityAccessManagement) GetCredentialManager() *credential.Credenti
 
 // LoadS3ApiConfigurationFromCredentialManager loads configuration using the credential manager
 func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager() error {
+	glog.V(0).Infof("IAM: reloading configuration from credential manager")
 	glog.V(1).Infof("Loading S3 API configuration from credential manager")
 
 	s3ApiConfiguration, err := iam.credentialManager.LoadConfiguration(context.Background())
@@ -1503,3 +1572,43 @@ func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity
 	// Use IAM integration for authorization
 	return iam.iamIntegration.AuthorizeAction(ctx, iamIdentity, action, bucket, object, r)
 }
+
+// PutPolicy adds or updates a policy
+func (iam *IdentityAccessManagement) PutPolicy(name string, content string) error {
+	iam.m.Lock()
+	defer iam.m.Unlock()
+	if iam.policies == nil {
+		iam.policies = make(map[string]*iam_pb.Policy)
+	}
+	iam.policies[name] = &iam_pb.Policy{Name: name, Content: content}
+	return nil
+}
+
+// GetPolicy retrieves a policy by name
+func (iam *IdentityAccessManagement) GetPolicy(name string) (*iam_pb.Policy, error) {
+	iam.m.RLock()
+	defer iam.m.RUnlock()
+	if policy, ok := iam.policies[name]; ok {
+		return policy, nil
+	}
+	return nil, fmt.Errorf("policy not found: %s", name)
+}
+
+// DeletePolicy removes a policy
+func (iam *IdentityAccessManagement) DeletePolicy(name string) error {
+	iam.m.Lock()
+	defer iam.m.Unlock()
+	delete(iam.policies, name)
+	return nil
+}
+
+// ListPolicies lists all policies
+func (iam *IdentityAccessManagement) ListPolicies() []*iam_pb.Policy {
+	iam.m.RLock()
+	defer iam.m.RUnlock()
+	var policies []*iam_pb.Policy
+	for _, p := range iam.policies {
+		policies = append(policies, p)
+	}
+	return policies
+}

weed/s3api/auth_credentials_test.go

@@ -13,6 +13,9 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 	jsonpb "google.golang.org/protobuf/encoding/protojson"
+
+	_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc"
+	_ "github.com/seaweedfs/seaweedfs/weed/credential/memory"
 )
 
 func TestIdentityListFileFormat(t *testing.T) {
@@ -742,3 +745,48 @@ func TestSignatureVerificationDoesNotCheckPermissions(t *testing.T) {
 	t.Log("Signature verification no longer checks for Write permission")
 	t.Log("This allows list-only and read-only users to authenticate via AWS Signature V4")
 }
+
+func TestStaticIdentityProtection(t *testing.T) {
+	iam := NewIdentityAccessManagement(&S3ApiServerOption{})
+
+	// Add a static identity
+	staticIdent := &Identity{
+		Name:     "static-user",
+		IsStatic: true,
+	}
+	iam.m.Lock()
+	if iam.nameToIdentity == nil {
+		iam.nameToIdentity = make(map[string]*Identity)
+	}
+	iam.identities = append(iam.identities, staticIdent)
+	iam.nameToIdentity[staticIdent.Name] = staticIdent
+	iam.m.Unlock()
+
+	// Add a dynamic identity
+	dynamicIdent := &Identity{
+		Name:     "dynamic-user",
+		IsStatic: false,
+	}
+	iam.m.Lock()
+	iam.identities = append(iam.identities, dynamicIdent)
+	iam.nameToIdentity[dynamicIdent.Name] = dynamicIdent
+	iam.m.Unlock()
+
+	// Try to remove static identity
+	iam.RemoveIdentity("static-user")
+
+	// Verify static identity still exists
+	iam.m.RLock()
+	_, ok := iam.nameToIdentity["static-user"]
+	iam.m.RUnlock()
+	assert.True(t, ok, "Static identity should not be removed")
+
+	// Try to remove dynamic identity
+	iam.RemoveIdentity("dynamic-user")
+
+	// Verify dynamic identity is removed
+	iam.m.RLock()
+	_, ok = iam.nameToIdentity["dynamic-user"]
+	iam.m.RUnlock()
+	assert.False(t, ok, "Dynamic identity should have been removed")
+}

weed/s3api/s3api_bucket_policy_engine.go

@@ -83,6 +83,16 @@ func (bpe *BucketPolicyEngine) HasPolicyForBucket(bucket string) bool {
 	return bpe.engine.HasPolicyForBucket(bucket)
 }
 
+// GetBucketPolicy gets the policy for a bucket
+func (bpe *BucketPolicyEngine) GetBucketPolicy(bucket string) (*policy_engine.PolicyDocument, error) {
+	return bpe.engine.GetBucketPolicy(bucket)
+}
+
+// ListBucketPolicies returns all buckets that have policies
+func (bpe *BucketPolicyEngine) ListBucketPolicies() []string {
+	return bpe.engine.GetAllBucketsWithPolicies()
+}
+
 // EvaluatePolicy evaluates whether an action is allowed by bucket policy
 //
 // Parameters:

weed/s3api/s3api_embedded_iam.go

@@ -38,13 +38,15 @@ type EmbeddedIamApi struct {
 	getS3ApiConfigurationFunc func(*iam_pb.S3ApiConfiguration) error
 	putS3ApiConfigurationFunc func(*iam_pb.S3ApiConfiguration) error
 	reloadConfigurationFunc   func() error
+	readOnly                  bool
 }
 
 // NewEmbeddedIamApi creates a new embedded IAM API handler.
-func NewEmbeddedIamApi(credentialManager *credential.CredentialManager, iam *IdentityAccessManagement) *EmbeddedIamApi {
+func NewEmbeddedIamApi(credentialManager *credential.CredentialManager, iam *IdentityAccessManagement, readOnly bool) *EmbeddedIamApi {
 	return &EmbeddedIamApi{
 		credentialManager: credentialManager,
 		iam:               iam,
+		readOnly:          readOnly,
 	}
 }
 
@@ -160,6 +162,8 @@ func (e *EmbeddedIamApi) writeIamErrorResponse(w http.ResponseWriter, r *http.Re
 		s3err.WriteXMLResponse(w, r, http.StatusConflict, errorResp)
 	case iam.ErrCodeMalformedPolicyDocumentException, iam.ErrCodeInvalidInputException:
 		s3err.WriteXMLResponse(w, r, http.StatusBadRequest, errorResp)
+	case "AccessDenied", iam.ErrCodeLimitExceededException:
+		s3err.WriteXMLResponse(w, r, http.StatusForbidden, errorResp)
 	case iam.ErrCodeServiceFailureException:
 		s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse)
 	default:
@@ -190,6 +194,7 @@ func (e *EmbeddedIamApi) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration)
 
 // ReloadConfiguration reloads the IAM configuration from the credential manager.
 func (e *EmbeddedIamApi) ReloadConfiguration() error {
+	glog.V(4).Infof("IAM: reloading configuration via EmbeddedIamApi")
 	if e.reloadConfigurationFunc != nil {
 		return e.reloadConfigurationFunc()
 	}
@@ -1043,11 +1048,22 @@ func (e *EmbeddedIamApi) AuthIam(f http.HandlerFunc, _ Action) http.HandlerFunc
 }
 
 // ExecuteAction executes an IAM action with the given values.
-func (e *EmbeddedIamApi) ExecuteAction(values url.Values) (interface{}, *iamError) {
+// If skipPersist is true, the changed configuration is not saved to the persistent store.
+func (e *EmbeddedIamApi) ExecuteAction(values url.Values, skipPersist bool) (interface{}, *iamError) {
 	// Lock to prevent concurrent read-modify-write race conditions
 	e.policyLock.Lock()
 	defer e.policyLock.Unlock()
 
+	action := values.Get("Action")
+	if e.readOnly {
+		switch action {
+		case "ListUsers", "ListAccessKeys", "GetUser", "GetUserPolicy", "ListServiceAccounts", "GetServiceAccount":
+			// Allowed read-only actions
+		default:
+			return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code, Error: fmt.Errorf("IAM write operations are disabled on this server")}
+		}
+	}
+
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 	if err := e.GetS3ApiConfiguration(s3cfg); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
 		return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrInternalError).Code, Error: fmt.Errorf("failed to get s3 api configuration: %v", err)}
@@ -1165,9 +1181,11 @@ func (e *EmbeddedIamApi) ExecuteAction(values url.Values) (interface{}, *iamErro
 		return nil, &iamError{Code: s3err.GetAPIError(s3err.ErrNotImplemented).Code, Error: errors.New(s3err.GetAPIError(s3err.ErrNotImplemented).Description)}
 	}
 	if changed {
-		if err := e.PutS3ApiConfiguration(s3cfg); err != nil {
-			iamErr = &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
-			return nil, iamErr
+		if !skipPersist {
+			if err := e.PutS3ApiConfiguration(s3cfg); err != nil {
+				iamErr = &iamError{Code: iam.ErrCodeServiceFailureException, Error: err}
+				return nil, iamErr
+			}
 		}
 		// Reload in-memory identity maps so subsequent LookupByAccessKey calls
 		// can see newly created or deleted keys immediately
@@ -1196,7 +1214,7 @@ func (e *EmbeddedIamApi) DoActions(w http.ResponseWriter, r *http.Request) {
 		values.Set("CreatedBy", createdBy)
 	}
 
-	response, iamErr := e.ExecuteAction(values)
+	response, iamErr := e.ExecuteAction(values, false)
 	if iamErr != nil {
 		e.writeIamErrorResponse(w, r, iamErr)
 		return

weed/s3api/s3api_embedded_iam_test.go

@@ -87,7 +87,19 @@ func (e *EmbeddedIamApiForTest) DoActions(w http.ResponseWriter, r *http.Request
 	var iamErr *iamError
 	changed := true
 
-	switch r.Form.Get("Action") {
+	action := r.Form.Get("Action")
+
+	if e.readOnly {
+		switch action {
+		case "ListUsers", "ListAccessKeys", "GetUser", "GetUserPolicy", "ListServiceAccounts", "GetServiceAccount":
+			// Allowed read-only actions
+		default:
+			e.writeIamErrorResponse(w, r, &iamError{Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code, Error: fmt.Errorf("IAM write operations are disabled on this server")})
+			return
+		}
+	}
+
+	switch action {
 	case "ListUsers":
 		response = e.ListUsers(s3cfg, values)
 		changed = false
@@ -1691,7 +1703,7 @@ func TestEmbeddedIamExecuteAction(t *testing.T) {
 	vals.Set("Action", "CreateUser")
 	vals.Set("UserName", "ExecuteActionUser")
 
-	resp, iamErr := api.ExecuteAction(vals)
+	resp, iamErr := api.ExecuteAction(vals, false)
 	assert.Nil(t, iamErr)
 
 	// Verify response type
@@ -1703,3 +1715,33 @@ func TestEmbeddedIamExecuteAction(t *testing.T) {
 	assert.Len(t, api.mockConfig.Identities, 1)
 	assert.Equal(t, "ExecuteActionUser", api.mockConfig.Identities[0].Name)
 }
+
+// TestEmbeddedIamReadOnly tests that write operations are blocked when readOnly is true
+func TestEmbeddedIamReadOnly(t *testing.T) {
+	api := NewEmbeddedIamApiForTest()
+	api.readOnly = true
+
+	// Try CreateUser (Write)
+	userName := aws.String("ReadOnlyUser")
+	params := &iam.CreateUserInput{UserName: userName}
+	req, _ := iam.New(session.New()).CreateUserRequest(params)
+	_ = req.Build()
+
+	response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, response.Code)
+
+	code, msg := extractEmbeddedIamErrorCodeAndMessage(response)
+	assert.Equal(t, "AccessDenied", code)
+	assert.Contains(t, msg, "IAM write operations are disabled")
+
+	// Try ListUsers (Read) - Should succeed
+	paramsList := &iam.ListUsersInput{}
+	reqList, _ := iam.New(session.New()).ListUsersRequest(paramsList)
+	_ = reqList.Build()
+
+	outList := iamListUsersResponse{}
+	responseList, err := executeEmbeddedIamRequest(api, reqList.HTTPRequest, &outList)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, responseList.Code)
+}

weed/s3api/s3api_server.go

@@ -52,11 +52,14 @@ type S3ApiServerOption struct {
 	ConcurrentUploadLimit     int64
 	ConcurrentFileUploadLimit int64
 	EnableIam                 bool // Enable embedded IAM API on the same port
+	IamReadOnly               bool // Disable IAM write operations on this server
 	Cipher                    bool // encrypt data on volume servers
+	BindIp                    string
+	GrpcPort                  int
 }
 
 type S3ApiServer struct {
-	s3_pb.UnimplementedSeaweedS3Server
+	s3_pb.UnimplementedSeaweedS3IamCacheServer
 	option                *S3ApiServerOption
 	iam                   *IdentityAccessManagement
 	iamIntegration        *S3IAMIntegration // Advanced IAM integration for JWT authentication
@@ -114,13 +117,17 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 	// Uses the battle-tested vidMap with filer-based lookups
 	// Supports multiple filer addresses with automatic failover for high availability
 	var filerClient *wdclient.FilerClient
-	if len(option.Masters) > 0 && option.FilerGroup != "" {
+	if len(option.Masters) > 0 {
 		// Enable filer discovery via master
 		masterMap := make(map[string]pb.ServerAddress)
 		for i, addr := range option.Masters {
 			masterMap[fmt.Sprintf("master%d", i)] = addr
 		}
-		masterClient := wdclient.NewMasterClient(option.GrpcDialOption, option.FilerGroup, cluster.S3Type, "", "", "", *pb.NewServiceDiscoveryFromMap(masterMap))
+		clientHost := option.BindIp
+		if clientHost == "0.0.0.0" || clientHost == "" {
+			clientHost = util.DetectedHostAddress()
+		}
+		masterClient := wdclient.NewMasterClient(option.GrpcDialOption, option.FilerGroup, cluster.S3Type, pb.ServerAddress(util.JoinHostPort(clientHost, option.GrpcPort)), "", "", *pb.NewServiceDiscoveryFromMap(masterMap))
 		// Start the master client connection loop - required for GetMaster() to work
 		go masterClient.KeepConnectedToMaster(context.Background())
 
@@ -203,8 +210,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
 
 	// Initialize embedded IAM API if enabled
 	if option.EnableIam {
-		s3ApiServer.embeddedIam = NewEmbeddedIamApi(s3ApiServer.credentialManager, iam)
-		glog.V(1).Infof("Embedded IAM API initialized (use -iam=false to disable)")
+		s3ApiServer.embeddedIam = NewEmbeddedIamApi(s3ApiServer.credentialManager, iam, option.IamReadOnly)
+		if option.IamReadOnly {
+			glog.V(1).Infof("Embedded IAM API initialized in read-only mode (use -s3.iam.readOnly=false to enable write operations)")
+		} else {
+			glog.V(1).Infof("Embedded IAM API initialized in writable mode (WARNING: updates will not be propagated to other S3 servers)")
+		}
 	}
 
 	if option.Config != "" {

weed/s3api/s3api_server_grpc.go

@@ -4,341 +4,95 @@ import (
 	"context"
 
 	"fmt"
-	"net/url"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
 
-func (s3a *S3ApiServer) executeAction(values url.Values) (interface{}, error) {
-	if s3a.embeddedIam == nil {
-		return nil, fmt.Errorf("embedded iam is disabled")
-	}
-	response, iamErr := s3a.embeddedIam.ExecuteAction(values)
-	if iamErr != nil {
-		return nil, fmt.Errorf("IAM error: %s - %v", iamErr.Code, iamErr.Error)
-	}
-	return response, nil
-}
+// SeaweedS3IamCacheServer Implementation
+// This interface is dedicated to UNIDIRECTIONAL updates from Filer to S3 Server.
+// S3 Server acts purely as a cache.
 
-func (s3a *S3ApiServer) ListUsers(ctx context.Context, req *iam_pb.ListUsersRequest) (*iam_pb.ListUsersResponse, error) {
-	values := url.Values{}
-	values.Set("Action", "ListUsers")
-	resp, err := s3a.executeAction(values)
-	if err != nil {
+func (s3a *S3ApiServer) PutIdentity(ctx context.Context, req *iam_pb.PutIdentityRequest) (*iam_pb.PutIdentityResponse, error) {
+	if req.Identity == nil {
+		return nil, fmt.Errorf("identity is required")
+	}
+	// Direct in-memory cache update
+	glog.V(1).Infof("IAM: received identity update for %s", req.Identity.Name)
+	if err := s3a.iam.UpsertIdentity(req.Identity); err != nil {
+		glog.Errorf("failed to update identity cache for %s: %v", req.Identity.Name, err)
 		return nil, err
 	}
-	iamResp, ok := resp.(iamListUsersResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListUsers response type %T", resp)
-	}
-	var usernames []string
-	for _, user := range iamResp.ListUsersResult.Users {
-		if user != nil && user.UserName != nil {
-			usernames = append(usernames, *user.UserName)
-		}
-	}
-	return &iam_pb.ListUsersResponse{Usernames: usernames}, nil
+	return &iam_pb.PutIdentityResponse{}, nil
 }
 
-func (s3a *S3ApiServer) CreateUser(ctx context.Context, req *iam_pb.CreateUserRequest) (*iam_pb.CreateUserResponse, error) {
-	if req.Identity == nil || req.Identity.Name == "" {
-		return nil, fmt.Errorf("username name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateUser")
-	values.Set("UserName", req.Identity.Name)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetUser(ctx context.Context, req *iam_pb.GetUserRequest) (*iam_pb.GetUserResponse, error) {
+func (s3a *S3ApiServer) RemoveIdentity(ctx context.Context, req *iam_pb.RemoveIdentityRequest) (*iam_pb.RemoveIdentityResponse, error) {
 	if req.Username == "" {
 		return nil, fmt.Errorf("username is required")
 	}
-	values := url.Values{}
-	values.Set("Action", "GetUser")
-	values.Set("UserName", req.Username)
-	resp, err := s3a.executeAction(values)
+	// Direct in-memory cache update
+	glog.V(1).Infof("IAM: received identity removal for %s", req.Username)
+	s3a.iam.RemoveIdentity(req.Username)
+	return &iam_pb.RemoveIdentityResponse{}, nil
+}
+
+func (s3a *S3ApiServer) PutPolicy(ctx context.Context, req *iam_pb.PutPolicyRequest) (*iam_pb.PutPolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+
+	// Update IAM policy cache
+	glog.V(1).Infof("IAM: received policy update for %s", req.Name)
+	if s3a.iam != nil {
+		if err := s3a.iam.PutPolicy(req.Name, req.Content); err != nil {
+			glog.Errorf("failed to update policy cache for %s: %v", req.Name, err)
+			return nil, err
+		}
+	}
+	return &iam_pb.PutPolicyResponse{}, nil
+}
+
+func (s3a *S3ApiServer) DeletePolicy(ctx context.Context, req *iam_pb.DeletePolicyRequest) (*iam_pb.DeletePolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+
+	// Delete from IAM policy cache
+	glog.V(1).Infof("IAM: received policy removal for %s", req.Name)
+	if s3a.iam != nil {
+		if err := s3a.iam.DeletePolicy(req.Name); err != nil {
+			glog.Errorf("failed to delete policy cache for %s: %v", req.Name, err)
+			return nil, err
+		}
+	}
+	return &iam_pb.DeletePolicyResponse{}, nil
+}
+
+func (s3a *S3ApiServer) GetPolicy(ctx context.Context, req *iam_pb.GetPolicyRequest) (*iam_pb.GetPolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+	if s3a.iam == nil {
+		return &iam_pb.GetPolicyResponse{}, nil
+	}
+	policy, err := s3a.iam.GetPolicy(req.Name)
 	if err != nil {
-		return nil, err
+		return &iam_pb.GetPolicyResponse{}, nil // Not found is fine for cache
 	}
-	iamResp, ok := resp.(iamGetUserResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetUser response type %T", resp)
-	}
-
-	var username string
-	if iamResp.GetUserResult.User.UserName != nil {
-		username = *iamResp.GetUserResult.User.UserName
-	}
-
-	return &iam_pb.GetUserResponse{
-		Identity: &iam_pb.Identity{
-			Name: username,
-		},
+	return &iam_pb.GetPolicyResponse{
+		Name:    policy.Name,
+		Content: policy.Content,
 	}, nil
 }
 
-func (s3a *S3ApiServer) UpdateUser(ctx context.Context, req *iam_pb.UpdateUserRequest) (*iam_pb.UpdateUserResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
+func (s3a *S3ApiServer) ListPolicies(ctx context.Context, req *iam_pb.ListPoliciesRequest) (*iam_pb.ListPoliciesResponse, error) {
+	resp := &iam_pb.ListPoliciesResponse{}
+	if s3a.iam == nil {
+		return resp, nil
 	}
-	values := url.Values{}
-	values.Set("Action", "UpdateUser")
-	values.Set("UserName", req.Username)
-	// UpdateUser in DoActions expects "NewUserName" if renaming, but CreateUser just takes UserName.
-	// Looking at s3api_embedded_iam.go, UpdateUser uses "NewUserName" to change name.
-	if req.Identity != nil && req.Identity.Name != "" {
-		values.Set("NewUserName", req.Identity.Name)
+	policies := s3a.iam.ListPolicies()
+	for _, policy := range policies {
+		resp.Policies = append(resp.Policies, policy)
 	}
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.UpdateUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteUser(ctx context.Context, req *iam_pb.DeleteUserRequest) (*iam_pb.DeleteUserResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteUser")
-	values.Set("UserName", req.Username)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) ListAccessKeys(ctx context.Context, req *iam_pb.ListAccessKeysRequest) (*iam_pb.ListAccessKeysResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "ListAccessKeys")
-	values.Set("UserName", req.Username)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamListAccessKeysResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListAccessKeys response type %T", resp)
-	}
-	var accessKeys []*iam_pb.Credential
-	for _, meta := range iamResp.ListAccessKeysResult.AccessKeyMetadata {
-		if meta != nil && meta.AccessKeyId != nil && meta.Status != nil {
-			accessKeys = append(accessKeys, &iam_pb.Credential{
-				AccessKey: *meta.AccessKeyId,
-				Status:    *meta.Status,
-			})
-		}
-	}
-	return &iam_pb.ListAccessKeysResponse{AccessKeys: accessKeys}, nil
-}
-
-func (s3a *S3ApiServer) CreateAccessKey(ctx context.Context, req *iam_pb.CreateAccessKeyRequest) (*iam_pb.CreateAccessKeyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateAccessKey")
-	values.Set("UserName", req.Username)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateAccessKeyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteAccessKey(ctx context.Context, req *iam_pb.DeleteAccessKeyRequest) (*iam_pb.DeleteAccessKeyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.AccessKey == "" {
-		return nil, fmt.Errorf("access key is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteAccessKey")
-	values.Set("UserName", req.Username)
-	values.Set("AccessKeyId", req.AccessKey)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteAccessKeyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) PutUserPolicy(ctx context.Context, req *iam_pb.PutUserPolicyRequest) (*iam_pb.PutUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "PutUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	values.Set("PolicyDocument", req.PolicyDocument)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.PutUserPolicyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetUserPolicy(ctx context.Context, req *iam_pb.GetUserPolicyRequest) (*iam_pb.GetUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "GetUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamGetUserPolicyResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetUserPolicy response type %T", resp)
-	}
-	return &iam_pb.GetUserPolicyResponse{
-		Username:       iamResp.GetUserPolicyResult.UserName,
-		PolicyName:     iamResp.GetUserPolicyResult.PolicyName,
-		PolicyDocument: iamResp.GetUserPolicyResult.PolicyDocument,
-	}, nil
-}
-
-func (s3a *S3ApiServer) DeleteUserPolicy(ctx context.Context, req *iam_pb.DeleteUserPolicyRequest) (*iam_pb.DeleteUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteUserPolicyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) ListServiceAccounts(ctx context.Context, req *iam_pb.ListServiceAccountsRequest) (*iam_pb.ListServiceAccountsResponse, error) {
-	values := url.Values{}
-	values.Set("Action", "ListServiceAccounts")
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamListServiceAccountsResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListServiceAccounts response type %T", resp)
-	}
-	var serviceAccounts []*iam_pb.ServiceAccount
-	for _, sa := range iamResp.ListServiceAccountsResult.ServiceAccounts {
-		if sa != nil {
-			serviceAccounts = append(serviceAccounts, &iam_pb.ServiceAccount{
-				Id:          sa.ServiceAccountId,
-				ParentUser:  sa.ParentUser,
-				Description: sa.Description,
-				Credential: &iam_pb.Credential{
-					AccessKey: sa.AccessKeyId,
-					Status:    sa.Status,
-				},
-			})
-		}
-	}
-	return &iam_pb.ListServiceAccountsResponse{ServiceAccounts: serviceAccounts}, nil
-}
-
-func (s3a *S3ApiServer) CreateServiceAccount(ctx context.Context, req *iam_pb.CreateServiceAccountRequest) (*iam_pb.CreateServiceAccountResponse, error) {
-	if req.ServiceAccount == nil || req.ServiceAccount.CreatedBy == "" {
-		return nil, fmt.Errorf("service account owner is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateServiceAccount")
-	values.Set("CreatedBy", req.ServiceAccount.CreatedBy)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) UpdateServiceAccount(ctx context.Context, req *iam_pb.UpdateServiceAccountRequest) (*iam_pb.UpdateServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "UpdateServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	if req.ServiceAccount != nil && req.ServiceAccount.Disabled {
-		values.Set("Status", "Inactive")
-	}
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.UpdateServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteServiceAccount(ctx context.Context, req *iam_pb.DeleteServiceAccountRequest) (*iam_pb.DeleteServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetServiceAccount(ctx context.Context, req *iam_pb.GetServiceAccountRequest) (*iam_pb.GetServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "GetServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamGetServiceAccountResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetServiceAccount response type %T", resp)
-	}
-
-	var serviceAccount *iam_pb.ServiceAccount
-	sa := iamResp.GetServiceAccountResult.ServiceAccount
-	serviceAccount = &iam_pb.ServiceAccount{
-		Id:          sa.ServiceAccountId,
-		ParentUser:  sa.ParentUser,
-		Description: sa.Description,
-		Credential: &iam_pb.Credential{
-			AccessKey: sa.AccessKeyId,
-			Status:    sa.Status,
-		},
-	}
-
-	return &iam_pb.GetServiceAccountResponse{
-		ServiceAccount: serviceAccount,
-	}, nil
+	return resp, nil
 }

weed/s3api/s3api_server_routing_test.go

@@ -31,7 +31,7 @@ func setupRoutingTestServer(t *testing.T) *S3ApiServer {
 		option:            opt,
 		iam:               iam,
 		credentialManager: iam.credentialManager,
-		embeddedIam:       NewEmbeddedIamApi(iam.credentialManager, iam),
+		embeddedIam:       NewEmbeddedIamApi(iam.credentialManager, iam, false),
 		stsHandlers:       &STSHandlers{},
 	}