Implement IAM propagation to S3 servers (#8130)

* Implement IAM propagation to S3 servers

- Add PropagatingCredentialStore to propagate IAM changes to S3 servers via gRPC
- Add Policy management RPCs to S3 proto and S3ApiServer
- Update CredentialManager to use PropagatingCredentialStore when MasterClient is available
- Wire FilerServer to enable propagation

* Implement parallel IAM propagation and fix S3 cluster registration

- Parallelized IAM change propagation with 10s timeout.
- Refined context usage in PropagatingCredentialStore.
- Added S3Type support to cluster node management.
- Enabled S3 servers to register with gRPC address to the master.
- Ensured IAM configuration reload after policy updates via gRPC.

* Optimize IAM propagation with direct in-memory cache updates

* Secure IAM propagation: Use metadata to skip persistence only on propagation

* pb: refactor IAM and S3 services for unidirectional IAM propagation

- Move SeaweedS3IamCache service from iam.proto to s3.proto.
- Remove legacy IAM management RPCs and empty SeaweedS3 service from s3.proto.
- Enforce that S3 servers only use the synchronization interface.

* pb: regenerate Go code for IAM and S3 services

Updated generated code following the proto refactoring of IAM synchronization services.

* s3api: implement read-only mode for Embedded IAM API

- Add readOnly flag to EmbeddedIamApi to reject write operations via HTTP.
- Enable read-only mode by default in S3ApiServer.
- Handle AccessDenied error in writeIamErrorResponse.
- Embed SeaweedS3IamCacheServer in S3ApiServer.

* credential: refactor PropagatingCredentialStore for unidirectional IAM flow

- Update to use s3_pb.SeaweedS3IamCacheClient for propagation to S3 servers.
- Propagate full Identity object via PutIdentity for consistency.
- Remove redundant propagation of specific user/account/policy management RPCs.
- Add timeout context for propagation calls.

* s3api: implement SeaweedS3IamCacheServer for unidirectional sync

- Update S3ApiServer to implement the cache synchronization gRPC interface.
- Methods (PutIdentity, RemoveIdentity, etc.) now perform direct in-memory cache updates.
- Register SeaweedS3IamCacheServer in command/s3.go.
- Remove registration for the legacy and now empty SeaweedS3 service.

* s3api: update tests for read-only IAM and propagation

- Added TestEmbeddedIamReadOnly to verify rejection of write operations in read-only mode.
- Update test setup to pass readOnly=false to NewEmbeddedIamApi in routing tests.
- Updated EmbeddedIamApiForTest helper with read-only checks matching production behavior.

* s3api: add back temporary debug logs for IAM updates

Log IAM updates received via:
- gRPC propagation (PutIdentity, PutPolicy, etc.)
- Metadata configuration reloads (LoadS3ApiConfigurationFromCredentialManager)
- Core identity management (UpsertIdentity, RemoveIdentity)

* IAM: finalize propagation fix with reduced logging and clarified architecture

* Allow configuring IAM read-only mode for S3 server integration tests

* s3api: add defensive validation to UpsertIdentity

* s3api: fix log message to reference correct IAM read-only flag

* test/s3/iam: ensure WaitForS3Service checks for IAM write permissions

* test: enable writable IAM in Makefile for integration tests

* IAM: add GetPolicy/ListPolicies RPCs to s3.proto

* S3: add GetBucketPolicy and ListBucketPolicies helpers

* S3: support storing generic IAM policies in IdentityAccessManagement

* S3: implement IAM policy RPCs using IdentityAccessManagement

* IAM: fix stale user identity on rename propagation

weed/s3api/s3api_server_grpc.go

@@ -4,341 +4,95 @@ import (
 	"context"
 
 	"fmt"
-	"net/url"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
 
-func (s3a *S3ApiServer) executeAction(values url.Values) (interface{}, error) {
-	if s3a.embeddedIam == nil {
-		return nil, fmt.Errorf("embedded iam is disabled")
-	}
-	response, iamErr := s3a.embeddedIam.ExecuteAction(values)
-	if iamErr != nil {
-		return nil, fmt.Errorf("IAM error: %s - %v", iamErr.Code, iamErr.Error)
-	}
-	return response, nil
-}
+// SeaweedS3IamCacheServer Implementation
+// This interface is dedicated to UNIDIRECTIONAL updates from Filer to S3 Server.
+// S3 Server acts purely as a cache.
 
-func (s3a *S3ApiServer) ListUsers(ctx context.Context, req *iam_pb.ListUsersRequest) (*iam_pb.ListUsersResponse, error) {
-	values := url.Values{}
-	values.Set("Action", "ListUsers")
-	resp, err := s3a.executeAction(values)
-	if err != nil {
+func (s3a *S3ApiServer) PutIdentity(ctx context.Context, req *iam_pb.PutIdentityRequest) (*iam_pb.PutIdentityResponse, error) {
+	if req.Identity == nil {
+		return nil, fmt.Errorf("identity is required")
+	}
+	// Direct in-memory cache update
+	glog.V(1).Infof("IAM: received identity update for %s", req.Identity.Name)
+	if err := s3a.iam.UpsertIdentity(req.Identity); err != nil {
+		glog.Errorf("failed to update identity cache for %s: %v", req.Identity.Name, err)
 		return nil, err
 	}
-	iamResp, ok := resp.(iamListUsersResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListUsers response type %T", resp)
-	}
-	var usernames []string
-	for _, user := range iamResp.ListUsersResult.Users {
-		if user != nil && user.UserName != nil {
-			usernames = append(usernames, *user.UserName)
-		}
-	}
-	return &iam_pb.ListUsersResponse{Usernames: usernames}, nil
+	return &iam_pb.PutIdentityResponse{}, nil
 }
 
-func (s3a *S3ApiServer) CreateUser(ctx context.Context, req *iam_pb.CreateUserRequest) (*iam_pb.CreateUserResponse, error) {
-	if req.Identity == nil || req.Identity.Name == "" {
-		return nil, fmt.Errorf("username name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateUser")
-	values.Set("UserName", req.Identity.Name)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetUser(ctx context.Context, req *iam_pb.GetUserRequest) (*iam_pb.GetUserResponse, error) {
+func (s3a *S3ApiServer) RemoveIdentity(ctx context.Context, req *iam_pb.RemoveIdentityRequest) (*iam_pb.RemoveIdentityResponse, error) {
 	if req.Username == "" {
 		return nil, fmt.Errorf("username is required")
 	}
-	values := url.Values{}
-	values.Set("Action", "GetUser")
-	values.Set("UserName", req.Username)
-	resp, err := s3a.executeAction(values)
+	// Direct in-memory cache update
+	glog.V(1).Infof("IAM: received identity removal for %s", req.Username)
+	s3a.iam.RemoveIdentity(req.Username)
+	return &iam_pb.RemoveIdentityResponse{}, nil
+}
+
+func (s3a *S3ApiServer) PutPolicy(ctx context.Context, req *iam_pb.PutPolicyRequest) (*iam_pb.PutPolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+
+	// Update IAM policy cache
+	glog.V(1).Infof("IAM: received policy update for %s", req.Name)
+	if s3a.iam != nil {
+		if err := s3a.iam.PutPolicy(req.Name, req.Content); err != nil {
+			glog.Errorf("failed to update policy cache for %s: %v", req.Name, err)
+			return nil, err
+		}
+	}
+	return &iam_pb.PutPolicyResponse{}, nil
+}
+
+func (s3a *S3ApiServer) DeletePolicy(ctx context.Context, req *iam_pb.DeletePolicyRequest) (*iam_pb.DeletePolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+
+	// Delete from IAM policy cache
+	glog.V(1).Infof("IAM: received policy removal for %s", req.Name)
+	if s3a.iam != nil {
+		if err := s3a.iam.DeletePolicy(req.Name); err != nil {
+			glog.Errorf("failed to delete policy cache for %s: %v", req.Name, err)
+			return nil, err
+		}
+	}
+	return &iam_pb.DeletePolicyResponse{}, nil
+}
+
+func (s3a *S3ApiServer) GetPolicy(ctx context.Context, req *iam_pb.GetPolicyRequest) (*iam_pb.GetPolicyResponse, error) {
+	if req.Name == "" {
+		return nil, fmt.Errorf("policy name is required")
+	}
+	if s3a.iam == nil {
+		return &iam_pb.GetPolicyResponse{}, nil
+	}
+	policy, err := s3a.iam.GetPolicy(req.Name)
 	if err != nil {
-		return nil, err
+		return &iam_pb.GetPolicyResponse{}, nil // Not found is fine for cache
 	}
-	iamResp, ok := resp.(iamGetUserResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetUser response type %T", resp)
-	}
-
-	var username string
-	if iamResp.GetUserResult.User.UserName != nil {
-		username = *iamResp.GetUserResult.User.UserName
-	}
-
-	return &iam_pb.GetUserResponse{
-		Identity: &iam_pb.Identity{
-			Name: username,
-		},
+	return &iam_pb.GetPolicyResponse{
+		Name:    policy.Name,
+		Content: policy.Content,
 	}, nil
 }
 
-func (s3a *S3ApiServer) UpdateUser(ctx context.Context, req *iam_pb.UpdateUserRequest) (*iam_pb.UpdateUserResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
+func (s3a *S3ApiServer) ListPolicies(ctx context.Context, req *iam_pb.ListPoliciesRequest) (*iam_pb.ListPoliciesResponse, error) {
+	resp := &iam_pb.ListPoliciesResponse{}
+	if s3a.iam == nil {
+		return resp, nil
 	}
-	values := url.Values{}
-	values.Set("Action", "UpdateUser")
-	values.Set("UserName", req.Username)
-	// UpdateUser in DoActions expects "NewUserName" if renaming, but CreateUser just takes UserName.
-	// Looking at s3api_embedded_iam.go, UpdateUser uses "NewUserName" to change name.
-	if req.Identity != nil && req.Identity.Name != "" {
-		values.Set("NewUserName", req.Identity.Name)
+	policies := s3a.iam.ListPolicies()
+	for _, policy := range policies {
+		resp.Policies = append(resp.Policies, policy)
 	}
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.UpdateUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteUser(ctx context.Context, req *iam_pb.DeleteUserRequest) (*iam_pb.DeleteUserResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteUser")
-	values.Set("UserName", req.Username)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteUserResponse{}, nil
-}
-
-func (s3a *S3ApiServer) ListAccessKeys(ctx context.Context, req *iam_pb.ListAccessKeysRequest) (*iam_pb.ListAccessKeysResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "ListAccessKeys")
-	values.Set("UserName", req.Username)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamListAccessKeysResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListAccessKeys response type %T", resp)
-	}
-	var accessKeys []*iam_pb.Credential
-	for _, meta := range iamResp.ListAccessKeysResult.AccessKeyMetadata {
-		if meta != nil && meta.AccessKeyId != nil && meta.Status != nil {
-			accessKeys = append(accessKeys, &iam_pb.Credential{
-				AccessKey: *meta.AccessKeyId,
-				Status:    *meta.Status,
-			})
-		}
-	}
-	return &iam_pb.ListAccessKeysResponse{AccessKeys: accessKeys}, nil
-}
-
-func (s3a *S3ApiServer) CreateAccessKey(ctx context.Context, req *iam_pb.CreateAccessKeyRequest) (*iam_pb.CreateAccessKeyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateAccessKey")
-	values.Set("UserName", req.Username)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateAccessKeyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteAccessKey(ctx context.Context, req *iam_pb.DeleteAccessKeyRequest) (*iam_pb.DeleteAccessKeyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.AccessKey == "" {
-		return nil, fmt.Errorf("access key is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteAccessKey")
-	values.Set("UserName", req.Username)
-	values.Set("AccessKeyId", req.AccessKey)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteAccessKeyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) PutUserPolicy(ctx context.Context, req *iam_pb.PutUserPolicyRequest) (*iam_pb.PutUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "PutUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	values.Set("PolicyDocument", req.PolicyDocument)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.PutUserPolicyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetUserPolicy(ctx context.Context, req *iam_pb.GetUserPolicyRequest) (*iam_pb.GetUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "GetUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamGetUserPolicyResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetUserPolicy response type %T", resp)
-	}
-	return &iam_pb.GetUserPolicyResponse{
-		Username:       iamResp.GetUserPolicyResult.UserName,
-		PolicyName:     iamResp.GetUserPolicyResult.PolicyName,
-		PolicyDocument: iamResp.GetUserPolicyResult.PolicyDocument,
-	}, nil
-}
-
-func (s3a *S3ApiServer) DeleteUserPolicy(ctx context.Context, req *iam_pb.DeleteUserPolicyRequest) (*iam_pb.DeleteUserPolicyResponse, error) {
-	if req.Username == "" {
-		return nil, fmt.Errorf("username is required")
-	}
-	if req.PolicyName == "" {
-		return nil, fmt.Errorf("policy name is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteUserPolicy")
-	values.Set("UserName", req.Username)
-	values.Set("PolicyName", req.PolicyName)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteUserPolicyResponse{}, nil
-}
-
-func (s3a *S3ApiServer) ListServiceAccounts(ctx context.Context, req *iam_pb.ListServiceAccountsRequest) (*iam_pb.ListServiceAccountsResponse, error) {
-	values := url.Values{}
-	values.Set("Action", "ListServiceAccounts")
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamListServiceAccountsResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM ListServiceAccounts response type %T", resp)
-	}
-	var serviceAccounts []*iam_pb.ServiceAccount
-	for _, sa := range iamResp.ListServiceAccountsResult.ServiceAccounts {
-		if sa != nil {
-			serviceAccounts = append(serviceAccounts, &iam_pb.ServiceAccount{
-				Id:          sa.ServiceAccountId,
-				ParentUser:  sa.ParentUser,
-				Description: sa.Description,
-				Credential: &iam_pb.Credential{
-					AccessKey: sa.AccessKeyId,
-					Status:    sa.Status,
-				},
-			})
-		}
-	}
-	return &iam_pb.ListServiceAccountsResponse{ServiceAccounts: serviceAccounts}, nil
-}
-
-func (s3a *S3ApiServer) CreateServiceAccount(ctx context.Context, req *iam_pb.CreateServiceAccountRequest) (*iam_pb.CreateServiceAccountResponse, error) {
-	if req.ServiceAccount == nil || req.ServiceAccount.CreatedBy == "" {
-		return nil, fmt.Errorf("service account owner is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "CreateServiceAccount")
-	values.Set("CreatedBy", req.ServiceAccount.CreatedBy)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.CreateServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) UpdateServiceAccount(ctx context.Context, req *iam_pb.UpdateServiceAccountRequest) (*iam_pb.UpdateServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "UpdateServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	if req.ServiceAccount != nil && req.ServiceAccount.Disabled {
-		values.Set("Status", "Inactive")
-	}
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.UpdateServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) DeleteServiceAccount(ctx context.Context, req *iam_pb.DeleteServiceAccountRequest) (*iam_pb.DeleteServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "DeleteServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	_, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	return &iam_pb.DeleteServiceAccountResponse{}, nil
-}
-
-func (s3a *S3ApiServer) GetServiceAccount(ctx context.Context, req *iam_pb.GetServiceAccountRequest) (*iam_pb.GetServiceAccountResponse, error) {
-	if req.Id == "" {
-		return nil, fmt.Errorf("service account id is required")
-	}
-	values := url.Values{}
-	values.Set("Action", "GetServiceAccount")
-	values.Set("ServiceAccountId", req.Id)
-	resp, err := s3a.executeAction(values)
-	if err != nil {
-		return nil, err
-	}
-	iamResp, ok := resp.(iamGetServiceAccountResponse)
-	if !ok {
-		return nil, fmt.Errorf("unexpected IAM GetServiceAccount response type %T", resp)
-	}
-
-	var serviceAccount *iam_pb.ServiceAccount
-	sa := iamResp.GetServiceAccountResult.ServiceAccount
-	serviceAccount = &iam_pb.ServiceAccount{
-		Id:          sa.ServiceAccountId,
-		ParentUser:  sa.ParentUser,
-		Description: sa.Description,
-		Credential: &iam_pb.Credential{
-			AccessKey: sa.AccessKeyId,
-			Status:    sa.Status,
-		},
-	}
-
-	return &iam_pb.GetServiceAccountResponse{
-		ServiceAccount: serviceAccount,
-	}, nil
+	return resp, nil
 }