IAM Policy Management via gRPC (#8109)

* Add IAM gRPC service definition

- Add GetConfiguration/PutConfiguration for config management
- Add CreateUser/GetUser/UpdateUser/DeleteUser/ListUsers for user management
- Add CreateAccessKey/DeleteAccessKey/GetUserByAccessKey for access key management
- Methods mirror existing IAM HTTP API functionality

* Add IAM gRPC handlers on filer server

- Implement IamGrpcServer with CredentialManager integration
- Handle configuration get/put operations
- Handle user CRUD operations
- Handle access key create/delete operations
- All methods delegate to CredentialManager for actual storage

* Wire IAM gRPC service to filer server

- Add CredentialManager field to FilerOption and FilerServer
- Import credential store implementations in filer command
- Initialize CredentialManager from credential.toml if available
- Register IAM gRPC service on filer gRPC server
- Enable credential management via gRPC alongside existing filer services

* Regenerate IAM protobuf with gRPC service methods

* iam_pb: add Policy Management to protobuf definitions

* credential: implement PolicyManager in credential stores

* filer: implement IAM Policy Management RPCs

* shell: add s3.policy command

* test: add integration test for s3.policy

* test: fix compilation errors in policy_test

* pb

* fmt

* test

* weed shell: add -policies flag to s3.configure

This allows linking/unlinking IAM policies to/from identities
directly from the s3.configure command.

* test: verify s3.configure policy linking and fix port allocation

- Added test case for linking policies to users via s3.configure
- Implemented findAvailablePortPair to ensure HTTP and gRPC ports
  are both available, avoiding conflicts with randomized port assignments.
- Updated assertion to match jsonpb output (policyNames)

* credential: add StoreTypeGrpc constant

* credential: add IAM gRPC store boilerplate

* credential: implement identity methods in gRPC store

* credential: implement policy methods in gRPC store

* admin: use gRPC credential store for AdminServer

This ensures that all IAM and policy changes made through the Admin UI
are persisted via the Filer's IAM gRPC service instead of direct file manipulation.

* shell: s3.configure use granular IAM gRPC APIs instead of full config patching

* shell: s3.configure use granular IAM gRPC APIs

* shell: replace deprecated ioutil with os in s3.policy

* filer: use gRPC FailedPrecondition for unconfigured credential manager

* test: improve s3.policy integration tests and fix error checks

* ci: add s3 policy shell integration tests to github workflow

* filer: fix LoadCredentialConfiguration error handling

* credential/grpc: propagate unmarshal errors in GetPolicies

* filer/grpc: improve error handling and validation

* shell: use gRPC status codes in s3.configure

* credential: document PutPolicy as create-or-replace

* credential/postgres: reuse CreatePolicy in PutPolicy to deduplicate logic

* shell: add timeout context and strictly enforce flags in s3.policy

* iam: standardize policy content field naming in gRPC and proto

* shell: extract slice helper functions in s3.configure

* filer: map credential store errors to gRPC status codes

* filer: add input validation for UpdateUser and CreateAccessKey

* iam: improve validation in policy and config handlers

* filer: ensure IAM service registration by defaulting credential manager

* credential: add GetStoreName method to manager

* test: verify policy deletion in integration test
This commit is contained in:
Chris Lu
2026-01-25 13:39:30 -08:00
committed by GitHub
parent 59d40f7186
commit 6bf088cec9
38 changed files with 3853 additions and 741 deletions

View File

@@ -2,16 +2,18 @@ package shell
import (
"bytes"
"context"
"flag"
"fmt"
"io"
"sort"
"strings"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func init() {
@@ -32,7 +34,7 @@ func (c *commandS3Configure) Help() string {
s3.configure
# create a new identity with account information
s3.configure -user=username -actions=Read,Write,List,Tagging -buckets=bucket-name -access_key=key -secret_key=secret -account_id=id -account_display_name=name -account_email=email@example.com -apply
s3.configure -user=username -actions=Read,Write,List,Tagging -buckets=bucket-name -policies=policy1,policy2 -access_key=key -secret_key=secret -account_id=id -account_display_name=name -account_email=email@example.com -apply
`
}
@@ -51,178 +53,229 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
accountId := s3ConfigureCommand.String("account_id", "", "specify the account id")
accountDisplayName := s3ConfigureCommand.String("account_display_name", "", "specify the account display name")
accountEmail := s3ConfigureCommand.String("account_email", "", "specify the account email address")
isDelete := s3ConfigureCommand.Bool("delete", false, "delete users, actions or access keys")
policies := s3ConfigureCommand.String("policies", "", "comma separated policy names")
isDelete := s3ConfigureCommand.Bool("delete", false, "delete users, actions, access keys or policies")
apply := s3ConfigureCommand.Bool("apply", false, "update and apply s3 configuration")
if err = s3ConfigureCommand.Parse(args); err != nil {
return nil
}
// Check which account flags were provided and build update functions
var accountUpdates []func(*iam_pb.Account)
s3ConfigureCommand.Visit(func(f *flag.Flag) {
switch f.Name {
case "account_id":
accountUpdates = append(accountUpdates, func(a *iam_pb.Account) { a.Id = *accountId })
case "account_display_name":
accountUpdates = append(accountUpdates, func(a *iam_pb.Account) { a.DisplayName = *accountDisplayName })
case "account_email":
accountUpdates = append(accountUpdates, func(a *iam_pb.Account) { a.EmailAddress = *accountEmail })
}
})
// Helper function to update account information on an identity
updateAccountInfo := func(account **iam_pb.Account) {
if len(accountUpdates) > 0 {
if *account == nil {
*account = &iam_pb.Account{}
}
for _, update := range accountUpdates {
update(*account)
}
}
// Case 1: List configuration (no user specified)
if *user == "" {
return c.listConfiguration(commandEnv, writer)
}
var buf bytes.Buffer
if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer.ReadEntry(commandEnv.MasterClient, client, filer.IamConfigDirectory, filer.IamIdentityFile, &buf)
}); err != nil && err != filer_pb.ErrNotFound {
return err
}
// Case 2: Modify specific user
var identity *iam_pb.Identity
var isNewUser bool
s3cfg := &iam_pb.S3ApiConfiguration{}
if buf.Len() > 0 {
if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil {
err = pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
// Try to get existing user
resp, getErr := client.GetUser(context.Background(), &iam_pb.GetUserRequest{
Username: *user,
})
if getErr == nil {
identity = resp.Identity
if identity == nil {
// Should not happen if err is nil, but handle defensively
isNewUser = true
identity = &iam_pb.Identity{Name: *user}
}
} else {
st, ok := status.FromError(getErr)
if ok && st.Code() == codes.NotFound {
isNewUser = true
identity = &iam_pb.Identity{
Name: *user,
Credentials: []*iam_pb.Credential{},
Actions: []string{},
PolicyNames: []string{},
}
} else {
return fmt.Errorf("failed to get user %s: %v", *user, getErr)
}
}
// Apply changes to identity object
if err := c.applyChanges(identity, isNewUser, actions, buckets, accessKey, secretKey, policies, isDelete, accountId, accountDisplayName, accountEmail); err != nil {
return err
}
}
idx := 0
changed := false
if *user != "" {
for i, identity := range s3cfg.Identities {
if *user == identity.Name {
idx = i
changed = true
break
// Print changes (Simulation)
var buf bytes.Buffer
filer.ProtoToText(&buf, identity)
fmt.Fprint(writer, buf.String())
fmt.Fprintln(writer)
if !*apply {
infoAboutSimulationMode(writer, *apply, "-apply")
return nil
}
// Apply changes
if *isDelete && *actions == "" && *accessKey == "" && *buckets == "" && *policies == "" {
// Delete User
_, err := client.DeleteUser(context.Background(), &iam_pb.DeleteUserRequest{Username: *user})
return err
} else {
// Create or Update User
if isNewUser {
_, err := client.CreateUser(context.Background(), &iam_pb.CreateUserRequest{Identity: identity})
return err
} else {
_, err := client.UpdateUser(context.Background(), &iam_pb.UpdateUserRequest{Username: *user, Identity: identity})
return err
}
}
}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
return err
}
func (c *commandS3Configure) listConfiguration(commandEnv *CommandEnv, writer io.Writer) error {
return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
resp, err := client.GetConfiguration(context.Background(), &iam_pb.GetConfigurationRequest{})
if err != nil {
return err
}
var buf bytes.Buffer
filer.ProtoToText(&buf, resp.Configuration)
fmt.Fprint(writer, buf.String())
fmt.Fprintln(writer)
return nil
}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
}
func (c *commandS3Configure) applyChanges(identity *iam_pb.Identity, isNewUser bool, actions, buckets, accessKey, secretKey, policies *string, isDelete *bool, accountId, accountDisplayName, accountEmail *string) error {
// Helper to update account info
if *accountId != "" || *accountDisplayName != "" || *accountEmail != "" {
if identity.Account == nil {
identity.Account = &iam_pb.Account{}
}
if *accountId != "" {
identity.Account.Id = *accountId
}
if *accountDisplayName != "" {
identity.Account.DisplayName = *accountDisplayName
}
if *accountEmail != "" {
identity.Account.EmailAddress = *accountEmail
}
}
// Prepare lists
var cmdActions []string
for _, action := range strings.Split(*actions, ",") {
if *buckets == "" {
cmdActions = append(cmdActions, action)
} else {
for _, bucket := range strings.Split(*buckets, ",") {
cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket))
if *actions != "" {
for _, action := range strings.Split(*actions, ",") {
if *buckets == "" {
cmdActions = append(cmdActions, action)
} else {
for _, bucket := range strings.Split(*buckets, ",") {
cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket))
}
}
}
}
if changed {
infoAboutSimulationMode(writer, *apply, "-apply")
if *isDelete {
var exists []int
for _, cmdAction := range cmdActions {
for i, currentAction := range s3cfg.Identities[idx].Actions {
if cmdAction == currentAction {
exists = append(exists, i)
}
}
}
sort.Sort(sort.Reverse(sort.IntSlice(exists)))
for _, i := range exists {
s3cfg.Identities[idx].Actions = append(
s3cfg.Identities[idx].Actions[:i],
s3cfg.Identities[idx].Actions[i+1:]...,
)
}
if *accessKey != "" {
exists = []int{}
for i, credential := range s3cfg.Identities[idx].Credentials {
if credential.AccessKey == *accessKey {
exists = append(exists, i)
}
}
sort.Sort(sort.Reverse(sort.IntSlice(exists)))
for _, i := range exists {
s3cfg.Identities[idx].Credentials = append(
s3cfg.Identities[idx].Credentials[:i],
s3cfg.Identities[idx].Credentials[i+1:]...,
)
}
var cmdPolicies []string
if *policies != "" {
for _, policy := range strings.Split(*policies, ",") {
if policy != "" {
cmdPolicies = append(cmdPolicies, policy)
}
if *actions == "" && *accessKey == "" && *buckets == "" {
s3cfg.Identities = append(s3cfg.Identities[:idx], s3cfg.Identities[idx+1:]...)
}
} else {
if *actions != "" {
for _, cmdAction := range cmdActions {
found := false
for _, action := range s3cfg.Identities[idx].Actions {
if cmdAction == action {
found = true
break
}
}
if !found {
s3cfg.Identities[idx].Actions = append(s3cfg.Identities[idx].Actions, cmdAction)
}
}
}
if *accessKey != "" && *user != "anonymous" {
found := false
for _, credential := range s3cfg.Identities[idx].Credentials {
if credential.AccessKey == *accessKey {
found = true
credential.SecretKey = *secretKey
break
}
}
if !found {
s3cfg.Identities[idx].Credentials = append(s3cfg.Identities[idx].Credentials, &iam_pb.Credential{
AccessKey: *accessKey,
SecretKey: *secretKey,
})
}
}
// Update account information if provided
updateAccountInfo(&s3cfg.Identities[idx].Account)
}
} else if *user != "" && *actions != "" {
infoAboutSimulationMode(writer, *apply, "-apply")
identity := iam_pb.Identity{
Name: *user,
Actions: cmdActions,
Credentials: []*iam_pb.Credential{},
}
if *user != "anonymous" {
identity.Credentials = append(identity.Credentials,
&iam_pb.Credential{AccessKey: *accessKey, SecretKey: *secretKey})
}
// Add account information if provided
updateAccountInfo(&identity.Account)
s3cfg.Identities = append(s3cfg.Identities, &identity)
}
if err = filer.CheckDuplicateAccessKey(s3cfg); err != nil {
return err
}
if *isDelete {
// DELETE LOGIC
buf.Reset()
filer.ProtoToText(&buf, s3cfg)
fmt.Fprint(writer, buf.String())
fmt.Fprintln(writer)
if *apply {
if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile, buf.Bytes())
}); err != nil {
return err
// Remove Actions
if len(cmdActions) > 0 {
identity.Actions = removeFromSlice(identity.Actions, cmdActions)
}
// Remove Credentials
if *accessKey != "" {
var keepCredentials []*iam_pb.Credential
for _, cred := range identity.Credentials {
if cred.AccessKey != *accessKey {
keepCredentials = append(keepCredentials, cred)
}
}
identity.Credentials = keepCredentials
}
// Remove Policies
if len(cmdPolicies) > 0 {
identity.PolicyNames = removeFromSlice(identity.PolicyNames, cmdPolicies)
}
} else {
// ADD/UPDATE LOGIC
// Add Actions
identity.Actions = addUniqueToSlice(identity.Actions, cmdActions)
// Add/Update Credentials
if *accessKey != "" && identity.Name != "anonymous" {
found := false
for _, cred := range identity.Credentials {
if cred.AccessKey == *accessKey {
found = true
if *secretKey != "" {
cred.SecretKey = *secretKey
}
break
}
}
if !found {
if *secretKey == "" {
return fmt.Errorf("secret_key is required when adding a new access_key")
}
identity.Credentials = append(identity.Credentials, &iam_pb.Credential{
AccessKey: *accessKey,
SecretKey: *secretKey,
})
}
}
// Add Policies
identity.PolicyNames = addUniqueToSlice(identity.PolicyNames, cmdPolicies)
}
return nil
}
// Helper to remove items from a slice
func removeFromSlice(current []string, toRemove []string) []string {
removeSet := make(map[string]struct{}, len(toRemove))
for _, item := range toRemove {
removeSet[item] = struct{}{}
}
var result []string
for _, item := range current {
if _, found := removeSet[item]; !found {
result = append(result, item)
}
}
return result
}
// Helper to add unique items to a slice
func addUniqueToSlice(current []string, toAdd []string) []string {
existingSet := make(map[string]struct{}, len(current))
for _, item := range current {
existingSet[item] = struct{}{}
}
for _, item := range toAdd {
if _, found := existingSet[item]; !found {
current = append(current, item)
}
}
return current
}

View File

@@ -0,0 +1,150 @@
package shell
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"os"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"google.golang.org/grpc"
)
func init() {
Commands = append(Commands, &commandS3Policy{})
}
type commandS3Policy struct {
}
func (c *commandS3Policy) Name() string {
return "s3.policy"
}
func (c *commandS3Policy) Help() string {
return `manage s3 policies
# create or update a policy
s3.policy -put -name=mypolicy -file=policy.json
# list all policies
s3.policy -list
# get a policy
s3.policy -get -name=mypolicy
# delete a policy
s3.policy -delete -name=mypolicy
`
}
func (c *commandS3Policy) HasTag(CommandTag) bool {
return false
}
func (c *commandS3Policy) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
s3PolicyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
put := s3PolicyCommand.Bool("put", false, "create or update a policy")
get := s3PolicyCommand.Bool("get", false, "get a policy")
list := s3PolicyCommand.Bool("list", false, "list all policies")
del := s3PolicyCommand.Bool("delete", false, "delete a policy")
name := s3PolicyCommand.String("name", "", "policy name")
file := s3PolicyCommand.String("file", "", "policy file (json)")
if err = s3PolicyCommand.Parse(args); err != nil {
return err
}
actionCount := 0
for _, v := range []bool{*put, *get, *list, *del} {
if v {
actionCount++
}
}
if actionCount == 0 {
return fmt.Errorf("one of -put, -get, -list, -delete must be specified")
}
if actionCount > 1 {
return fmt.Errorf("only one of -put, -get, -list, -delete can be specified")
}
return pb.WithGrpcClient(false, 0, func(conn *grpc.ClientConn) error {
client := iam_pb.NewSeaweedIdentityAccessManagementClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if *put {
if *name == "" {
return fmt.Errorf("-name is required")
}
if *file == "" {
return fmt.Errorf("-file is required")
}
data, err := os.ReadFile(*file)
if err != nil {
return fmt.Errorf("failed to read policy file: %v", err)
}
// Validate JSON
var policy policy_engine.PolicyDocument
if err := json.Unmarshal(data, &policy); err != nil {
return fmt.Errorf("invalid policy json: %v", err)
}
_, err = client.PutPolicy(ctx, &iam_pb.PutPolicyRequest{
Name: *name,
Content: string(data),
})
return err
}
if *get {
if *name == "" {
return fmt.Errorf("-name is required")
}
resp, err := client.GetPolicy(ctx, &iam_pb.GetPolicyRequest{
Name: *name,
})
if err != nil {
return err
}
if resp.Content == "" {
return fmt.Errorf("policy not found")
}
fmt.Fprintf(writer, "%s\n", resp.Content)
return nil
}
if *list {
resp, err := client.ListPolicies(ctx, &iam_pb.ListPoliciesRequest{})
if err != nil {
return err
}
for _, policy := range resp.Policies {
fmt.Fprintf(writer, "Name: %s\n", policy.Name)
fmt.Fprintf(writer, "Content: %s\n", policy.Content)
fmt.Fprintf(writer, "---\n")
}
return nil
}
if *del {
if *name == "" {
return fmt.Errorf("-name is required")
}
_, err := client.DeletePolicy(ctx, &iam_pb.DeletePolicyRequest{
Name: *name,
})
return err
}
return nil
}, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption)
}