From d1823d3784bc4d9ab6bd715c477825169a5468b0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 3 Apr 2026 20:01:28 -0700 Subject: [PATCH] fix(s3): include static identities in listing operations (#8903) * fix(s3): include static identities in listing operations Static identities loaded from -s3.config file were only stored in the S3 API server's in-memory state. Listing operations (s3.configure shell command, aws iam list-users) queried the credential manager which only returned dynamic identities from the backend store. Register static identities with the credential manager after loading so they are included in LoadConfiguration and ListUsers results, and filtered out before SaveConfiguration to avoid persisting them to the dynamic store. Fixes https://github.com/seaweedfs/seaweedfs/discussions/8896 * fix: avoid mutating caller's config and defensive copies - SaveConfiguration: use shallow struct copy instead of mutating the caller's config.Identities field - SetStaticIdentities: skip nil entries to avoid panics - GetStaticIdentities: defensively copy PolicyNames slice to avoid aliasing the original * fix: filter nil static identities and sync on config reload - SetStaticIdentities: filter nil entries from the stored slice (not just from staticNames) to prevent panics in LoadConfiguration/ListUsers - Extract updateCredentialManagerStaticIdentities helper and call it from both startup and the grace.OnReload handler so the credential manager's static snapshot stays current after config file reloads * fix: add mutex for static identity fields and fix ListUsers for store callers - Add sync.RWMutex to protect staticIdentities/staticNames against concurrent reads during config reload - Revert CredentialManager.ListUsers to return only store users, since internal callers (e.g. DeletePolicy) look up each user in the store and fail on non-existent static entries - Merge static usernames in the filer gRPC ListUsers handler instead, via the new GetStaticUsernames method - Fix CI: TestIAMPolicyManagement/managed_policy_crud_lifecycle was failing because DeletePolicy iterated static users that don't exist in the store * fix: show static identities in admin UI and weed shell The admin UI and weed shell s3.configure command query the filer's credential manager via gRPC, which is a separate instance from the S3 server's credential manager. Static identities were only registered on the S3 server's credential manager, so they never appeared in the filer's responses. - Add CredentialManager.LoadS3ConfigFile to parse a static S3 config file and register its identities - Add FilerOptions.s3ConfigFile so the filer can load the same static config that the S3 server uses - Wire s3ConfigFile through in weed mini and weed server modes - Merge static usernames in filer gRPC ListUsers handler - Add CredentialManager.GetStaticUsernames helper - Add sync.RWMutex to protect concurrent access to static identity fields - Avoid importing weed/filer from weed/credential (which pulled in filer store init() registrations and broke test isolation) - Add docker/compose/s3_static_users_example.json * fix(admin): make static users read-only in admin UI Static users loaded from the -s3.config file should not be editable or deletable through the admin UI since they are managed via the config file. - Add IsStatic field to ObjectStoreUser, set from credential manager - Hide edit, delete, and access key buttons for static users in the users table template - Show a "static" badge next to static user names - Return 403 Forbidden from UpdateUser and DeleteUser API handlers when the target user is a static identity * fix(admin): show details for static users GetObjectStoreUserDetails called credentialManager.GetUser which only queries the dynamic store. For static users this returned ErrUserNotFound. Fall back to GetStaticIdentity when the store lookup fails. * fix(admin): load static S3 identities in admin server The admin server has its own credential manager (gRPC store) which is a separate instance from the S3 server's and filer's. It had no static identity data, so IsStaticIdentity returned false (edit/delete buttons shown) and GetStaticIdentity returned nil (details page failed). Pass the -s3.config file path through to the admin server and call LoadS3ConfigFile on its credential manager, matching the approach used for the filer. * fix: use protobuf is_static field instead of passing config file path The previous approach passed -s3.config file path to every component (filer, admin). This is wrong because the admin server should not need to know about S3 config files. Instead, add an is_static field to the Identity protobuf message. The field is set when static identities are serialized (in GetStaticIdentities and LoadS3ConfigFile). Any gRPC client that loads configuration via GetConfiguration automatically sees which identities are static, without needing the config file. - Add is_static field (tag 8) to iam_pb.Identity proto message - Set IsStatic=true in GetStaticIdentities and LoadS3ConfigFile - Admin GetObjectStoreUsers reads identity.IsStatic from proto - Admin IsStaticUser helper loads config via gRPC to check the flag - Filer GetUser gRPC handler falls back to GetStaticIdentity - Remove s3ConfigFile from AdminOptions and NewAdminServer signature --- docker/compose/s3_static_users_example.json | 53 +++++++ weed/admin/dash/admin_data.go | 1 + weed/admin/dash/admin_server.go | 19 +++ weed/admin/dash/user_management.go | 2 +- weed/admin/handlers/user_handlers.go | 10 ++ weed/admin/view/app/object_store_users.templ | 27 ++-- .../view/app/object_store_users_templ.go | 116 ++++++++------- weed/command/filer.go | 10 ++ weed/command/mini.go | 4 + weed/command/server.go | 2 + weed/credential/credential_manager.go | 132 +++++++++++++++++- weed/pb/iam.proto | 1 + weed/pb/iam_pb/iam.pb.go | 13 +- weed/s3api/auth_credentials.go | 57 ++++++++ weed/s3api/s3api_embedded_iam.go | 2 + weed/s3api/s3api_server.go | 1 + weed/server/filer_server.go | 1 + weed/server/filer_server_handlers_iam_grpc.go | 18 +++ 18 files changed, 404 insertions(+), 65 deletions(-) create mode 100644 docker/compose/s3_static_users_example.json diff --git a/docker/compose/s3_static_users_example.json b/docker/compose/s3_static_users_example.json new file mode 100644 index 000000000..d8d87340b --- /dev/null +++ b/docker/compose/s3_static_users_example.json @@ -0,0 +1,53 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "AKIAIOSFODNN7EXAMPLE", + "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + ], + "actions": [ + "Admin", + "Read", + "List", + "Tagging", + "Write" + ] + }, + { + "name": "steward", + "credentials": [ + { + "accessKey": "steward-key", + "secretKey": "steward-secret" + } + ], + "actions": [ + "Read", + "List", + "Write" + ] + }, + { + "name": "le001", + "credentials": [ + { + "accessKey": "le001-key", + "secretKey": "le001-secret" + } + ], + "actions": [ + "Read", + "List" + ] + }, + { + "name": "anonymous", + "actions": [ + "Read" + ] + } + ] +} diff --git a/weed/admin/dash/admin_data.go b/weed/admin/dash/admin_data.go index f91fc7084..bddaae65d 100644 --- a/weed/admin/dash/admin_data.go +++ b/weed/admin/dash/admin_data.go @@ -45,6 +45,7 @@ type ObjectStoreUser struct { SecretKey string `json:"secret_key"` Permissions []string `json:"permissions"` PolicyNames []string `json:"policy_names"` + IsStatic bool `json:"is_static"` // loaded from static config file, not editable } type ObjectStoreUsersData struct { diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index ed59b22f4..a339e4d39 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -867,6 +867,24 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error { }) } +// IsStaticUser checks if a user is a static identity by loading the +// configuration from the credential manager and checking the IsStatic flag. +func (s *AdminServer) IsStaticUser(username string) bool { + if s.credentialManager == nil { + return false + } + s3cfg, err := s.credentialManager.LoadConfiguration(context.Background()) + if err != nil { + return false + } + for _, ident := range s3cfg.Identities { + if ident.Name == username { + return ident.IsStatic + } + } + return false +} + // GetObjectStoreUsers retrieves object store users from identity.json func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) { if s.credentialManager == nil { @@ -890,6 +908,7 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse user := ObjectStoreUser{ Username: identity.Name, Permissions: identity.Actions, + IsStatic: identity.IsStatic, } // Set email from account if available diff --git a/weed/admin/dash/user_management.go b/weed/admin/dash/user_management.go index 7335d70cb..009f5cfca 100644 --- a/weed/admin/dash/user_management.go +++ b/weed/admin/dash/user_management.go @@ -175,7 +175,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails, ctx := context.Background() - // Get user using credential manager + // Get user using credential manager (resolves static users via filer gRPC) identity, err := s.credentialManager.GetUser(ctx, username) if err != nil { if err == credential.ErrUserNotFound { diff --git a/weed/admin/handlers/user_handlers.go b/weed/admin/handlers/user_handlers.go index dbace9f2c..e439b3ad8 100644 --- a/weed/admin/handlers/user_handlers.go +++ b/weed/admin/handlers/user_handlers.go @@ -93,6 +93,11 @@ func (h *UserHandlers) UpdateUser(w http.ResponseWriter, r *http.Request) { return } + if h.adminServer.IsStaticUser(username) { + writeJSONError(w, http.StatusForbidden, "Cannot modify static user "+username+" (loaded from config file)") + return + } + var req dash.UpdateUserRequest if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil { writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error()) @@ -120,6 +125,11 @@ func (h *UserHandlers) DeleteUser(w http.ResponseWriter, r *http.Request) { return } + if h.adminServer.IsStaticUser(username) { + writeJSONError(w, http.StatusForbidden, "Cannot delete static user "+username+" (loaded from config file)") + return + } + err := h.adminServer.DeleteObjectStoreUser(username) if err != nil { glog.Errorf("Failed to delete user %s: %v", username, err) diff --git a/weed/admin/view/app/object_store_users.templ b/weed/admin/view/app/object_store_users.templ index 0bc6f2ab3..cc16a3e53 100644 --- a/weed/admin/view/app/object_store_users.templ +++ b/weed/admin/view/app/object_store_users.templ @@ -125,6 +125,9 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
{user.Username} + if user.IsStatic { + static + }
{user.Email} @@ -133,24 +136,28 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
- - - if user.Username != "anonymous" { + if !user.IsStatic { + + } + if user.Username != "anonymous" && !user.IsStatic { } - + if !user.IsStatic { + + }
diff --git a/weed/admin/view/app/object_store_users_templ.go b/weed/admin/view/app/object_store_users_templ.go index 91a212b6e..2917d2f3f 100644 --- a/weed/admin/view/app/object_store_users_templ.go +++ b/weed/admin/view/app/object_store_users_templ.go @@ -41,7 +41,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component { var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalUsers)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 38, Col: 71} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 38, Col: 71} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) if templ_7745c5c3_Err != nil { @@ -54,7 +54,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Users))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 58, Col: 71} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 58, Col: 71} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { @@ -67,7 +67,7 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("15:04")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 78, Col: 69} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 78, Col: 69} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) if templ_7745c5c3_Err != nil { @@ -85,140 +85,162 @@ func ObjectStoreUsers(data dash.ObjectStoreUsersData) templ.Component { var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 127, Col: 74} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 127, Col: 74} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if user.IsStatic { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "static") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(user.Email) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 130, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 133, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(user.AccessKey) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 132, Col: 88} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 135, Col: 88} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 141, Col: 113} + if !user.IsStatic { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "\"> ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if user.Username != "anonymous" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "\"> ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } - var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(user.Username) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 151, Col: 115} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "\">
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } if len(data.Users) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "
No users found

Create your first object store user to get started.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "
No users found

Create your first object store user to get started.

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
Last updated: ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
Last updated: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var12 string templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/object_store_users.templ`, Line: 182, Col: 81} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/object_store_users.templ`, Line: 189, Col: 81} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
Create New User
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
Create New User
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if !data.HasAnonymousUser { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
Hold Ctrl/Cmd to select multiple permissions
Apply selected permissions to specific buckets or all buckets
Hold Ctrl/Cmd to select multiple buckets
Hold Ctrl/Cmd to select multiple policies
Edit User
Apply selected permissions to specific buckets or all buckets
Hold Ctrl/Cmd to select multiple buckets
User Details
Manage Access Keys
Access Keys for

Leave blank to auto-generate.

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
Hold Ctrl/Cmd to select multiple permissions
Apply selected permissions to specific buckets or all buckets
Hold Ctrl/Cmd to select multiple buckets
Hold Ctrl/Cmd to select multiple policies
Edit User
Apply selected permissions to specific buckets or all buckets
Hold Ctrl/Cmd to select multiple buckets
User Details
Manage Access Keys
Access Keys for

Leave blank to auto-generate.

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/command/filer.go b/weed/command/filer.go index edfd8afef..75d3ccfd4 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -81,6 +81,7 @@ type FilerOptions struct { exposeDirectoryData *bool tusBasePath *string certProvider certprovider.Provider + s3ConfigFile *string // optional path to static S3 identity config } func init() { @@ -343,6 +344,15 @@ func (fo *FilerOptions) startFiler() { glog.V(0).Infof("Initialized credential manager: %s", credentialManager.GetStoreName()) } + // Load static S3 identities from config file if specified + if fo.s3ConfigFile != nil && *fo.s3ConfigFile != "" { + if credentialManager != nil { + if err := credentialManager.LoadS3ConfigFile(*fo.s3ConfigFile); err != nil { + glog.Warningf("Failed to load S3 config file for static identities: %v", err) + } + } + } + fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{ Masters: fo.masters, FilerGroup: *fo.filerGroup, diff --git a/weed/command/mini.go b/weed/command/mini.go index 38257bdff..f23797b5e 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -820,6 +820,10 @@ func runMini(cmd *Command, args []string) bool { miniFilerOptions.disableHttp = miniDisableHttp miniMasterOptions.disableHttp = miniDisableHttp + // Share the S3 static identity config file with the filer so its + // credential manager can also serve static users. + miniFilerOptions.s3ConfigFile = miniS3Config + filerAddress := string(pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)) miniS3Options.filer = &filerAddress miniWebDavOptions.filer = &filerAddress diff --git a/weed/command/server.go b/weed/command/server.go index deb1c5e71..9533c967e 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -372,6 +372,8 @@ func runServer(cmd *Command, args []string) bool { } else if *serverIamConfig != "" && *s3Options.iamConfig != *serverIamConfig { glog.V(0).Infof("both -s3.iam.config(%s) and -iam.config(%s) provided; using -s3.iam.config", *s3Options.iamConfig, *serverIamConfig) } + // Share the S3 static identity config file with the filer + filerOptions.s3ConfigFile = s3Options.config go func() { time.Sleep(2 * time.Second) s3Options.localFilerSocket = filerOptions.localSocket diff --git a/weed/credential/credential_manager.go b/weed/credential/credential_manager.go index d9df17e50..a53015c2a 100644 --- a/weed/credential/credential_manager.go +++ b/weed/credential/credential_manager.go @@ -3,14 +3,18 @@ package credential import ( "context" "fmt" + "os" "strings" + "sync" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" "google.golang.org/grpc" + "google.golang.org/protobuf/encoding/protojson" ) // FilerAddressSetter is an interface for credential stores that need a dynamic filer address @@ -21,6 +25,15 @@ type FilerAddressSetter interface { // CredentialManager manages user credentials using a configurable store type CredentialManager struct { Store CredentialStore + // staticMu protects staticIdentities and staticNames, which are written + // by SetStaticIdentities (startup + config reload) and read concurrently + // by LoadConfiguration, SaveConfiguration, GetStaticUsernames, and IsStaticIdentity. + staticMu sync.RWMutex + // staticIdentities holds identities loaded from a static config file (-s3.config). + // These are included in LoadConfiguration so that listing operations + // return all configured identities, not just dynamic ones from the store. + staticIdentities []*iam_pb.Identity + staticNames map[string]bool } // NewCredentialManager creates a new credential manager with the specified store @@ -74,13 +87,97 @@ func (cm *CredentialManager) GetStoreName() string { return "" } -// LoadConfiguration loads the S3 API configuration -func (cm *CredentialManager) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { - return cm.Store.LoadConfiguration(ctx) +// SetStaticIdentities registers identities loaded from a static config file. +// These identities are included in LoadConfiguration and ListUsers results +// but are never persisted to the dynamic store. +func (cm *CredentialManager) SetStaticIdentities(identities []*iam_pb.Identity) { + filtered := make([]*iam_pb.Identity, 0, len(identities)) + names := make(map[string]bool, len(identities)) + for _, ident := range identities { + if ident != nil { + filtered = append(filtered, ident) + names[ident.Name] = true + } + } + cm.staticMu.Lock() + cm.staticIdentities = filtered + cm.staticNames = names + cm.staticMu.Unlock() } -// SaveConfiguration saves the S3 API configuration +// IsStaticIdentity returns true if the named identity was loaded from static config. +func (cm *CredentialManager) IsStaticIdentity(name string) bool { + cm.staticMu.RLock() + defer cm.staticMu.RUnlock() + return cm.staticNames[name] +} + +// GetStaticIdentity returns the protobuf identity for a static user, or nil. +func (cm *CredentialManager) GetStaticIdentity(name string) *iam_pb.Identity { + cm.staticMu.RLock() + defer cm.staticMu.RUnlock() + for _, ident := range cm.staticIdentities { + if ident.Name == name { + return ident + } + } + return nil +} + +// GetStaticUsernames returns the names of all static identities. +func (cm *CredentialManager) GetStaticUsernames() []string { + cm.staticMu.RLock() + defer cm.staticMu.RUnlock() + names := make([]string, 0, len(cm.staticIdentities)) + for _, ident := range cm.staticIdentities { + names = append(names, ident.Name) + } + return names +} + +// LoadConfiguration loads the S3 API configuration from the store and merges +// in any static identities so that listing operations show all users. +func (cm *CredentialManager) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) { + config, err := cm.Store.LoadConfiguration(ctx) + if err != nil { + return config, err + } + // Merge static identities that are not already in the dynamic config + cm.staticMu.RLock() + staticIdents := cm.staticIdentities + cm.staticMu.RUnlock() + if len(staticIdents) > 0 { + dynamicNames := make(map[string]bool, len(config.Identities)) + for _, ident := range config.Identities { + dynamicNames[ident.Name] = true + } + for _, si := range staticIdents { + if !dynamicNames[si.Name] { + config.Identities = append(config.Identities, si) + } + } + } + return config, nil +} + +// SaveConfiguration saves the S3 API configuration. +// Static identities are filtered out before saving to the store. +// The caller's config is not mutated. func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error { + cm.staticMu.RLock() + staticNames := cm.staticNames + cm.staticMu.RUnlock() + if len(staticNames) > 0 { + var dynamicOnly []*iam_pb.Identity + for _, ident := range config.Identities { + if !staticNames[ident.Name] { + dynamicOnly = append(dynamicOnly, ident) + } + } + configCopy := *config + configCopy.Identities = dynamicOnly + return cm.Store.SaveConfiguration(ctx, &configCopy) + } return cm.Store.SaveConfiguration(ctx, config) } @@ -104,7 +201,12 @@ func (cm *CredentialManager) DeleteUser(ctx context.Context, username string) er return cm.Store.DeleteUser(ctx, username) } -// ListUsers returns all usernames +// ListUsers returns usernames from the dynamic store via cm.Store.ListUsers. +// On store error the error is returned directly without merging static entries. +// Static identities (cm.staticIdentities) are NOT included here because +// internal callers (e.g. DeletePolicy) look up each user in the store and +// would fail on non-existent static entries. External callers that need the +// full list should merge GetStaticUsernames separately. func (cm *CredentialManager) ListUsers(ctx context.Context) ([]string, error) { return cm.Store.ListUsers(ctx) } @@ -169,6 +271,26 @@ func (cm *CredentialManager) UpdatePolicy(ctx context.Context, name string, docu return cm.Store.PutPolicy(ctx, name, document) } +// LoadS3ConfigFile reads a static S3 identity config file and registers +// the identities so they appear in LoadConfiguration and listing results. +func (cm *CredentialManager) LoadS3ConfigFile(path string) error { + content, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + config := &iam_pb.S3ApiConfiguration{} + opts := protojson.UnmarshalOptions{DiscardUnknown: true, AllowPartial: true} + if err := opts.Unmarshal(content, config); err != nil { + return fmt.Errorf("parse %s: %w", path, err) + } + for _, ident := range config.Identities { + ident.IsStatic = true + } + cm.SetStaticIdentities(config.Identities) + glog.V(1).Infof("Loaded %d static identities from %s", len(config.Identities), path) + return nil +} + // Shutdown performs cleanup func (cm *CredentialManager) Shutdown() { if cm.Store != nil { diff --git a/weed/pb/iam.proto b/weed/pb/iam.proto index 05b575ce8..a37b8d8c7 100644 --- a/weed/pb/iam.proto +++ b/weed/pb/iam.proto @@ -186,6 +186,7 @@ message Identity { bool disabled = 5; // User status: false = enabled (default), true = disabled repeated string service_account_ids = 6; // IDs of service accounts owned by this user repeated string policy_names = 7; + bool is_static = 8; // Loaded from static config file (read-only, not editable via API) } message Credential { diff --git a/weed/pb/iam_pb/iam.pb.go b/weed/pb/iam_pb/iam.pb.go index 9cbf29ea8..252742b89 100644 --- a/weed/pb/iam_pb/iam.pb.go +++ b/weed/pb/iam_pb/iam.pb.go @@ -1399,6 +1399,7 @@ type Identity struct { Disabled bool `protobuf:"varint,5,opt,name=disabled,proto3" json:"disabled,omitempty"` // User status: false = enabled (default), true = disabled ServiceAccountIds []string `protobuf:"bytes,6,rep,name=service_account_ids,json=serviceAccountIds,proto3" json:"service_account_ids,omitempty"` // IDs of service accounts owned by this user PolicyNames []string `protobuf:"bytes,7,rep,name=policy_names,json=policyNames,proto3" json:"policy_names,omitempty"` + IsStatic bool `protobuf:"varint,8,opt,name=is_static,json=isStatic,proto3" json:"is_static,omitempty"` // Loaded from static config file (read-only, not editable via API) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1482,6 +1483,13 @@ func (x *Identity) GetPolicyNames() []string { return nil } +func (x *Identity) GetIsStatic() bool { + if x != nil { + return x.IsStatic + } + return false +} + type Credential struct { state protoimpl.MessageState `protogen:"open.v1"` AccessKey string `protobuf:"bytes,1,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` @@ -3013,7 +3021,7 @@ const file_iam_proto_rawDesc = "" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" + "\amembers\x18\x02 \x03(\tR\amembers\x12!\n" + "\fpolicy_names\x18\x03 \x03(\tR\vpolicyNames\x12\x1a\n" + - "\bdisabled\x18\x04 \x01(\bR\bdisabled\"\x88\x02\n" + + "\bdisabled\x18\x04 \x01(\bR\bdisabled\"\xa5\x02\n" + "\bIdentity\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x124\n" + "\vcredentials\x18\x02 \x03(\v2\x12.iam_pb.CredentialR\vcredentials\x12\x18\n" + @@ -3021,7 +3029,8 @@ const file_iam_proto_rawDesc = "" + "\aaccount\x18\x04 \x01(\v2\x0f.iam_pb.AccountR\aaccount\x12\x1a\n" + "\bdisabled\x18\x05 \x01(\bR\bdisabled\x12.\n" + "\x13service_account_ids\x18\x06 \x03(\tR\x11serviceAccountIds\x12!\n" + - "\fpolicy_names\x18\a \x03(\tR\vpolicyNames\"b\n" + + "\fpolicy_names\x18\a \x03(\tR\vpolicyNames\x12\x1b\n" + + "\tis_static\x18\b \x01(\bR\bisStatic\"b\n" + "\n" + "Credential\x12\x1d\n" + "\n" + diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index ec950cbab..cc0670c02 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -295,6 +295,10 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient // This serves as an in-memory "static" configuration iam.loadEnvironmentVariableCredentials() + // Update credential manager with all static identities (file + env vars) + // so that listing operations via filer gRPC also include them. + iam.updateCredentialManagerStaticIdentities() + // Determine whether to enable S3 authentication based on configuration // For "weed mini" without any S3 config, default to allowing all access (isAuthEnabled = false) // If any credentials are configured (via file, filer, or env vars), enable authentication @@ -1069,6 +1073,59 @@ func (iam *IdentityAccessManagement) IsStaticIdentity(identityName string) bool return iam.staticIdentityNames[identityName] } +// updateCredentialManagerStaticIdentities syncs the current set of static +// identities to the credential manager. Call this after any operation that +// changes static identities (startup, config file reload, etc.). +func (iam *IdentityAccessManagement) updateCredentialManagerStaticIdentities() { + if iam.credentialManager != nil { + iam.credentialManager.SetStaticIdentities(iam.GetStaticIdentities()) + } +} + +// GetStaticIdentities returns protobuf representations of all static identities. +// This is used to include static identities in listing operations (ListUsers, etc.) +func (iam *IdentityAccessManagement) GetStaticIdentities() []*iam_pb.Identity { + iam.m.RLock() + defer iam.m.RUnlock() + + var result []*iam_pb.Identity + for _, ident := range iam.identities { + if !ident.IsStatic { + continue + } + var policyNames []string + if len(ident.PolicyNames) > 0 { + policyNames = make([]string, len(ident.PolicyNames)) + copy(policyNames, ident.PolicyNames) + } + pbIdent := &iam_pb.Identity{ + Name: ident.Name, + Disabled: ident.Disabled, + PolicyNames: policyNames, + IsStatic: true, + } + for _, action := range ident.Actions { + pbIdent.Actions = append(pbIdent.Actions, string(action)) + } + for _, cred := range ident.Credentials { + pbIdent.Credentials = append(pbIdent.Credentials, &iam_pb.Credential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + Status: cred.Status, + }) + } + if ident.Account != nil { + pbIdent.Account = &iam_pb.Account{ + Id: ident.Account.Id, + DisplayName: ident.Account.DisplayName, + EmailAddress: ident.Account.EmailAddress, + } + } + result = append(result, pbIdent) + } + return result +} + func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { iam.m.RLock() defer iam.m.RUnlock() diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index 8a817e8ab..4a367a1fb 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -215,6 +215,7 @@ func (e *EmbeddedIamApi) writeIamErrorResponse(w http.ResponseWriter, r *http.Re } // GetS3ApiConfiguration loads the S3 API configuration from the credential manager. +// The credential manager automatically includes static identities in the result. func (e *EmbeddedIamApi) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { if e.getS3ApiConfigurationFunc != nil { return e.getS3ApiConfigurationFunc(s3cfg) @@ -228,6 +229,7 @@ func (e *EmbeddedIamApi) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) } // PutS3ApiConfiguration saves the S3 API configuration to the credential manager. +// The credential manager automatically filters out static identities before saving. func (e *EmbeddedIamApi) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { if e.putS3ApiConfigurationFunc != nil { return e.putS3ApiConfigurationFunc(s3cfg) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index d6a1c1437..1596b247f 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -276,6 +276,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl glog.Errorf("fail to load config file %s: %v", option.Config, err) } else { glog.V(1).Infof("Loaded %d identities from config file %s", len(s3ApiServer.iam.identities), option.Config) + s3ApiServer.iam.updateCredentialManagerStaticIdentities() } }) } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 809c2737e..91564bb22 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -82,6 +82,7 @@ type FilerOption struct { AllowedOrigins []string ExposeDirectoryData bool TusBasePath string + S3ConfigFile string // optional path to static S3 identity config file CredentialManager *credential.CredentialManager } diff --git a/weed/server/filer_server_handlers_iam_grpc.go b/weed/server/filer_server_handlers_iam_grpc.go index c77cf8914..2a97cfd68 100644 --- a/weed/server/filer_server_handlers_iam_grpc.go +++ b/weed/server/filer_server_handlers_iam_grpc.go @@ -101,6 +101,10 @@ func (s *IamGrpcServer) GetUser(ctx context.Context, req *iam_pb.GetUserRequest) identity, err := s.credentialManager.GetUser(ctx, req.Username) if err != nil { if err == credential.ErrUserNotFound { + // Fall back to static identities (loaded from -s3.config file) + if si := s.credentialManager.GetStaticIdentity(req.Username); si != nil { + return &iam_pb.GetUserResponse{Identity: si}, nil + } return nil, status.Errorf(codes.NotFound, "user %s not found", req.Username) } glog.Errorf("Failed to get user %s: %v", req.Username, err) @@ -166,6 +170,20 @@ func (s *IamGrpcServer) ListUsers(ctx context.Context, req *iam_pb.ListUsersRequ return nil, err } + // Merge static identities (from -s3.config file) into the result + staticNames := s.credentialManager.GetStaticUsernames() + if len(staticNames) > 0 { + dynamicSet := make(map[string]bool, len(usernames)) + for _, name := range usernames { + dynamicSet[name] = true + } + for _, name := range staticNames { + if !dynamicSet[name] { + usernames = append(usernames, name) + } + } + } + return &iam_pb.ListUsersResponse{ Usernames: usernames, }, nil