gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: IAM authentication with AWS Signature V4 and environment credentials (#8099)

Browse Source 
* fix: IAM authentication with AWS Signature V4 and environment credentials

Three key fixes for authenticated IAM requests to work:

1. Fix request body consumption before signature verification
   - iamMatcher was calling r.ParseForm() which consumed POST body
   - This broke AWS Signature V4 verification on subsequent reads
   - Now only check query string in matcher, preserving body for verification
   - File: weed/s3api/s3api_server.go

2. Preserve environment variable credentials across config reloads
   - After IAM mutations, config reload overwrote env var credentials
   - Extract env var loading into loadEnvironmentVariableCredentials()
   - Call after every config reload to persist credentials
   - File: weed/s3api/auth_credentials.go

3. Add authenticated IAM tests and test infrastructure
   - New TestIAMAuthenticated suite with AWS SDK + Signature V4
   - Dynamic port allocation for independent test execution
   - Flag reset to prevent state leakage between tests
   - CI workflow to run S3 and IAM tests separately
   - Files: test/s3/example/*, .github/workflows/s3-example-integration-tests.yml

All tests pass:
- TestIAMCreateUser (unauthenticated)
- TestIAMAuthenticated (with AWS Signature V4)
- S3 integration tests

* fmt

* chore: rename test/s3/example to test/s3/normal

* simplify: CI runs all integration tests in single job

* Update s3-example-integration-tests.yml

* ci: run each test group separately to avoid raft registry conflicts
This commit is contained in:
Chris Lu
2026-01-23 16:27:42 -08:00
committed by GitHub
parent afbe52f262
commit d664ca5ed3
5 changed files with 812 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
weed/s3api/auth_credentials.go
View File

@@ -201,83 +201,7 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto
// Check for AWS environment variables and merge them if present
// This serves as an in-memory "static" configuration
accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKeyId != "" && secretAccessKey != "" {
// Create environment variable identity name
identityNameSuffix := accessKeyId
if len(accessKeyId) > 8 {
identityNameSuffix = accessKeyId[:8]
}
identityName := "admin-" + identityNameSuffix
// Create admin identity with environment variable credentials
envIdentity := &Identity{
Name: identityName,
Account: &AccountAdmin,
Credentials: []*Credential{
{
AccessKey: accessKeyId,
SecretKey: secretAccessKey,
},
},
Actions: []Action{
s3_constants.ACTION_ADMIN,
},
}
iam.m.Lock()
// Initialize maps if they are nil (if no config loaded yet)
if iam.staticIdentityNames == nil {
iam.staticIdentityNames = make(map[string]bool)
}
// Check if identity already exists (avoid duplicates)
exists := false
for _, ident := range iam.identities {
if ident.Name == identityName {
exists = true
break
}
}
if !exists {
glog.V(1).Infof("Added admin identity from AWS environment variables: %s", envIdentity.Name)
// Add to identities list
iam.identities = append(iam.identities, envIdentity)
// Update credential mappings
if iam.accessKeyIdent == nil {
iam.accessKeyIdent = make(map[string]*Identity)
}
iam.accessKeyIdent[accessKeyId] = envIdentity
if iam.nameToIdentity == nil {
iam.nameToIdentity = make(map[string]*Identity)
}
iam.nameToIdentity[envIdentity.Name] = envIdentity
// Treat env var identity as static (immutable)
iam.staticIdentityNames[envIdentity.Name] = true
// Ensure defaults exist if this is the first identity
if iam.accounts == nil {
iam.accounts = make(map[string]*Account)
iam.accounts[AccountAdmin.Id] = &AccountAdmin
iam.accounts[AccountAnonymous.Id] = &AccountAnonymous
}
if iam.emailAccount == nil {
iam.emailAccount = make(map[string]*Account)
iam.emailAccount[AccountAdmin.EmailAddress] = &AccountAdmin
iam.emailAccount[AccountAnonymous.EmailAddress] = &AccountAnonymous
}
}
iam.m.Unlock()
}
iam.loadEnvironmentVariableCredentials()
// Determine whether to enable S3 authentication based on configuration
// For "weed mini" without any S3 config, default to allowing all access (isAuthEnabled = false)
@@ -303,6 +227,90 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto
return iam
}
// loadEnvironmentVariableCredentials loads AWS credentials from environment variables
// and adds them as a static admin identity. This function is idempotent and can be
// called multiple times (e.g., after configuration reloads).
func (iam *IdentityAccessManagement) loadEnvironmentVariableCredentials() {
accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
if accessKeyId == "" || secretAccessKey == "" {
return
}
// Create environment variable identity name
identityNameSuffix := accessKeyId
if len(accessKeyId) > 8 {
identityNameSuffix = accessKeyId[:8]
}
identityName := "admin-" + identityNameSuffix
// Create admin identity with environment variable credentials
envIdentity := &Identity{
Name: identityName,
Account: &AccountAdmin,
Credentials: []*Credential{
{
AccessKey: accessKeyId,
SecretKey: secretAccessKey,
},
},
Actions: []Action{
s3_constants.ACTION_ADMIN,
},
}
iam.m.Lock()
defer iam.m.Unlock()
// Initialize maps if they are nil
if iam.staticIdentityNames == nil {
iam.staticIdentityNames = make(map[string]bool)
}
if iam.accessKeyIdent == nil {
iam.accessKeyIdent = make(map[string]*Identity)
}
if iam.nameToIdentity == nil {
iam.nameToIdentity = make(map[string]*Identity)
}
// Check if identity already exists (avoid duplicates)
exists := false
for _, ident := range iam.identities {
if ident.Name == identityName {
exists = true
break
}
}
if !exists {
glog.Infof("Added admin identity from AWS environment variables: name=%s, accessKey=%s", envIdentity.Name, accessKeyId)
// Add to identities list
iam.identities = append(iam.identities, envIdentity)
// Update credential mappings
iam.accessKeyIdent[accessKeyId] = envIdentity
iam.nameToIdentity[envIdentity.Name] = envIdentity
// Treat env var identity as static (immutable)
iam.staticIdentityNames[envIdentity.Name] = true
// Ensure defaults exist
if iam.accounts == nil {
iam.accounts = make(map[string]*Account)
}
iam.accounts[AccountAdmin.Id] = &AccountAdmin
iam.accounts[AccountAnonymous.Id] = &AccountAnonymous
if iam.emailAccount == nil {
iam.emailAccount = make(map[string]*Account)
}
iam.emailAccount[AccountAdmin.EmailAddress] = &AccountAdmin
iam.emailAccount[AccountAnonymous.EmailAddress] = &AccountAnonymous
}
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
return iam.LoadS3ApiConfigurationFromCredentialManager()
}
@@ -486,15 +494,21 @@ func (iam *IdentityAccessManagement) replaceS3ApiConfiguration(config *iam_pb.S3
glog.V(1).Infof("S3 authentication enabled - credentials were added dynamically")
}
// Log configuration summary
glog.V(1).Infof("Loaded %d identities, %d accounts, %d access keys. Auth enabled: %v",
len(identities), len(accounts), len(accessKeyIdent), iam.isAuthEnabled)
// Re-add environment variable credentials if they exist
// This ensures env var credentials persist across configuration reloads
iam.loadEnvironmentVariableCredentials()
// Log configuration summary - always log to help debugging
glog.Infof("Loaded %d identities, %d accounts, %d access keys. Auth enabled: %v",
len(iam.identities), len(iam.accounts), len(iam.accessKeyIdent), iam.isAuthEnabled)
if glog.V(2) {
glog.V(2).Infof("Access key to identity mapping:")
for accessKey, identity := range accessKeyIdent {
iam.m.RLock()
for accessKey, identity := range iam.accessKeyIdent {
glog.V(2).Infof(" %s -> %s (actions: %d)", accessKey, identity.Name, len(identity.Actions))
}
iam.m.RUnlock()
}
return nil

30
weed/s3api/s3api_server.go
View File

@@ -671,29 +671,31 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
return false
}
// Check Action parameter in both form data and query string
// We iterate ParseForm but ignore errors to ensure we attempt to parse the body
// even if it's malformed, then check FormValue which covers both body and query.
// This guards against misrouting STS requests if the body is invalid.
r.ParseForm()
action := r.FormValue("Action")
// If FormValue yielded nothing (possibly due to ParseForm failure failing to populate Form),
// explicitly fallback to Query string to be safe.
if action == "" {
action = r.URL.Query().Get("Action")
}
// Exclude STS actions - let them be handled by STS handlers
// IMPORTANT: Do NOT call r.ParseForm() here!
// ParseForm() consumes the request body, which breaks AWS Signature V4 verification
// for IAM requests. The signature must be calculated on the original body.
// Instead, check only the query string for the Action parameter.
// For IAM requests, the Action is typically in the POST body, not query string
// So we match all authenticated POST / requests and let AuthIam validate them
// This is safe because:
// 1. STS actions are excluded (handled by separate STS routes)
// 2. S3 operations don't POST to / (they use /<bucket> or /<bucket>/<key>)
// 3. IAM operations all POST to /
// Only exclude STS actions which might be in query string
action := r.URL.Query().Get("Action")
if action == "AssumeRole" || action == "AssumeRoleWithWebIdentity" || action == "AssumeRoleWithLDAPIdentity" {
return false
}
// Match all other authenticated POST / requests (IAM operations)
return true
}
apiRouter.Methods(http.MethodPost).Path("/").MatcherFunc(iamMatcher).
HandlerFunc(track(s3a.embeddedIam.AuthIam(s3a.cb.Limit(s3a.embeddedIam.DoActions, ACTION_WRITE)), "IAM"))
glog.V(1).Infof("Embedded IAM API enabled on S3 port")
}