gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add credential storage (#6938)

Browse Source 
* add credential store interface

* load credential.toml

* lint

* create credentialManager with explicit store type

* add type name

* InitializeCredentialManager

* remove unused functions

* fix missing import

* fix import

* fix nil configuration
This commit is contained in:
Chris Lu
2025-07-02 18:03:17 -07:00
committed by GitHub
parent 6b706f9ccd
commit 1db7c2b8aa
23 changed files with 3656 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
weed/s3api/auth_credentials.go
View File

@@ -1,19 +1,21 @@
package s3api
import (
"context"
"fmt"
"net/http"
"os"
"strings"
"sync"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"google.golang.org/grpc"
)
type Action string
@@ -35,6 +37,9 @@ type IdentityAccessManagement struct {
hashMu sync.RWMutex
domain string
isAuthEnabled bool
credentialManager *credential.CredentialManager
filerClient filer_pb.SeaweedFilerClient
grpcDialOption grpc.DialOption
}
type Identity struct {
@@ -114,19 +119,40 @@ func (action Action) getPermission() Permission {
}
func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, "")
}
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitStore string) *IdentityAccessManagement {
iam := &IdentityAccessManagement{
domain: option.DomainName,
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
}
// Always initialize credential manager with fallback to defaults
credentialManager, err := credential.NewCredentialManagerWithDefaults(credential.CredentialStoreTypeName(explicitStore))
if err != nil {
glog.Fatalf("failed to initialize credential manager: %v", err)
}
// For stores that need filer client details, set them
if store := credentialManager.GetStore(); store != nil {
if filerClientSetter, ok := store.(interface {
SetFilerClient(string, grpc.DialOption)
}); ok {
filerClientSetter.SetFilerClient(string(option.Filer), option.GrpcDialOption)
}
}
iam.credentialManager = credentialManager
if option.Config != "" {
glog.V(3).Infof("loading static config file %s", option.Config)
if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
glog.Fatalf("fail to load config file %s: %v", option.Config, err)
}
} else {
glog.V(3).Infof("no static config file specified... loading config from filer %s", option.Filer)
glog.V(3).Infof("no static config file specified... loading config from credential manager")
if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
glog.Warningf("fail to load config: %v", err)
}
@@ -134,17 +160,8 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
return iam
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) (err error) {
var content []byte
err = pb.WithFilerClient(false, 0, option.Filer, option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
glog.V(3).Infof("loading config %s from filer %s", filer.IamConfigDirectory+"/"+filer.IamIdentityFile, option.Filer)
content, err = filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile)
return err
})
if err != nil {
return fmt.Errorf("read S3 config: %v", err)
}
return iam.LoadS3ApiConfigurationFromBytes(content)
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
return iam.LoadS3ApiConfigurationFromCredentialManager()
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
@@ -516,3 +533,22 @@ func (identity *Identity) isAdmin() bool {
}
return false
}
// GetCredentialManager returns the credential manager instance
func (iam *IdentityAccessManagement) GetCredentialManager() *credential.CredentialManager {
return iam.credentialManager
}
// LoadS3ApiConfigurationFromCredentialManager loads configuration using the credential manager
func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager() error {
s3ApiConfiguration, err := iam.credentialManager.LoadConfiguration(context.Background())
if err != nil {
return fmt.Errorf("failed to load configuration from credential manager: %v", err)
}
if len(s3ApiConfiguration.Identities) == 0 {
return fmt.Errorf("no identities found")
}
return iam.loadS3ApiConfiguration(s3ApiConfiguration)
}

8
weed/s3api/s3api_put_object_helper_test.go
View File

@@ -5,13 +5,15 @@ import (
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/credential"
_ "github.com/seaweedfs/seaweedfs/weed/credential/memory"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
// Ensure IAM is disabled for this test
s3a.iam.isAuthEnabled = false
@@ -85,7 +87,7 @@ func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = false
@@ -120,7 +122,7 @@ func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
func TestGetRequestDataReader_IAMEnabled(t *testing.T) {
// Create an S3ApiServer with IAM enabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagement(&S3ApiServerOption{}),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = true

40
weed/s3api/s3api_server.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
@@ -41,16 +42,21 @@ type S3ApiServerOption struct {
type S3ApiServer struct {
s3_pb.UnimplementedSeaweedS3Server
option *S3ApiServerOption
iam *IdentityAccessManagement
cb *CircuitBreaker
randomClientId int32
filerGuard *security.Guard
client util_http_client.HTTPClientInterface
bucketRegistry *BucketRegistry
option *S3ApiServerOption
iam *IdentityAccessManagement
cb *CircuitBreaker
randomClientId int32
filerGuard *security.Guard
client util_http_client.HTTPClientInterface
bucketRegistry *BucketRegistry
credentialManager *credential.CredentialManager
}
func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
return NewS3ApiServerWithStore(router, option, "")
}
func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, explicitStore string) (s3ApiServer *S3ApiServer, err error) {
startTsNs := time.Now().UnixNano()
v := util.GetViper()
@@ -64,19 +70,25 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
v.SetDefault("cors.allowed_origins.values", "*")
if (option.AllowedOrigins == nil) || (len(option.AllowedOrigins) == 0) {
if len(option.AllowedOrigins) == 0 {
allowedOrigins := v.GetString("cors.allowed_origins.values")
domains := strings.Split(allowedOrigins, ",")
option.AllowedOrigins = domains
}
var iam *IdentityAccessManagement
iam = NewIdentityAccessManagementWithStore(option, explicitStore)
s3ApiServer = &S3ApiServer{
option: option,
iam: NewIdentityAccessManagement(option),
randomClientId: util.RandomInt32(),
filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
cb: NewCircuitBreaker(option),
option: option,
iam: iam,
randomClientId: util.RandomInt32(),
filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
cb: NewCircuitBreaker(option),
credentialManager: iam.credentialManager,
}
if option.Config != "" {
grace.OnReload(func() {
if err := s3ApiServer.iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
@@ -119,7 +131,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
func(w http.ResponseWriter, r *http.Request) {
origin := r.Header.Get("Origin")
if origin != "" {
if s3a.option.AllowedOrigins == nil || len(s3a.option.AllowedOrigins) == 0 || s3a.option.AllowedOrigins[0] == "*" {
if len(s3a.option.AllowedOrigins) == 0 || s3a.option.AllowedOrigins[0] == "*" {
origin = "*"
} else {
originFound := false