Add credential storage (#6938)

* add credential store interface * load credential.toml * lint * create credentialManager with explicit store type * add type name * InitializeCredentialManager * remove unused functions * fix missing import * fix import * fix nil configuration
2025-07-02 18:03:17 -07:00
parent 6b706f9ccd
commit 1db7c2b8aa
23 changed files with 3656 additions and 288 deletions
--- a/weed/credential/memory/memory_store.go
+++ b/weed/credential/memory/memory_store.go
@@ -0,0 +1,373 @@
+package memory
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	credential.Stores = append(credential.Stores, &MemoryStore{})
+}
+
+// MemoryStore implements CredentialStore using in-memory storage
+// This is primarily intended for testing purposes
+type MemoryStore struct {
+	mu          sync.RWMutex
+	users       map[string]*iam_pb.Identity // username -> identity
+	accessKeys  map[string]string           // access_key -> username
+	initialized bool
+}
+
+func (store *MemoryStore) GetName() credential.CredentialStoreTypeName {
+	return credential.StoreTypeMemory
+}
+
+func (store *MemoryStore) Initialize(configuration util.Configuration, prefix string) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if store.initialized {
+		return nil
+	}
+
+	store.users = make(map[string]*iam_pb.Identity)
+	store.accessKeys = make(map[string]string)
+	store.initialized = true
+
+	return nil
+}
+
+func (store *MemoryStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	if !store.initialized {
+		return nil, fmt.Errorf("store not initialized")
+	}
+
+	config := &iam_pb.S3ApiConfiguration{}
+
+	// Convert all users to identities
+	for _, user := range store.users {
+		// Deep copy the identity to avoid mutation issues
+		identityCopy := store.deepCopyIdentity(user)
+		config.Identities = append(config.Identities, identityCopy)
+	}
+
+	return config, nil
+}
+
+func (store *MemoryStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	// Clear existing data
+	store.users = make(map[string]*iam_pb.Identity)
+	store.accessKeys = make(map[string]string)
+
+	// Add all identities
+	for _, identity := range config.Identities {
+		// Deep copy to avoid mutation issues
+		identityCopy := store.deepCopyIdentity(identity)
+		store.users[identity.Name] = identityCopy
+
+		// Index access keys
+		for _, credential := range identity.Credentials {
+			store.accessKeys[credential.AccessKey] = identity.Name
+		}
+	}
+
+	return nil
+}
+
+func (store *MemoryStore) CreateUser(ctx context.Context, identity *iam_pb.Identity) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	if _, exists := store.users[identity.Name]; exists {
+		return credential.ErrUserAlreadyExists
+	}
+
+	// Check for duplicate access keys
+	for _, cred := range identity.Credentials {
+		if _, exists := store.accessKeys[cred.AccessKey]; exists {
+			return fmt.Errorf("access key %s already exists", cred.AccessKey)
+		}
+	}
+
+	// Deep copy to avoid mutation issues
+	identityCopy := store.deepCopyIdentity(identity)
+	store.users[identity.Name] = identityCopy
+
+	// Index access keys
+	for _, cred := range identity.Credentials {
+		store.accessKeys[cred.AccessKey] = identity.Name
+	}
+
+	return nil
+}
+
+func (store *MemoryStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	if !store.initialized {
+		return nil, fmt.Errorf("store not initialized")
+	}
+
+	user, exists := store.users[username]
+	if !exists {
+		return nil, credential.ErrUserNotFound
+	}
+
+	// Return a deep copy to avoid mutation issues
+	return store.deepCopyIdentity(user), nil
+}
+
+func (store *MemoryStore) UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	existingUser, exists := store.users[username]
+	if !exists {
+		return credential.ErrUserNotFound
+	}
+
+	// Remove old access keys from index
+	for _, cred := range existingUser.Credentials {
+		delete(store.accessKeys, cred.AccessKey)
+	}
+
+	// Check for duplicate access keys (excluding current user)
+	for _, cred := range identity.Credentials {
+		if existingUsername, exists := store.accessKeys[cred.AccessKey]; exists && existingUsername != username {
+			return fmt.Errorf("access key %s already exists", cred.AccessKey)
+		}
+	}
+
+	// Deep copy to avoid mutation issues
+	identityCopy := store.deepCopyIdentity(identity)
+	store.users[username] = identityCopy
+
+	// Re-index access keys
+	for _, cred := range identity.Credentials {
+		store.accessKeys[cred.AccessKey] = username
+	}
+
+	return nil
+}
+
+func (store *MemoryStore) DeleteUser(ctx context.Context, username string) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	user, exists := store.users[username]
+	if !exists {
+		return credential.ErrUserNotFound
+	}
+
+	// Remove access keys from index
+	for _, cred := range user.Credentials {
+		delete(store.accessKeys, cred.AccessKey)
+	}
+
+	// Remove user
+	delete(store.users, username)
+
+	return nil
+}
+
+func (store *MemoryStore) ListUsers(ctx context.Context) ([]string, error) {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	if !store.initialized {
+		return nil, fmt.Errorf("store not initialized")
+	}
+
+	var usernames []string
+	for username := range store.users {
+		usernames = append(usernames, username)
+	}
+
+	return usernames, nil
+}
+
+func (store *MemoryStore) GetUserByAccessKey(ctx context.Context, accessKey string) (*iam_pb.Identity, error) {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	if !store.initialized {
+		return nil, fmt.Errorf("store not initialized")
+	}
+
+	username, exists := store.accessKeys[accessKey]
+	if !exists {
+		return nil, credential.ErrAccessKeyNotFound
+	}
+
+	user, exists := store.users[username]
+	if !exists {
+		// This should not happen, but handle it gracefully
+		return nil, credential.ErrUserNotFound
+	}
+
+	// Return a deep copy to avoid mutation issues
+	return store.deepCopyIdentity(user), nil
+}
+
+func (store *MemoryStore) CreateAccessKey(ctx context.Context, username string, cred *iam_pb.Credential) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	user, exists := store.users[username]
+	if !exists {
+		return credential.ErrUserNotFound
+	}
+
+	// Check if access key already exists
+	if _, exists := store.accessKeys[cred.AccessKey]; exists {
+		return fmt.Errorf("access key %s already exists", cred.AccessKey)
+	}
+
+	// Add credential to user
+	user.Credentials = append(user.Credentials, &iam_pb.Credential{
+		AccessKey: cred.AccessKey,
+		SecretKey: cred.SecretKey,
+	})
+
+	// Index the access key
+	store.accessKeys[cred.AccessKey] = username
+
+	return nil
+}
+
+func (store *MemoryStore) DeleteAccessKey(ctx context.Context, username string, accessKey string) error {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if !store.initialized {
+		return fmt.Errorf("store not initialized")
+	}
+
+	user, exists := store.users[username]
+	if !exists {
+		return credential.ErrUserNotFound
+	}
+
+	// Find and remove the credential
+	var newCredentials []*iam_pb.Credential
+	found := false
+	for _, cred := range user.Credentials {
+		if cred.AccessKey == accessKey {
+			found = true
+			// Remove from access key index
+			delete(store.accessKeys, accessKey)
+		} else {
+			newCredentials = append(newCredentials, cred)
+		}
+	}
+
+	if !found {
+		return credential.ErrAccessKeyNotFound
+	}
+
+	user.Credentials = newCredentials
+	return nil
+}
+
+func (store *MemoryStore) Shutdown() {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	// Clear all data
+	store.users = nil
+	store.accessKeys = nil
+	store.initialized = false
+}
+
+// deepCopyIdentity creates a deep copy of an identity to avoid mutation issues
+func (store *MemoryStore) deepCopyIdentity(identity *iam_pb.Identity) *iam_pb.Identity {
+	if identity == nil {
+		return nil
+	}
+
+	// Use JSON marshaling/unmarshaling for deep copy
+	// This is simple and safe for protobuf messages
+	data, err := json.Marshal(identity)
+	if err != nil {
+		// Fallback to shallow copy if JSON fails
+		return &iam_pb.Identity{
+			Name:        identity.Name,
+			Account:     identity.Account,
+			Credentials: identity.Credentials,
+			Actions:     identity.Actions,
+		}
+	}
+
+	var copy iam_pb.Identity
+	if err := json.Unmarshal(data, &copy); err != nil {
+		// Fallback to shallow copy if JSON fails
+		return &iam_pb.Identity{
+			Name:        identity.Name,
+			Account:     identity.Account,
+			Credentials: identity.Credentials,
+			Actions:     identity.Actions,
+		}
+	}
+
+	return &copy
+}
+
+// Reset clears all data in the store (useful for testing)
+func (store *MemoryStore) Reset() {
+	store.mu.Lock()
+	defer store.mu.Unlock()
+
+	if store.initialized {
+		store.users = make(map[string]*iam_pb.Identity)
+		store.accessKeys = make(map[string]string)
+	}
+}
+
+// GetUserCount returns the number of users in the store (useful for testing)
+func (store *MemoryStore) GetUserCount() int {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	return len(store.users)
+}
+
+// GetAccessKeyCount returns the number of access keys in the store (useful for testing)
+func (store *MemoryStore) GetAccessKeyCount() int {
+	store.mu.RLock()
+	defer store.mu.RUnlock()
+
+	return len(store.accessKeys)
+}
--- a/weed/credential/memory/memory_store_test.go
+++ b/weed/credential/memory/memory_store_test.go
@@ -0,0 +1,315 @@
+package memory
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestMemoryStore(t *testing.T) {
+	store := &MemoryStore{}
+
+	// Test initialization
+	config := util.GetViper()
+	if err := store.Initialize(config, "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// Test creating a user
+	identity := &iam_pb.Identity{
+		Name: "testuser",
+		Credentials: []*iam_pb.Credential{
+			{
+				AccessKey: "access123",
+				SecretKey: "secret123",
+			},
+		},
+	}
+
+	if err := store.CreateUser(ctx, identity); err != nil {
+		t.Fatalf("Failed to create user: %v", err)
+	}
+
+	// Test getting user
+	retrievedUser, err := store.GetUser(ctx, "testuser")
+	if err != nil {
+		t.Fatalf("Failed to get user: %v", err)
+	}
+
+	if retrievedUser.Name != "testuser" {
+		t.Errorf("Expected username 'testuser', got '%s'", retrievedUser.Name)
+	}
+
+	if len(retrievedUser.Credentials) != 1 {
+		t.Errorf("Expected 1 credential, got %d", len(retrievedUser.Credentials))
+	}
+
+	// Test getting user by access key
+	userByAccessKey, err := store.GetUserByAccessKey(ctx, "access123")
+	if err != nil {
+		t.Fatalf("Failed to get user by access key: %v", err)
+	}
+
+	if userByAccessKey.Name != "testuser" {
+		t.Errorf("Expected username 'testuser', got '%s'", userByAccessKey.Name)
+	}
+
+	// Test listing users
+	users, err := store.ListUsers(ctx)
+	if err != nil {
+		t.Fatalf("Failed to list users: %v", err)
+	}
+
+	if len(users) != 1 || users[0] != "testuser" {
+		t.Errorf("Expected ['testuser'], got %v", users)
+	}
+
+	// Test creating access key
+	newCred := &iam_pb.Credential{
+		AccessKey: "access456",
+		SecretKey: "secret456",
+	}
+
+	if err := store.CreateAccessKey(ctx, "testuser", newCred); err != nil {
+		t.Fatalf("Failed to create access key: %v", err)
+	}
+
+	// Verify user now has 2 credentials
+	updatedUser, err := store.GetUser(ctx, "testuser")
+	if err != nil {
+		t.Fatalf("Failed to get updated user: %v", err)
+	}
+
+	if len(updatedUser.Credentials) != 2 {
+		t.Errorf("Expected 2 credentials, got %d", len(updatedUser.Credentials))
+	}
+
+	// Test deleting access key
+	if err := store.DeleteAccessKey(ctx, "testuser", "access456"); err != nil {
+		t.Fatalf("Failed to delete access key: %v", err)
+	}
+
+	// Verify user now has 1 credential again
+	finalUser, err := store.GetUser(ctx, "testuser")
+	if err != nil {
+		t.Fatalf("Failed to get final user: %v", err)
+	}
+
+	if len(finalUser.Credentials) != 1 {
+		t.Errorf("Expected 1 credential, got %d", len(finalUser.Credentials))
+	}
+
+	// Test deleting user
+	if err := store.DeleteUser(ctx, "testuser"); err != nil {
+		t.Fatalf("Failed to delete user: %v", err)
+	}
+
+	// Verify user is gone
+	_, err = store.GetUser(ctx, "testuser")
+	if err != credential.ErrUserNotFound {
+		t.Errorf("Expected ErrUserNotFound, got %v", err)
+	}
+
+	// Test error cases
+	if err := store.CreateUser(ctx, identity); err != nil {
+		t.Fatalf("Failed to create user for error tests: %v", err)
+	}
+
+	// Try to create duplicate user
+	if err := store.CreateUser(ctx, identity); err != credential.ErrUserAlreadyExists {
+		t.Errorf("Expected ErrUserAlreadyExists, got %v", err)
+	}
+
+	// Try to get non-existent user
+	_, err = store.GetUser(ctx, "nonexistent")
+	if err != credential.ErrUserNotFound {
+		t.Errorf("Expected ErrUserNotFound, got %v", err)
+	}
+
+	// Try to get user by non-existent access key
+	_, err = store.GetUserByAccessKey(ctx, "nonexistent")
+	if err != credential.ErrAccessKeyNotFound {
+		t.Errorf("Expected ErrAccessKeyNotFound, got %v", err)
+	}
+}
+
+func TestMemoryStoreConcurrency(t *testing.T) {
+	store := &MemoryStore{}
+	config := util.GetViper()
+	if err := store.Initialize(config, "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// Test concurrent access
+	done := make(chan bool, 10)
+	for i := 0; i < 10; i++ {
+		go func(i int) {
+			defer func() { done <- true }()
+
+			username := fmt.Sprintf("user%d", i)
+			identity := &iam_pb.Identity{
+				Name: username,
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: fmt.Sprintf("access%d", i),
+						SecretKey: fmt.Sprintf("secret%d", i),
+					},
+				},
+			}
+
+			if err := store.CreateUser(ctx, identity); err != nil {
+				t.Errorf("Failed to create user %s: %v", username, err)
+				return
+			}
+
+			if _, err := store.GetUser(ctx, username); err != nil {
+				t.Errorf("Failed to get user %s: %v", username, err)
+				return
+			}
+		}(i)
+	}
+
+	// Wait for all goroutines to complete
+	for i := 0; i < 10; i++ {
+		<-done
+	}
+
+	// Verify all users were created
+	users, err := store.ListUsers(ctx)
+	if err != nil {
+		t.Fatalf("Failed to list users: %v", err)
+	}
+
+	if len(users) != 10 {
+		t.Errorf("Expected 10 users, got %d", len(users))
+	}
+}
+
+func TestMemoryStoreReset(t *testing.T) {
+	store := &MemoryStore{}
+	config := util.GetViper()
+	if err := store.Initialize(config, "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// Create a user
+	identity := &iam_pb.Identity{
+		Name: "testuser",
+		Credentials: []*iam_pb.Credential{
+			{
+				AccessKey: "access123",
+				SecretKey: "secret123",
+			},
+		},
+	}
+
+	if err := store.CreateUser(ctx, identity); err != nil {
+		t.Fatalf("Failed to create user: %v", err)
+	}
+
+	// Verify user exists
+	if store.GetUserCount() != 1 {
+		t.Errorf("Expected 1 user, got %d", store.GetUserCount())
+	}
+
+	if store.GetAccessKeyCount() != 1 {
+		t.Errorf("Expected 1 access key, got %d", store.GetAccessKeyCount())
+	}
+
+	// Reset the store
+	store.Reset()
+
+	// Verify store is empty
+	if store.GetUserCount() != 0 {
+		t.Errorf("Expected 0 users after reset, got %d", store.GetUserCount())
+	}
+
+	if store.GetAccessKeyCount() != 0 {
+		t.Errorf("Expected 0 access keys after reset, got %d", store.GetAccessKeyCount())
+	}
+
+	// Verify user is gone
+	_, err := store.GetUser(ctx, "testuser")
+	if err != credential.ErrUserNotFound {
+		t.Errorf("Expected ErrUserNotFound after reset, got %v", err)
+	}
+}
+
+func TestMemoryStoreConfigurationSaveLoad(t *testing.T) {
+	store := &MemoryStore{}
+	config := util.GetViper()
+	if err := store.Initialize(config, "credential."); err != nil {
+		t.Fatalf("Failed to initialize store: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// Create initial configuration
+	originalConfig := &iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{
+			{
+				Name: "user1",
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "access1",
+						SecretKey: "secret1",
+					},
+				},
+			},
+			{
+				Name: "user2",
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "access2",
+						SecretKey: "secret2",
+					},
+				},
+			},
+		},
+	}
+
+	// Save configuration
+	if err := store.SaveConfiguration(ctx, originalConfig); err != nil {
+		t.Fatalf("Failed to save configuration: %v", err)
+	}
+
+	// Load configuration
+	loadedConfig, err := store.LoadConfiguration(ctx)
+	if err != nil {
+		t.Fatalf("Failed to load configuration: %v", err)
+	}
+
+	// Verify configuration matches
+	if len(loadedConfig.Identities) != 2 {
+		t.Errorf("Expected 2 identities, got %d", len(loadedConfig.Identities))
+	}
+
+	// Check users exist
+	user1, err := store.GetUser(ctx, "user1")
+	if err != nil {
+		t.Fatalf("Failed to get user1: %v", err)
+	}
+
+	if len(user1.Credentials) != 1 || user1.Credentials[0].AccessKey != "access1" {
+		t.Errorf("User1 credentials not correct: %+v", user1.Credentials)
+	}
+
+	user2, err := store.GetUser(ctx, "user2")
+	if err != nil {
+		t.Fatalf("Failed to get user2: %v", err)
+	}
+
+	if len(user2.Credentials) != 1 || user2.Credentials[0].AccessKey != "access2" {
+		t.Errorf("User2 credentials not correct: %+v", user2.Credentials)
+	}
+}