diff --git a/test/s3/sse/Makefile b/test/s3/sse/Makefile index 87c171486..cd7d50e9e 100644 --- a/test/s3/sse/Makefile +++ b/test/s3/sse/Makefile @@ -98,8 +98,8 @@ start-seaweedfs: check-binary # Create S3 configuration with SSE-KMS support @printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json - # Start weed mini - @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \ + # Start weed mini (WEED_S3_SSE_KEY enables SSE-S3 encryption) + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key $(SEAWEEDFS_BINARY) mini \ -dir=/tmp/seaweedfs-test-sse \ -s3.port=$(S3_PORT) \ -s3.config=/tmp/seaweedfs-sse-s3.json \ @@ -337,8 +337,9 @@ start-seaweedfs-ci: check-binary s3-config-template.json > /tmp/seaweedfs-s3.json # Start weed mini with embedded S3 using the JSON config (with verbose logging) + # WEED_S3_SSE_KEY enables SSE-S3 encryption for testing (KEK derived via HKDF) @echo "Starting weed mini with embedded S3..." - @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ -dir=/tmp/seaweedfs-test-sse \ -s3.port=$(S3_PORT) \ -s3.config=/tmp/seaweedfs-s3.json \ @@ -482,7 +483,7 @@ test-volume-encryption: build-weed -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \ s3-config-template.json > /tmp/seaweedfs-s3.json @echo "Starting weed mini with S3 volume encryption..." - @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ -dir=/tmp/seaweedfs-test-sse \ -s3.port=$(S3_PORT) \ -s3.config=/tmp/seaweedfs-s3.json \ diff --git a/weed/command/scaffold/security.toml b/weed/command/scaffold/security.toml index 6c8aaa475..ca797ca5c 100644 --- a/weed/command/scaffold/security.toml +++ b/weed/command/scaffold/security.toml @@ -179,6 +179,19 @@ password = "" user = "" password = "" +# SSE-S3 server-side encryption key management +# These settings configure the Key Encryption Key (KEK) for S3 SSE-S3 encryption. +# Set exactly one of kek or key. If neither is set, SSE-S3 is disabled. +# Can also be set via env vars: WEED_S3_SSE_KEK, WEED_S3_SSE_KEY +[s3.sse] +# hex-encoded 256-bit key, same format as the legacy /etc/s3/sse_kek filer file. +# Use this to migrate from a filer-stored KEK: copy the value from /etc/s3/sse_kek. +# Generate a new one with: openssl rand -hex 32 +kek = "" +# any secret string; a 256-bit key is derived automatically via HKDF-SHA256. +# Cannot be used while /etc/s3/sse_kek exists on the filer — delete it first. +key = "" + # white list. It's checking request ip address. [guard] white_list = "" diff --git a/weed/s3api/s3_sse_s3.go b/weed/s3api/s3_sse_s3.go index e869308f3..d9ea5a919 100644 --- a/weed/s3api/s3_sse_s3.go +++ b/weed/s3api/s3_sse_s3.go @@ -1,6 +1,7 @@ package s3api import ( + "bytes" "context" "crypto/aes" "crypto/cipher" @@ -14,7 +15,6 @@ import ( "io" mathrand "math/rand" "net/http" - "os" "sync" "time" @@ -258,14 +258,20 @@ type SSES3KeyManager struct { } const ( - // KEK storage directory and file name in filer - SSES3KEKDirectory = "/etc/s3" - SSES3KEKParentDir = "/etc" - SSES3KEKDirName = "s3" - SSES3KEKFileName = "sse_kek" + // Legacy KEK path on the filer (backward compatibility) + defaultKEKPath = "/etc/s3/sse_kek" - // Full KEK path in filer - defaultKEKPath = SSES3KEKDirectory + "/" + SSES3KEKFileName + // security.toml keys (also settable via env vars WEED_S3_SSE_KEK / WEED_S3_SSE_KEY): + // + // s3.sse.kek: hex-encoded 256-bit key, same format as /etc/s3/sse_kek. + // Drop-in replacement for the filer-stored KEK. If /etc/s3/sse_kek also + // exists, the values must match or the server refuses to start. + // + // s3.sse.key: any secret string; a 256-bit key is derived via HKDF-SHA256. + // Cannot be used while /etc/s3/sse_kek exists — the filer file must be + // deleted first (to avoid silently orphaning old data). + sseS3KEKConfigKey = "s3.sse.kek" + sseS3KeyConfigKey = "s3.sse.key" ) // NewSSES3KeyManager creates a new SSE-S3 key manager with envelope encryption @@ -276,34 +282,145 @@ func NewSSES3KeyManager() *SSES3KeyManager { } } -// InitializeWithFiler initializes the key manager with a filer client -func (km *SSES3KeyManager) InitializeWithFiler(filerClient filer_pb.FilerClient) error { - km.mu.Lock() - defer km.mu.Unlock() +// deriveKeyFromSecret derives a 256-bit key from an arbitrary secret string +// using HKDF-SHA256. The derivation is deterministic: the same secret always +// produces the same key. +func deriveKeyFromSecret(secret string) ([]byte, error) { + hkdfReader := hkdf.New(sha256.New, []byte(secret), nil, []byte("seaweedfs-sse-s3-kek")) + key := make([]byte, SSES3KeySize) + if _, err := io.ReadFull(hkdfReader, key); err != nil { + return nil, fmt.Errorf("failed to derive key: %w", err) + } + return key, nil +} - km.filerClient = filerClient - - // Try to load existing KEK from filer with retries to handle transient connectivity issues during startup - var err error +// loadFilerKEK tries to load the KEK from /etc/s3/sse_kek on the filer. +// Returns the key bytes on success, nil if the file does not exist or filer +// is not configured, or an error on transient failures (retries internally). +func (km *SSES3KeyManager) loadFilerKEK() ([]byte, error) { + if km.filerClient == nil { + return nil, nil // no filer configured + } + var lastErr error for i := 0; i < 10; i++ { - err = km.loadSuperKeyFromFiler() + err := km.loadSuperKeyFromFiler() if err == nil { - glog.V(1).Infof("SSE-S3 KeyManager: Loaded KEK from filer %s", km.kekPath) - return nil + // loadSuperKeyFromFiler sets km.superKey; grab a copy + key := make([]byte, len(km.superKey)) + copy(key, km.superKey) + km.superKey = nil // will be set by caller + return key, nil } if errors.Is(err, filer_pb.ErrNotFound) { - glog.V(1).Infof("SSE-S3 KeyManager: KEK not found, generating new KEK (load from filer %s: %v)", km.kekPath, err) - if genErr := km.generateAndSaveSuperKeyToFiler(); genErr != nil { - return fmt.Errorf("failed to generate and save SSE-S3 super key: %w", genErr) - } - return nil + return nil, nil // file does not exist } + lastErr = err glog.Warningf("SSE-S3 KeyManager: failed to load KEK (attempt %d/10): %v", i+1, err) time.Sleep(2 * time.Second) } + return nil, fmt.Errorf("failed to load KEK from %s after 10 attempts: %w", km.kekPath, lastErr) +} - // If we're here, all retries failed - return fmt.Errorf("failed to load SSE-S3 super key from %s after 10 attempts: %w", km.kekPath, err) +// InitializeWithFiler initializes the key manager with a filer client. +// +// Key source priority (via security.toml or WEED_ env vars): +// 1. s3.sse.kek (env: WEED_S3_SSE_KEK) — hex-encoded, same format as /etc/s3/sse_kek. +// If the filer file also exists, they must match. +// 2. s3.sse.key (env: WEED_S3_SSE_KEY) — any string; 256-bit key derived via HKDF. +// Refused if /etc/s3/sse_kek exists — delete the filer file first. +// 3. Existing /etc/s3/sse_kek on the filer (backward compat). +// 4. SSE-S3 disabled (fail on first encrypt/decrypt attempt). +func (km *SSES3KeyManager) InitializeWithFiler(filerClient filer_pb.FilerClient) error { + // Set filerClient under lock, then release — the rest may do slow I/O + // (filer retries with sleep) and must not block encrypt/decrypt callers. + km.mu.Lock() + km.filerClient = filerClient + km.mu.Unlock() + + v := util.GetViper() + cfgKEK := v.GetString(sseS3KEKConfigKey) // hex-encoded, drop-in for filer file + cfgKey := v.GetString(sseS3KeyConfigKey) // any string, HKDF-derived + + if cfgKEK != "" && cfgKey != "" { + return fmt.Errorf("only one of %s and %s may be set, not both", sseS3KEKConfigKey, sseS3KeyConfigKey) + } + + var resolvedKey []byte + + switch { + // --- Case 1: s3.sse.kek (hex, same format as filer file) --- + case cfgKEK != "": + key, err := hex.DecodeString(cfgKEK) + if err != nil { + return fmt.Errorf("invalid %s: must be hex-encoded: %w", sseS3KEKConfigKey, err) + } + if len(key) != SSES3KeySize { + return fmt.Errorf("invalid %s: must be %d bytes (%d hex chars), got %d bytes", + sseS3KEKConfigKey, SSES3KeySize, SSES3KeySize*2, len(key)) + } + + // Best-effort consistency check: if the filer file exists, warn on + // mismatch. A temporarily unreachable filer must not block startup + // when the operator has explicitly provided a KEK. + filerKey, err := km.loadFilerKEK() + if err != nil { + glog.Warningf("SSE-S3 KeyManager: could not reach filer to verify %s against %s: %v (proceeding with configured KEK)", + sseS3KEKConfigKey, km.kekPath, err) + } else if filerKey != nil && !bytes.Equal(filerKey, key) { + return fmt.Errorf("%s does not match existing %s — "+ + "use the same key value as the filer file, or migrate existing data to the new key. "+ + "See the Server-Side-Encryption wiki for migration steps", + sseS3KEKConfigKey, km.kekPath) + } + + resolvedKey = key + glog.V(0).Infof("SSE-S3 KeyManager: Loaded KEK from %s config", sseS3KEKConfigKey) + + // --- Case 2: s3.sse.key (any string, HKDF-derived) --- + case cfgKey != "": + // If the filer still has a legacy KEK file, the operator must migrate + // existing data first — using a derived key would silently orphan + // objects encrypted with the old KEK. + filerKey, err := km.loadFilerKEK() + if err != nil { + glog.Warningf("SSE-S3 KeyManager: could not reach filer to check for legacy %s: %v (proceeding with configured key)", + km.kekPath, err) + } else if filerKey != nil { + return fmt.Errorf("%s cannot be used while %s exists on the filer — "+ + "existing objects are encrypted with the filer KEK. "+ + "Migrate to %s first (copy the filer KEK value) or follow the key-rotation steps in the Server-Side-Encryption wiki", + sseS3KeyConfigKey, km.kekPath, sseS3KEKConfigKey) + } + + derived, err := deriveKeyFromSecret(cfgKey) + if err != nil { + return err + } + resolvedKey = derived + glog.V(0).Infof("SSE-S3 KeyManager: Derived KEK from %s config", sseS3KeyConfigKey) + + // --- Case 3: Load existing filer KEK (backward compatibility) --- + default: + filerKey, err := km.loadFilerKEK() + if err != nil { + return err + } + if filerKey != nil { + resolvedKey = filerKey + glog.V(1).Infof("SSE-S3 KeyManager: Loaded KEK from filer %s", km.kekPath) + glog.V(0).Infof("SSE-S3 KeyManager: Consider setting %s in security.toml instead of storing KEK on filer", sseS3KEKConfigKey) + } else { + // --- Case 4: Nothing configured — SSE-S3 disabled --- + glog.V(0).Infof("SSE-S3 KeyManager: No KEK configured. SSE-S3 encryption is disabled. "+ + "Set %s or %s in security.toml to enable it.", sseS3KEKConfigKey, sseS3KeyConfigKey) + } + } + + // Only hold the lock to write the final state. + km.mu.Lock() + km.superKey = resolvedKey + km.mu.Unlock() + return nil } // loadSuperKeyFromFiler loads the KEK from the filer @@ -337,47 +454,6 @@ func (km *SSES3KeyManager) loadSuperKeyFromFiler() error { return nil } -// generateAndSaveSuperKeyToFiler generates a new KEK and saves it to the filer -func (km *SSES3KeyManager) generateAndSaveSuperKeyToFiler() error { - if km.filerClient == nil { - return fmt.Errorf("filer client not initialized") - } - - // Generate a random 256-bit super key (KEK) - superKey := make([]byte, SSES3KeySize) - if _, err := io.ReadFull(rand.Reader, superKey); err != nil { - return fmt.Errorf("failed to generate KEK: %w", err) - } - - // Encode as hex for storage - encodedKey := []byte(hex.EncodeToString(superKey)) - - // Create the entry in filer - // First ensure the parent directory exists - if err := filer_pb.Mkdir(context.Background(), km.filerClient, SSES3KEKParentDir, SSES3KEKDirName, func(entry *filer_pb.Entry) { - // Set appropriate permissions for the directory - entry.Attributes.FileMode = uint32(0700 | os.ModeDir) - }); err != nil { - // Only ignore "already exists" errors. - if !errors.Is(err, filer_pb.ErrEntryAlreadyExists) { - return fmt.Errorf("failed to create KEK directory %s: %w", SSES3KEKDirectory, err) - } - glog.V(3).Infof("Parent directory %s already exists, continuing.", SSES3KEKDirectory) - } - - // Create the KEK file - if err := filer_pb.MkFile(context.Background(), km.filerClient, SSES3KEKDirectory, SSES3KEKFileName, nil, func(entry *filer_pb.Entry) { - entry.Content = encodedKey - entry.Attributes.FileMode = 0600 // Read/write for owner only - entry.Attributes.FileSize = uint64(len(encodedKey)) - }); err != nil { - return fmt.Errorf("failed to create KEK file in filer: %w", err) - } - - km.superKey = superKey - glog.Infof("SSE-S3 KeyManager: Generated and saved new KEK to filer %s", km.kekPath) - return nil -} // GetOrCreateKey gets an existing key or creates a new one // With envelope encryption, we always generate a new DEK since we don't store them @@ -391,6 +467,10 @@ func (km *SSES3KeyManager) encryptKeyWithSuperKey(dek []byte) ([]byte, []byte, e km.mu.RLock() defer km.mu.RUnlock() + if len(km.superKey) == 0 { + return nil, nil, fmt.Errorf("SSE-S3 encryption is not configured — set %s or %s in security.toml", sseS3KEKConfigKey, sseS3KeyConfigKey) + } + block, err := aes.NewCipher(km.superKey) if err != nil { return nil, nil, fmt.Errorf("failed to create cipher: %w", err) @@ -418,6 +498,10 @@ func (km *SSES3KeyManager) decryptKeyWithSuperKey(encryptedDEK, nonce []byte) ([ km.mu.RLock() defer km.mu.RUnlock() + if len(km.superKey) == 0 { + return nil, fmt.Errorf("SSE-S3 decryption is not configured — set %s or %s in security.toml", sseS3KEKConfigKey, sseS3KeyConfigKey) + } + block, err := aes.NewCipher(km.superKey) if err != nil { return nil, fmt.Errorf("failed to create cipher: %w", err) diff --git a/weed/s3api/s3_sse_s3_test.go b/weed/s3api/s3_sse_s3_test.go index 8087111c2..af64850d9 100644 --- a/weed/s3api/s3_sse_s3_test.go +++ b/weed/s3api/s3_sse_s3_test.go @@ -2,6 +2,7 @@ package s3api import ( "bytes" + "encoding/hex" "fmt" "io" "net/http" @@ -11,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/util" ) // TestSSES3EncryptionDecryption tests basic SSE-S3 encryption and decryption @@ -681,6 +683,125 @@ func TestSSES3InvalidMetadataDeserialization(t *testing.T) { } } +// setViperKey is a test helper that sets a config key via its WEED_ env var. +func setViperKey(t *testing.T, key, value string) { + t.Helper() + util.GetViper().SetDefault(key, "") + t.Setenv("WEED_"+strings.ReplaceAll(strings.ToUpper(key), ".", "_"), value) +} + +// TestSSES3KEKConfig tests that sse_s3.kek (hex format) is used as KEK +func TestSSES3KEKConfig(t *testing.T) { + testKey := make([]byte, 32) + for i := range testKey { + testKey[i] = byte(i + 50) + } + setViperKey(t, sseS3KEKConfigKey, hex.EncodeToString(testKey)) + + km := NewSSES3KeyManager() + err := km.InitializeWithFiler(nil) + if err != nil { + t.Fatalf("InitializeWithFiler failed: %v", err) + } + + if !bytes.Equal(km.superKey, testKey) { + t.Errorf("superKey mismatch: expected %x, got %x", testKey, km.superKey) + } + + // Round-trip DEK encryption + dek := make([]byte, 32) + for i := range dek { + dek[i] = byte(i) + } + encrypted, nonce, err := km.encryptKeyWithSuperKey(dek) + if err != nil { + t.Fatalf("encryptKeyWithSuperKey failed: %v", err) + } + decrypted, err := km.decryptKeyWithSuperKey(encrypted, nonce) + if err != nil { + t.Fatalf("decryptKeyWithSuperKey failed: %v", err) + } + if !bytes.Equal(decrypted, dek) { + t.Error("round-trip DEK mismatch") + } +} + +// TestSSES3KEKConfigInvalidHex tests rejection of bad hex +func TestSSES3KEKConfigInvalidHex(t *testing.T) { + setViperKey(t, sseS3KEKConfigKey, "not-valid-hex") + + km := NewSSES3KeyManager() + err := km.InitializeWithFiler(nil) + if err == nil { + t.Fatal("expected error for invalid hex, got nil") + } + if !strings.Contains(err.Error(), "hex-encoded") { + t.Errorf("expected hex error, got: %v", err) + } +} + +// TestSSES3KEKConfigWrongSize tests rejection of wrong-size hex key +func TestSSES3KEKConfigWrongSize(t *testing.T) { + setViperKey(t, sseS3KEKConfigKey, hex.EncodeToString(make([]byte, 16))) + + km := NewSSES3KeyManager() + err := km.InitializeWithFiler(nil) + if err == nil { + t.Fatal("expected error for wrong key size, got nil") + } + if !strings.Contains(err.Error(), "32 bytes") { + t.Errorf("expected size error, got: %v", err) + } +} + +// TestSSES3KeyConfig tests that sse_s3.key (any string, HKDF) works +func TestSSES3KeyConfig(t *testing.T) { + setViperKey(t, sseS3KeyConfigKey, "my-secret-passphrase") + + km := NewSSES3KeyManager() + err := km.InitializeWithFiler(nil) + if err != nil { + t.Fatalf("InitializeWithFiler failed: %v", err) + } + + if len(km.superKey) != SSES3KeySize { + t.Fatalf("expected %d-byte superKey, got %d", SSES3KeySize, len(km.superKey)) + } + + // Deterministic: same input → same output + expected, err := deriveKeyFromSecret("my-secret-passphrase") + if err != nil { + t.Fatalf("deriveKeyFromSecret failed: %v", err) + } + if !bytes.Equal(km.superKey, expected) { + t.Errorf("superKey mismatch: expected %x, got %x", expected, km.superKey) + } +} + +// TestSSES3KeyConfigDifferentSecrets tests different strings produce different keys +func TestSSES3KeyConfigDifferentSecrets(t *testing.T) { + k1, _ := deriveKeyFromSecret("secret-one") + k2, _ := deriveKeyFromSecret("secret-two") + if bytes.Equal(k1, k2) { + t.Error("different secrets should produce different keys") + } +} + +// TestSSES3BothConfigsReject tests that setting both config keys is rejected +func TestSSES3BothConfigsReject(t *testing.T) { + setViperKey(t, sseS3KEKConfigKey, hex.EncodeToString(make([]byte, 32))) + setViperKey(t, sseS3KeyConfigKey, "some-passphrase") + + km := NewSSES3KeyManager() + err := km.InitializeWithFiler(nil) + if err == nil { + t.Fatal("expected error when both configs set, got nil") + } + if !strings.Contains(err.Error(), "only one") { + t.Errorf("expected 'only one' error, got: %v", err) + } +} + // TestGetSSES3Headers tests SSE-S3 header generation func TestGetSSES3Headers(t *testing.T) { headers := GetSSES3Headers()