fix: serialize SSE-KMS metadata when bucket default encryption applies KMS (#8780)
* fix: serialize SSE-KMS metadata when bucket default encryption applies KMS When a bucket has default SSE-KMS encryption enabled and a file is uploaded without explicit SSE headers, the encryption was applied correctly but the SSE-KMS metadata (x-seaweedfs-sse-kms-key) was not serialized. This caused downloads to fail with "empty SSE-KMS metadata" because the entry's Extended map stored an empty byte slice. The existing code already handled this for SSE-S3 bucket defaults (SerializeSSES3Metadata) but was missing the equivalent call to SerializeSSEKMSMetadata for the KMS path. Fixes seaweedfs/seaweedfs#8776 * ci: add KMS integration tests to GitHub Actions Add a kms-tests.yml workflow that runs on changes to KMS/SSE code with two jobs: 1. KMS provider tests: starts OpenBao via Docker, runs Go integration tests in test/kms/ against a real KMS backend 2. S3 KMS e2e tests: starts OpenBao + weed mini built from source, runs test_s3_kms.sh which covers bucket-default SSE-KMS upload/download (the exact scenario from #8776) Supporting changes: - test/kms/Makefile: add CI targets (test-provider-ci, test-s3-kms-ci) that manage OpenBao via plain Docker and run weed from source - test/kms/s3-config-openbao-template.json: S3 config template with OpenBao KMS provider for weed mini * refactor: combine SSE-S3 and SSE-KMS metadata serialization into else-if SSE-S3 and SSE-KMS bucket default encryption are mutually exclusive, so use a single if/else-if block instead of two independent if blocks. * Update .github/workflows/kms-tests.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * fix(ci): start weed mini from data dir to avoid Docker filer.toml weed mini reads filer.toml from the current working directory first. When running from test/kms/, it picked up the Docker-targeted filer.toml which has dir="/data/filerdb" (a path that doesn't exist in CI), causing a fatal crash at filer store initialization. Fix by cd-ing to the data directory before starting weed mini. Also improve log visibility on failure. --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
This commit is contained in:
Chris Lu
140
.github/workflows/kms-tests.yml
vendored
Normal file
140
.github/workflows/kms-tests.yml
vendored
Normal file
@@ -0,0 +1,140 @@
|
||||
name: "KMS Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
- '.github/workflows/kms-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/kms/**'
|
||||
- 'weed/s3api/s3_sse_*.go'
|
||||
- 'weed/s3api/s3api_object_handlers.go'
|
||||
- 'weed/s3api/s3api_object_handlers_put.go'
|
||||
- 'test/kms/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}-kms-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
kms-provider-tests:
|
||||
name: KMS Provider Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run KMS provider integration tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
|
||||
make test-provider-ci
|
||||
|
||||
- name: Show OpenBao logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-openbao-ci || true
|
||||
|
||||
s3-kms-e2e-tests:
|
||||
name: S3 KMS End-to-End Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 KMS end-to-end tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
docker --version
|
||||
aws --version
|
||||
|
||||
make test-s3-kms-ci
|
||||
|
||||
- name: Show logs on failure
|
||||
if: failure()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
|
||||
echo "=== SeaweedFS Server Logs ==="
|
||||
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-kms-e2e-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-kms-mini.log
|
||||
/tmp/openbao-ci-container.log
|
||||
/tmp/weed-kms-ci-setup.log
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
make stop-seaweedfs-ci || true
|
||||
make stop-openbao-ci || true
|
||||
make clean-ci || true
|
||||
@@ -7,6 +7,14 @@ SEAWEEDFS_S3_ENDPOINT ?= http://127.0.0.1:8333
|
||||
TEST_TIMEOUT ?= 5m
|
||||
DOCKER_COMPOSE ?= docker-compose
|
||||
|
||||
# CI configuration (build weed from source, run natively)
|
||||
SEAWEEDFS_BINARY ?= weed
|
||||
S3_PORT ?= 8333
|
||||
ACCESS_KEY ?= some_access_key1
|
||||
SECRET_KEY ?= some_secret_key1
|
||||
TEST_DIR := $(shell pwd)
|
||||
SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
|
||||
|
||||
# Colors for output
|
||||
BLUE := \033[36m
|
||||
GREEN := \033[32m
|
||||
@@ -14,7 +22,10 @@ YELLOW := \033[33m
|
||||
RED := \033[31m
|
||||
NC := \033[0m # No Color
|
||||
|
||||
.PHONY: help setup test test-unit test-integration test-e2e clean logs status
|
||||
.PHONY: help setup test test-unit test-integration test-e2e clean logs status \
|
||||
build-weed check-binary start-openbao-ci stop-openbao-ci \
|
||||
start-seaweedfs-ci stop-seaweedfs-ci clean-ci \
|
||||
test-provider-ci test-s3-kms-ci
|
||||
|
||||
help: ## Show this help message
|
||||
@echo "$(BLUE)SeaweedFS KMS Integration Testing$(NC)"
|
||||
@@ -124,16 +135,146 @@ check-env: ## Check test environment setup
|
||||
@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
|
||||
@make install-deps
|
||||
|
||||
# CI targets
|
||||
ci-test: ## Run tests in CI environment
|
||||
# CI targets (docker-compose based)
|
||||
ci-test: ## Run tests in CI environment (docker-compose)
|
||||
@echo "$(YELLOW)Running CI tests...$(NC)"
|
||||
@make setup
|
||||
@make test-unit
|
||||
@make test-integration
|
||||
@make clean
|
||||
|
||||
ci-e2e: ## Run end-to-end tests in CI
|
||||
ci-e2e: ## Run end-to-end tests in CI (docker-compose)
|
||||
@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
|
||||
@make setup-seaweedfs
|
||||
@make test-e2e
|
||||
@make clean
|
||||
|
||||
####################################################
|
||||
# GitHub Actions CI targets (build weed from source)
|
||||
####################################################
|
||||
|
||||
build-weed: ## Build SeaweedFS binary from source
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
|
||||
@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
|
||||
|
||||
check-binary:
|
||||
@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
|
||||
echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
start-openbao-ci: ## Start OpenBao via Docker for CI
|
||||
@echo "Starting OpenBao for CI..."
|
||||
@docker rm -f openbao-ci 2>/dev/null || true
|
||||
@docker run -d --name openbao-ci \
|
||||
-p 8200:8200 \
|
||||
-e BAO_DEV_ROOT_TOKEN_ID=$(OPENBAO_TOKEN) \
|
||||
-e BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 \
|
||||
ghcr.io/openbao/openbao:latest \
|
||||
bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200
|
||||
@echo "Waiting for OpenBao to be ready..."
|
||||
@for i in $$(seq 1 30); do \
|
||||
if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \
|
||||
echo "$(GREEN)OpenBao is ready$(NC)"; \
|
||||
break; \
|
||||
fi; \
|
||||
if [ $$i -eq 30 ]; then \
|
||||
echo "$(RED)Timeout waiting for OpenBao$(NC)"; \
|
||||
docker logs openbao-ci 2>&1 | tail -20; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
sleep 2; \
|
||||
done
|
||||
@chmod +x setup_openbao.sh
|
||||
@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
|
||||
|
||||
stop-openbao-ci: ## Stop OpenBao CI container
|
||||
@echo "Stopping OpenBao..."
|
||||
@docker stop openbao-ci 2>/dev/null || true
|
||||
@docker rm openbao-ci 2>/dev/null || true
|
||||
|
||||
start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config
|
||||
@echo "Starting SeaweedFS with OpenBao KMS..."
|
||||
@mkdir -p /tmp/seaweedfs-test-kms
|
||||
@rm -f /tmp/seaweedfs-kms-*.log || true
|
||||
@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
|
||||
-e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
|
||||
-e 's|OPENBAO_ADDR_PLACEHOLDER|$(OPENBAO_ADDR)|g' \
|
||||
-e 's/OPENBAO_TOKEN_PLACEHOLDER/$(OPENBAO_TOKEN)/g' \
|
||||
s3-config-openbao-template.json > /tmp/seaweedfs-kms-s3.json
|
||||
@# Start weed mini from the data dir to avoid picking up test/kms/filer.toml
|
||||
@# (filer.toml is read from "." first, and the one here has Docker-only paths)
|
||||
@cd /tmp/seaweedfs-test-kms && \
|
||||
AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
|
||||
-dir=/tmp/seaweedfs-test-kms \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=/tmp/seaweedfs-kms-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid
|
||||
@echo "Checking S3 service is ready..."
|
||||
@for i in $$(seq 1 30); do \
|
||||
if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
|
||||
echo "$(GREEN)S3 service is ready$(NC)"; \
|
||||
break; \
|
||||
fi; \
|
||||
if [ $$i -eq 30 ]; then \
|
||||
echo "$(RED)Timeout waiting for S3 service$(NC)"; \
|
||||
echo "=== Last 50 lines of server log ==="; \
|
||||
tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done
|
||||
|
||||
stop-seaweedfs-ci: ## Stop weed mini (CI-safe)
|
||||
@echo "Stopping SeaweedFS..."
|
||||
@if [ -f /tmp/weed-kms-mini.pid ]; then \
|
||||
kill $$(cat /tmp/weed-kms-mini.pid) 2>/dev/null || true; \
|
||||
rm -f /tmp/weed-kms-mini.pid; \
|
||||
fi
|
||||
@if command -v lsof >/dev/null 2>&1; then \
|
||||
lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
|
||||
fi
|
||||
@sleep 2
|
||||
|
||||
clean-ci: ## Clean up CI test artifacts
|
||||
@rm -rf /tmp/seaweedfs-test-kms
|
||||
@rm -f /tmp/seaweedfs-kms-*.log /tmp/seaweedfs-kms-s3.json /tmp/weed-kms-mini.pid
|
||||
|
||||
# Run KMS provider Go integration tests (GitHub Actions compatible)
|
||||
test-provider-ci: build-weed ## Run KMS provider tests with OpenBao in CI
|
||||
@echo "$(YELLOW)Starting KMS provider integration tests...$(NC)"
|
||||
@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
|
||||
echo "$(GREEN)OpenBao started and configured$(NC)"; \
|
||||
trap '$(MAKE) -C $(TEST_DIR) stop-openbao-ci || true' EXIT; \
|
||||
cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/... || exit 1; \
|
||||
echo "$(GREEN)KMS provider tests completed successfully$(NC)"; \
|
||||
else \
|
||||
echo "$(RED)Failed to start OpenBao$(NC)"; \
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Run S3 KMS end-to-end tests (GitHub Actions compatible)
|
||||
test-s3-kms-ci: build-weed ## Run S3 KMS e2e tests with weed mini + OpenBao in CI
|
||||
@echo "$(YELLOW)Starting S3 KMS end-to-end tests...$(NC)"
|
||||
@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
|
||||
echo "$(GREEN)OpenBao started and configured$(NC)"; \
|
||||
trap 'docker logs openbao-ci > /tmp/openbao-ci-container.log 2>&1 || true; $(MAKE) -C $(TEST_DIR) stop-seaweedfs-ci || true; $(MAKE) -C $(TEST_DIR) stop-openbao-ci || true; $(MAKE) -C $(TEST_DIR) clean-ci || true' EXIT; \
|
||||
if $(MAKE) start-seaweedfs-ci > /tmp/weed-kms-ci-setup.log 2>&1; then \
|
||||
echo "$(GREEN)SeaweedFS started with OpenBao KMS$(NC)"; \
|
||||
sleep 3; \
|
||||
chmod +x test_s3_kms.sh; \
|
||||
SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) ACCESS_KEY=$(ACCESS_KEY) SECRET_KEY=$(SECRET_KEY) ./test_s3_kms.sh || exit 1; \
|
||||
echo "$(GREEN)S3 KMS e2e tests completed successfully$(NC)"; \
|
||||
else \
|
||||
echo "$(RED)Failed to start SeaweedFS$(NC)"; \
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || true; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
else \
|
||||
echo "$(RED)Failed to start OpenBao$(NC)"; \
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
27
test/kms/s3-config-openbao-template.json
Normal file
27
test/kms/s3-config-openbao-template.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "ACCESS_KEY_PLACEHOLDER",
|
||||
"secretKey": "SECRET_KEY_PLACEHOLDER"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
}
|
||||
],
|
||||
"kms": {
|
||||
"default_provider": "openbao",
|
||||
"providers": {
|
||||
"openbao": {
|
||||
"type": "openbao",
|
||||
"address": "OPENBAO_ADDR_PLACEHOLDER",
|
||||
"token": "OPENBAO_TOKEN_PLACEHOLDER",
|
||||
"transit_path": "transit",
|
||||
"cache_enabled": true,
|
||||
"cache_ttl": "20m"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -365,14 +365,20 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
}
|
||||
}
|
||||
|
||||
// If SSE-S3 was applied by bucket default, prepare metadata (if not already done)
|
||||
if sseS3Key != nil && len(sseS3Metadata) == 0 {
|
||||
// If bucket default encryption was applied, serialize the metadata (SSE-S3 and SSE-KMS are mutually exclusive)
|
||||
var metaErr error
|
||||
if sseS3Key != nil && len(sseS3Metadata) == 0 {
|
||||
sseS3Metadata, metaErr = SerializeSSES3Metadata(sseS3Key)
|
||||
if metaErr != nil {
|
||||
glog.Errorf("Failed to serialize SSE-S3 metadata for bucket default encryption: %v", metaErr)
|
||||
return "", s3err.ErrInternalError, SSEResponseMetadata{}
|
||||
}
|
||||
} else if sseKMSKey != nil && len(sseKMSMetadata) == 0 {
|
||||
sseKMSMetadata, metaErr = SerializeSSEKMSMetadata(sseKMSKey)
|
||||
if metaErr != nil {
|
||||
glog.Errorf("Failed to serialize SSE-KMS metadata for bucket default encryption: %v", metaErr)
|
||||
return "", s3err.ErrInternalError, SSEResponseMetadata{}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption")
|
||||
|
||||
Reference in New Issue
Block a user