ci: add S3 mutation regression coverage (#8804)

* test/s3: stabilize distributed lock regression

* ci: add S3 mutation regression workflow

* test: fix delete regression readiness probe

* test: address mutation regression review comments

.github/workflows/s3-mutation-regression-tests.yml

@@ -0,0 +1,133 @@
+name: "S3 Mutation Regression Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/s3api/**'
+      - 'test/s3/delete/**'
+      - 'test/s3/distributed_lock/**'
+      - 'test/s3/versioning/**'
+      - 'test/volume_server/framework/**'
+      - 'docker/compose/s3.json'
+      - '.github/workflows/s3-mutation-regression-tests.yml'
+
+concurrency:
+  group: ${{ github.head_ref }}/s3-mutation-regression-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  s3-versioning-regressions:
+    name: S3 Versioning Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run S3 versioning regression tests
+        timeout-minutes: 20
+        working-directory: test/s3/versioning
+        run: |
+          set -x
+          make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/s3/versioning
+        run: |
+          echo "=== Server Logs ==="
+          if [ -f weed-test.log ]; then
+            tail -100 weed-test.log
+          fi
+
+      - name: Upload versioning logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-versioning-regression-logs
+          path: test/s3/versioning/weed-test*.log
+          retention-days: 3
+
+  s3-delete-regressions:
+    name: S3 Delete Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Run S3 delete regression tests
+        timeout-minutes: 15
+        working-directory: test/s3/delete
+        run: |
+          set -x
+          make test-with-server
+
+      - name: Show server logs on failure
+        if: failure()
+        working-directory: test/s3/delete
+        run: |
+          echo "=== Server Logs ==="
+          if [ -f weed-test.log ]; then
+            tail -100 weed-test.log
+          fi
+
+      - name: Upload delete logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-delete-regression-logs
+          path: test/s3/delete/weed-test*.log
+          retention-days: 3
+
+  s3-distributed-lock-regressions:
+    name: S3 Distributed Lock Regression Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build SeaweedFS
+        run: |
+          go build -o weed/weed -buildvcs=false ./weed
+
+      - name: Run distributed lock regressions
+        timeout-minutes: 25
+        env:
+          TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
+          S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
+          WEED_BINARY: ${{ github.workspace }}/weed/weed
+        run: |
+          set -x
+          mkdir -p "$TMPDIR"
+          go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
+
+      - name: Upload distributed lock logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-distributed-lock-regression-logs
+          path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
+          retention-days: 3

test/s3/delete/Makefile

@@ -0,0 +1,98 @@
+# S3 delete regression test harness
+
+.PHONY: all test help build-weed check-deps start-server stop-server test-delete test-with-server clean logs
+
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+TEST_TIMEOUT := 10m
+SERVER_DIR := ./test-volume-data
+S3_CONFIG := ./test_s3.json
+
+.DEFAULT_GOAL := help
+
+all: test
+
+test: test-with-server
+
+help:
+	@echo "S3 delete regression test harness"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  build-weed      - Build the SeaweedFS binary"
+	@echo "  start-server    - Start SeaweedFS with S3 enabled"
+	@echo "  stop-server     - Stop the test server"
+	@echo "  test-delete     - Run delete regression tests"
+	@echo "  test-with-server - Start server, run tests, stop server"
+	@echo "  logs            - Show server logs"
+	@echo "  clean           - Remove test artifacts"
+
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+
+check-deps: build-weed
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@test -f $(S3_CONFIG) || (echo "S3 config not found at $(S3_CONFIG)" && exit 1)
+
+start-server: check-deps
+	@echo "Starting SeaweedFS server for delete regression tests..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@$(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3 \
+		-s3.port=$(S3_PORT) \
+		-s3.config=$(S3_CONFIG) \
+		-master.peers=none \
+		> weed-test.log 2>&1 & echo $$! > weed-server.pid
+	@for i in $$(seq 1 60); do \
+		if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
+			echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "SeaweedFS S3 server failed to start"; \
+	echo "=== Server logs ==="; \
+	test -f weed-test.log && cat weed-test.log || true; \
+	exit 1
+
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		kill -TERM $$SERVER_PID 2>/dev/null || true; \
+		for i in $$(seq 1 20); do \
+			if ! kill -0 $$SERVER_PID >/dev/null 2>&1; then \
+				break; \
+			fi; \
+			sleep 0.1; \
+		done; \
+		if kill -0 $$SERVER_PID >/dev/null 2>&1; then \
+			kill -KILL $$SERVER_PID 2>/dev/null || true; \
+		fi; \
+		rm -f weed-server.pid; \
+	fi
+
+test-delete: check-deps
+	@echo "Running delete regression tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) .
+
+test-with-server: start-server
+	@trap "$(MAKE) stop-server" EXIT; \
+	$(MAKE) test-delete || (echo "=== Last 100 lines of server logs ===" && tail -100 weed-test.log && exit 1)
+	@$(MAKE) stop-server
+
+logs:
+	@if test -f weed-test.log; then \
+		tail -f weed-test.log; \
+	else \
+		echo "No server log file found."; \
+	fi
+
+clean:
+	@$(MAKE) stop-server
+	@rm -f weed-test.log weed-server.pid
+	@rm -rf $(SERVER_DIR)

test/s3/delete/test_s3.json

@@ -0,0 +1,20 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "admin",
+          "secretKey": "admin"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}

test/s3/distributed_lock/distributed_lock_test.go

@@ -74,13 +74,12 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str
 	t.Helper()
 
 	start := make(chan struct{})
-	results := make(chan putAttemptResult, len(clients)*2)
+	results := make(chan putAttemptResult, len(clients))
 	var wg sync.WaitGroup
 
 	for _, client := range clients {
-		for attempt := 0; attempt < 2; attempt++ {
 		wg.Add(1)
-			body := fmt.Sprintf("%s-attempt-%d", client.name, attempt)
+		body := fmt.Sprintf("%s-race", client.name)
 		go func(client s3RaceClient, body string) {
 			defer wg.Done()
 			<-start
@@ -101,7 +100,6 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str
 			}
 		}(client, body)
 	}
-	}
 
 	close(start)
 	wg.Wait()
@@ -127,7 +125,7 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str
 
 	require.Empty(t, unexpectedErrors, "unexpected race errors")
 	require.Equal(t, 1, successes, "exactly one write should win")
-	require.Equal(t, len(clients)*2-1, preconditionFailures, "all losing writes should fail with 412")
+	require.Equal(t, len(clients)-1, preconditionFailures, "all losing writes should fail with 412")
 
 	object, err := clients[0].client.GetObject(context.Background(), &s3.GetObjectInput{
 		Bucket: aws.String(bucket),

test/s3/versioning/Makefile

@@ -1,7 +1,7 @@
 # S3 API Test Makefile
 # This Makefile provides comprehensive targets for running S3 versioning tests
 
-.PHONY: help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-comprehensive test-all clean logs check-deps
+.PHONY: all test help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-regressions test-versioning-comprehensive test-all clean logs check-deps
 
 # Configuration
 WEED_BINARY := ../../../weed/weed_binary
@@ -12,6 +12,12 @@ FILER_PORT := 8888
 TEST_TIMEOUT := 10m
 TEST_PATTERN := TestVersioning
 
+.DEFAULT_GOAL := help
+
+all: test
+
+test: test-with-server
+
 # Default target
 help:
 	@echo "S3 API Test Makefile"
@@ -25,6 +31,7 @@ help:
 	@echo "  stop-server            - Stop SeaweedFS server"
 	@echo "  test-versioning        - Run all versioning tests"
 	@echo "  test-versioning-quick  - Run core versioning tests only"
+	@echo "  test-versioning-regressions - Run targeted versioning regression tests"
 	@echo "  test-versioning-simple - Run tests without server management"
 	@echo "  test-versioning-comprehensive - Run comprehensive versioning tests"
 	@echo "  test-all               - Run all S3 API tests"
@@ -179,6 +186,12 @@ test-versioning-quick: check-deps
 	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestBucketListReturnDataVersioning|TestVersioningBasicWorkflow|TestVersioningDeleteMarkers" .
 	@echo "✅ Core versioning tests completed"
 
+# Targeted regressions for conditional mutation, multipart, copy, and suspended delete flows.
+test-versioning-regressions: check-deps
+	@echo "Running targeted S3 versioning regression tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker" .
+	@echo "✅ Targeted versioning regression tests completed"
+
 # All versioning tests
 test-versioning: check-deps
 	@echo "Running all S3 versioning tests..."