s3api: make conditional mutations atomic and AWS-compatible (#8802)
* s3api: serialize conditional write finalization * s3api: add conditional delete mutation checks * s3api: enforce destination conditions for copy * s3api: revalidate multipart completion under lock * s3api: rollback failed put finalization hooks * s3api: report delete-marker version deletions * s3api: fix copy destination versioning edge cases * s3api: make versioned multipart completion idempotent * test/s3: cover conditional mutation regressions * s3api: rollback failed copy version finalization * s3api: resolve suspended delete conditions via latest entry * s3api: remove copy test null-version injection * s3api: reject out-of-order multipart completions * s3api: preserve multipart replay version metadata * s3api: surface copy destination existence errors * s3api: simplify delete condition target resolution * test/s3: make conditional delete assertions order independent * test/s3: add distributed lock gateway integration * s3api: fail closed multipart versioned completion * s3api: harden copy metadata and overwrite paths * s3api: create delete markers for suspended deletes * s3api: allow duplicate multipart completion parts
This commit is contained in:
Chris Lu
147
test/s3/versioning/s3_copy_versioning_regression_test.go
Normal file
147
test/s3/versioning/s3_copy_versioning_regression_test.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func versioningCopySource(bucketName, key string) string {
|
||||
return fmt.Sprintf("%s/%s", bucketName, url.PathEscape(key))
|
||||
}
|
||||
|
||||
func suspendVersioning(t *testing.T, client *s3.Client, bucketName string) {
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusSuspended,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestVersioningSelfCopyMetadataReplaceCreatesNewVersion(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
||||
|
||||
objectKey := "self-copy-versioned.txt"
|
||||
initialContent := []byte("copy me without changing the body")
|
||||
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(initialContent),
|
||||
Metadata: map[string]string{"stage": "one"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
copyResp, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
Metadata: map[string]string{"stage": "two"},
|
||||
MetadataDirective: types.MetadataDirectiveReplace,
|
||||
})
|
||||
require.NoError(t, err, "Self-copy with metadata replacement should succeed")
|
||||
require.NotNil(t, copyResp.VersionId, "Versioned self-copy should create a new version")
|
||||
require.NotEqual(t, *putResp.VersionId, *copyResp.VersionId, "Self-copy should create a distinct version")
|
||||
|
||||
headLatestResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "two", headLatestResp.Metadata["stage"], "Latest version should expose replaced metadata")
|
||||
|
||||
headOriginalResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "one", headOriginalResp.Metadata["stage"], "Previous version metadata should remain intact")
|
||||
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, initialContent, body, "Self-copy should not alter the object body")
|
||||
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, versionsResp.Versions, 2, "Self-copy should append a new current version")
|
||||
assert.Equal(t, *copyResp.VersionId, *versionsResp.Versions[0].VersionId, "New copy version should be latest")
|
||||
}
|
||||
|
||||
func TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
suspendVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusSuspended)
|
||||
|
||||
objectKey := "self-copy-suspended.txt"
|
||||
initialContent := []byte("null version content")
|
||||
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(initialContent),
|
||||
Metadata: map[string]string{"stage": "one"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
copyResp, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
Metadata: map[string]string{"stage": "two"},
|
||||
MetadataDirective: types.MetadataDirectiveReplace,
|
||||
})
|
||||
require.NoError(t, err, "Suspended self-copy with metadata replacement should succeed")
|
||||
assert.Nil(t, copyResp.VersionId, "Suspended versioning should not return a version header for the current null version")
|
||||
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "two", headResp.Metadata["stage"], "Null current version should be updated in place")
|
||||
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, versionsResp.Versions, 1, "Suspended self-copy should keep a single null current version")
|
||||
require.NotNil(t, versionsResp.Versions[0].VersionId)
|
||||
assert.Equal(t, "null", *versionsResp.Versions[0].VersionId, "Suspended self-copy should preserve null-version semantics")
|
||||
assert.True(t, *versionsResp.Versions[0].IsLatest, "Null version should remain latest")
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSuspendedDeleteCreatesDeleteMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "suspended-delete-marker.txt"
|
||||
versionedResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("versioned-content")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, versionedResp.VersionId)
|
||||
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("null-version-content")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, deleteResp.DeleteMarker)
|
||||
assert.True(t, *deleteResp.DeleteMarker)
|
||||
require.NotNil(t, deleteResp.VersionId)
|
||||
|
||||
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, listResp.DeleteMarkers, 1)
|
||||
|
||||
deleteMarker := listResp.DeleteMarkers[0]
|
||||
require.NotNil(t, deleteMarker.Key)
|
||||
assert.Equal(t, objectKey, *deleteMarker.Key)
|
||||
require.NotNil(t, deleteMarker.VersionId)
|
||||
assert.Equal(t, *deleteResp.VersionId, *deleteMarker.VersionId)
|
||||
require.NotNil(t, deleteMarker.IsLatest)
|
||||
assert.True(t, *deleteMarker.IsLatest)
|
||||
|
||||
_, err = client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
getVersionedResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: versionedResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getVersionedResp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(getVersionedResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "versioned-content", string(body))
|
||||
}
|
||||
@@ -499,12 +499,16 @@ func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
|
||||
t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
|
||||
|
||||
// Delete the delete marker to "undelete" the object
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
undeleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(deleteMarkerVersionId),
|
||||
})
|
||||
require.NoError(t, err, "Failed to delete the delete marker")
|
||||
require.NotNil(t, undeleteResp.DeleteMarker, "Deleting a delete marker version should report DeleteMarker=true")
|
||||
assert.True(t, *undeleteResp.DeleteMarker, "Deleting a delete marker version should report DeleteMarker=true")
|
||||
require.NotNil(t, undeleteResp.VersionId, "Deleting a delete marker version should echo the version ID")
|
||||
assert.Equal(t, deleteMarkerVersionId, *undeleteResp.VersionId, "DeleteObject should report the deleted delete-marker version ID")
|
||||
|
||||
// ListObjectsV2 should show the object again
|
||||
listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
@@ -518,3 +522,76 @@ func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
|
||||
|
||||
t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
|
||||
}
|
||||
|
||||
func TestVersioningCompleteMultipartUploadIsIdempotent(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
||||
|
||||
objectKey := "multipart-idempotent-object"
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload")
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
part1Data := bytes.Repeat([]byte("i"), partSize)
|
||||
part2Data := bytes.Repeat([]byte("j"), partSize)
|
||||
|
||||
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload first part")
|
||||
|
||||
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload second part")
|
||||
|
||||
completeInput := &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
|
||||
{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
firstCompleteResp, err := client.CompleteMultipartUpload(context.TODO(), completeInput)
|
||||
require.NoError(t, err, "First CompleteMultipartUpload should succeed")
|
||||
require.NotNil(t, firstCompleteResp.ETag)
|
||||
require.NotNil(t, firstCompleteResp.VersionId)
|
||||
|
||||
secondCompleteResp, err := client.CompleteMultipartUpload(context.TODO(), completeInput)
|
||||
require.NoError(t, err, "Repeated CompleteMultipartUpload should return the existing object instead of creating a second version")
|
||||
require.NotNil(t, secondCompleteResp.ETag)
|
||||
require.NotNil(t, secondCompleteResp.VersionId, "Repeated complete should report the existing version ID")
|
||||
assert.Equal(t, *firstCompleteResp.ETag, *secondCompleteResp.ETag, "Repeated complete should report the same ETag")
|
||||
assert.Equal(t, *firstCompleteResp.VersionId, *secondCompleteResp.VersionId, "Repeated complete should report the same version ID")
|
||||
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list object versions")
|
||||
require.Len(t, versionsResp.Versions, 1, "Repeated completion must not create a duplicate version")
|
||||
assert.Equal(t, *firstCompleteResp.VersionId, *versionsResp.Versions[0].VersionId, "The original multipart version should remain current")
|
||||
assert.Empty(t, versionsResp.DeleteMarkers, "Repeated completion should not create delete markers")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user