0adb78bc6b
* s3api: serialize conditional write finalization * s3api: add conditional delete mutation checks * s3api: enforce destination conditions for copy * s3api: revalidate multipart completion under lock * s3api: rollback failed put finalization hooks * s3api: report delete-marker version deletions * s3api: fix copy destination versioning edge cases * s3api: make versioned multipart completion idempotent * test/s3: cover conditional mutation regressions * s3api: rollback failed copy version finalization * s3api: resolve suspended delete conditions via latest entry * s3api: remove copy test null-version injection * s3api: reject out-of-order multipart completions * s3api: preserve multipart replay version metadata * s3api: surface copy destination existence errors * s3api: simplify delete condition target resolution * test/s3: make conditional delete assertions order independent * test/s3: add distributed lock gateway integration * s3api: fail closed multipart versioned completion * s3api: harden copy metadata and overwrite paths * s3api: create delete markers for suspended deletes * s3api: allow duplicate multipart completion parts
598 lines
23 KiB
Go
598 lines
23 KiB
Go
package s3api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/md5"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestMultipartUploadVersioningListETag tests that multipart uploaded objects
|
|
// in versioned buckets have correct ETags when listed.
|
|
// This covers a bug where synthetic entries for versioned objects didn't include
|
|
// proper ETag handling for multipart uploads (ETags with format "<md5>-<parts>").
|
|
func TestMultipartUploadVersioningListETag(t *testing.T) {
|
|
client := getS3Client(t)
|
|
bucketName := getNewBucketName()
|
|
|
|
// Create bucket
|
|
createBucket(t, client, bucketName)
|
|
defer deleteBucket(t, client, bucketName)
|
|
|
|
// Enable versioning
|
|
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucketName),
|
|
VersioningConfiguration: &types.VersioningConfiguration{
|
|
Status: types.BucketVersioningStatusEnabled,
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to enable versioning")
|
|
|
|
// Create multipart upload
|
|
objectKey := "multipart-test-object"
|
|
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to create multipart upload")
|
|
|
|
uploadId := *createResp.UploadId
|
|
|
|
// Upload 2 parts (minimum 5MB per part except last)
|
|
partSize := 5 * 1024 * 1024 // 5MB
|
|
part1Data := bytes.Repeat([]byte("a"), partSize)
|
|
part2Data := bytes.Repeat([]byte("b"), partSize)
|
|
|
|
// Calculate MD5 for each part
|
|
part1MD5 := md5.Sum(part1Data)
|
|
part2MD5 := md5.Sum(part2Data)
|
|
|
|
// Upload part 1
|
|
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: aws.String(uploadId),
|
|
PartNumber: aws.Int32(1),
|
|
Body: bytes.NewReader(part1Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part 1")
|
|
|
|
// Upload part 2
|
|
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: aws.String(uploadId),
|
|
PartNumber: aws.Int32(2),
|
|
Body: bytes.NewReader(part2Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part 2")
|
|
|
|
// Complete multipart upload
|
|
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: aws.String(uploadId),
|
|
MultipartUpload: &types.CompletedMultipartUpload{
|
|
Parts: []types.CompletedPart{
|
|
{
|
|
ETag: uploadPart1Resp.ETag,
|
|
PartNumber: aws.Int32(1),
|
|
},
|
|
{
|
|
ETag: uploadPart2Resp.ETag,
|
|
PartNumber: aws.Int32(2),
|
|
},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to complete multipart upload")
|
|
|
|
// Verify the ETag from CompleteMultipartUpload has the multipart format (md5-parts)
|
|
completeETag := strings.Trim(*completeResp.ETag, "\"")
|
|
assert.Contains(t, completeETag, "-", "Multipart ETag should contain '-' (format: md5-parts)")
|
|
assert.True(t, strings.HasSuffix(completeETag, "-2"), "Multipart ETag should end with '-2' for 2 parts")
|
|
|
|
t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
|
|
t.Logf("Part 1 MD5: %x", part1MD5)
|
|
t.Logf("Part 2 MD5: %x", part2MD5)
|
|
|
|
// HeadObject should return the same ETag
|
|
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to head object")
|
|
|
|
headETag := strings.Trim(*headResp.ETag, "\"")
|
|
assert.Equal(t, completeETag, headETag, "HeadObject ETag should match CompleteMultipartUpload ETag")
|
|
|
|
// ListObjectsV2 should return the same ETag
|
|
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects")
|
|
require.Len(t, listResp.Contents, 1, "Should have exactly one object")
|
|
|
|
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
|
assert.Equal(t, completeETag, listETag, "ListObjectsV2 ETag should match CompleteMultipartUpload ETag")
|
|
assert.NotEmpty(t, listETag, "ListObjectsV2 ETag should not be empty")
|
|
|
|
t.Logf("ListObjectsV2 ETag: %s", listETag)
|
|
|
|
// ListObjectVersions should also return the correct ETag
|
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list object versions")
|
|
require.Len(t, versionsResp.Versions, 1, "Should have exactly one version")
|
|
|
|
versionETag := strings.Trim(*versionsResp.Versions[0].ETag, "\"")
|
|
assert.Equal(t, completeETag, versionETag, "ListObjectVersions ETag should match CompleteMultipartUpload ETag")
|
|
assert.NotEmpty(t, versionETag, "ListObjectVersions ETag should not be empty")
|
|
|
|
t.Logf("ListObjectVersions ETag: %s", versionETag)
|
|
}
|
|
|
|
// TestMultipartUploadMultipleVersionsListETag tests that multiple versions
|
|
// of multipart uploaded objects all have correct ETags when listed.
|
|
func TestMultipartUploadMultipleVersionsListETag(t *testing.T) {
|
|
client := getS3Client(t)
|
|
bucketName := getNewBucketName()
|
|
|
|
// Create bucket
|
|
createBucket(t, client, bucketName)
|
|
defer deleteBucket(t, client, bucketName)
|
|
|
|
// Enable versioning
|
|
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucketName),
|
|
VersioningConfiguration: &types.VersioningConfiguration{
|
|
Status: types.BucketVersioningStatusEnabled,
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to enable versioning")
|
|
|
|
objectKey := "multipart-multi-version-object"
|
|
partSize := 5 * 1024 * 1024 // 5MB
|
|
var expectedETags []string
|
|
|
|
// Create 3 versions using multipart upload
|
|
for version := 1; version <= 3; version++ {
|
|
// Create multipart upload
|
|
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to create multipart upload for version %d", version)
|
|
|
|
uploadId := *createResp.UploadId
|
|
|
|
// Create unique data for each version
|
|
partData := bytes.Repeat([]byte(fmt.Sprintf("%d", version)), partSize)
|
|
|
|
// Upload single part (still results in multipart ETag format)
|
|
uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: aws.String(uploadId),
|
|
PartNumber: aws.Int32(1),
|
|
Body: bytes.NewReader(partData),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part for version %d", version)
|
|
|
|
// Complete multipart upload
|
|
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: aws.String(uploadId),
|
|
MultipartUpload: &types.CompletedMultipartUpload{
|
|
Parts: []types.CompletedPart{
|
|
{
|
|
ETag: uploadPartResp.ETag,
|
|
PartNumber: aws.Int32(1),
|
|
},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to complete multipart upload for version %d", version)
|
|
|
|
etag := strings.Trim(*completeResp.ETag, "\"")
|
|
expectedETags = append(expectedETags, etag)
|
|
t.Logf("Version %d ETag: %s", version, etag)
|
|
}
|
|
|
|
// ListObjectVersions should return all versions with correct ETags
|
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list object versions")
|
|
require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
|
|
|
|
// Collect ETags from the listing
|
|
var listedETags []string
|
|
for _, v := range versionsResp.Versions {
|
|
etag := strings.Trim(*v.ETag, "\"")
|
|
listedETags = append(listedETags, etag)
|
|
assert.NotEmpty(t, etag, "Version ETag should not be empty")
|
|
assert.Contains(t, etag, "-", "Multipart ETag should contain '-'")
|
|
}
|
|
|
|
t.Logf("Expected ETags: %v", expectedETags)
|
|
t.Logf("Listed ETags: %v", listedETags)
|
|
|
|
// Verify all expected ETags are present (order may differ due to version ordering)
|
|
assert.ElementsMatch(t, expectedETags, listedETags, "Listed ETags should match all expected ETags")
|
|
|
|
// Regular ListObjectsV2 should return only the latest version with correct ETag
|
|
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects")
|
|
require.Len(t, listResp.Contents, 1, "Should have exactly one object in regular listing")
|
|
|
|
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
|
// The latest version (version 3) should be the one shown
|
|
assert.Equal(t, expectedETags[2], listETag, "ListObjectsV2 should show latest version's ETag")
|
|
}
|
|
|
|
// TestMixedSingleAndMultipartVersionsListETag tests that a mix of
|
|
// single-part and multipart uploaded versions all have correct ETags.
|
|
func TestMixedSingleAndMultipartVersionsListETag(t *testing.T) {
|
|
client := getS3Client(t)
|
|
bucketName := getNewBucketName()
|
|
|
|
// Create bucket
|
|
createBucket(t, client, bucketName)
|
|
defer deleteBucket(t, client, bucketName)
|
|
|
|
// Enable versioning
|
|
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucketName),
|
|
VersioningConfiguration: &types.VersioningConfiguration{
|
|
Status: types.BucketVersioningStatusEnabled,
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to enable versioning")
|
|
|
|
objectKey := "mixed-upload-versions"
|
|
|
|
// Version 1: Regular PutObject (single-part, pure MD5 ETag)
|
|
content1 := []byte("This is version 1 content - single part upload")
|
|
putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
Body: bytes.NewReader(content1),
|
|
})
|
|
require.NoError(t, err, "Failed to put version 1")
|
|
etag1 := strings.Trim(*putResp1.ETag, "\"")
|
|
assert.NotContains(t, etag1, "-", "Single-part ETag should not contain '-'")
|
|
t.Logf("Version 1 (PutObject) ETag: %s", etag1)
|
|
|
|
// Version 2: Multipart upload
|
|
partSize := 5 * 1024 * 1024
|
|
partData := bytes.Repeat([]byte("x"), partSize)
|
|
|
|
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to create multipart upload")
|
|
|
|
uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
PartNumber: aws.Int32(1),
|
|
Body: bytes.NewReader(partData),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part")
|
|
|
|
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
MultipartUpload: &types.CompletedMultipartUpload{
|
|
Parts: []types.CompletedPart{
|
|
{
|
|
ETag: uploadPartResp.ETag,
|
|
PartNumber: aws.Int32(1),
|
|
},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to complete multipart upload")
|
|
etag2 := strings.Trim(*completeResp.ETag, "\"")
|
|
assert.Contains(t, etag2, "-", "Multipart ETag should contain '-'")
|
|
t.Logf("Version 2 (Multipart) ETag: %s", etag2)
|
|
|
|
// Version 3: Another regular PutObject
|
|
content3 := []byte("This is version 3 content - another single part upload")
|
|
putResp3, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
Body: bytes.NewReader(content3),
|
|
})
|
|
require.NoError(t, err, "Failed to put version 3")
|
|
etag3 := strings.Trim(*putResp3.ETag, "\"")
|
|
assert.NotContains(t, etag3, "-", "Single-part ETag should not contain '-'")
|
|
t.Logf("Version 3 (PutObject) ETag: %s", etag3)
|
|
|
|
// ListObjectVersions should return all 3 versions with correct ETags
|
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list object versions")
|
|
require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
|
|
|
|
var listedETags []string
|
|
for _, v := range versionsResp.Versions {
|
|
etag := strings.Trim(*v.ETag, "\"")
|
|
assert.NotEmpty(t, etag, "Version ETag should not be empty")
|
|
listedETags = append(listedETags, etag)
|
|
t.Logf("Listed version %s ETag: %s, IsLatest: %v", *v.VersionId, etag, *v.IsLatest)
|
|
}
|
|
|
|
// Verify all ETags were found
|
|
assert.ElementsMatch(t, []string{etag1, etag2, etag3}, listedETags, "Listed ETags should match all expected ETags")
|
|
|
|
// Regular ListObjectsV2 should return only the latest (version 3)
|
|
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects")
|
|
require.Len(t, listResp.Contents, 1, "Should have exactly one object")
|
|
|
|
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
|
assert.Equal(t, etag3, listETag, "ListObjectsV2 should show latest version's ETag (version 3)")
|
|
}
|
|
|
|
// TestMultipartUploadDeleteMarkerListBehavior tests that delete markers work correctly
|
|
// with multipart uploaded objects in versioned buckets.
|
|
func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
|
|
client := getS3Client(t)
|
|
bucketName := getNewBucketName()
|
|
|
|
// Create bucket
|
|
createBucket(t, client, bucketName)
|
|
defer deleteBucket(t, client, bucketName)
|
|
|
|
// Enable versioning
|
|
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucketName),
|
|
VersioningConfiguration: &types.VersioningConfiguration{
|
|
Status: types.BucketVersioningStatusEnabled,
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to enable versioning")
|
|
|
|
objectKey := "multipart-delete-marker-test"
|
|
partSize := 5 * 1024 * 1024 // 5MB
|
|
|
|
// Create multipart upload
|
|
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to create multipart upload")
|
|
|
|
// Upload 2 parts
|
|
part1Data := bytes.Repeat([]byte("a"), partSize)
|
|
part2Data := bytes.Repeat([]byte("b"), partSize)
|
|
|
|
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
PartNumber: aws.Int32(1),
|
|
Body: bytes.NewReader(part1Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part 1")
|
|
|
|
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
PartNumber: aws.Int32(2),
|
|
Body: bytes.NewReader(part2Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload part 2")
|
|
|
|
// Complete multipart upload
|
|
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
MultipartUpload: &types.CompletedMultipartUpload{
|
|
Parts: []types.CompletedPart{
|
|
{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
|
|
{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err, "Failed to complete multipart upload")
|
|
|
|
multipartETag := strings.Trim(*completeResp.ETag, "\"")
|
|
multipartVersionId := *completeResp.VersionId
|
|
t.Logf("Multipart upload completed: ETag=%s, VersionId=%s", multipartETag, multipartVersionId)
|
|
|
|
// Verify object is visible in ListObjectsV2
|
|
listBeforeDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects before delete")
|
|
require.Len(t, listBeforeDelete.Contents, 1, "Object should be visible before delete")
|
|
assert.Equal(t, multipartETag, strings.Trim(*listBeforeDelete.Contents[0].ETag, "\""),
|
|
"Listed ETag should match multipart ETag before delete")
|
|
|
|
// Delete object (creates delete marker)
|
|
deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to delete object")
|
|
require.NotNil(t, deleteResp.DeleteMarker, "Should create delete marker")
|
|
assert.True(t, *deleteResp.DeleteMarker, "DeleteMarker should be true")
|
|
require.NotNil(t, deleteResp.VersionId, "Delete marker should have version ID")
|
|
|
|
deleteMarkerVersionId := *deleteResp.VersionId
|
|
t.Logf("Delete marker created: VersionId=%s", deleteMarkerVersionId)
|
|
|
|
// ListObjectsV2 should NOT show the object anymore
|
|
listAfterDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects after delete")
|
|
assert.Empty(t, listAfterDelete.Contents, "Object should NOT be visible after delete marker")
|
|
|
|
// ListObjectVersions should show both the original version AND the delete marker
|
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list object versions")
|
|
|
|
// Should have 1 version (the multipart object)
|
|
require.Len(t, versionsResp.Versions, 1, "Should have exactly 1 version (the multipart object)")
|
|
version := versionsResp.Versions[0]
|
|
assert.Equal(t, multipartVersionId, *version.VersionId, "Version ID should match")
|
|
assert.Equal(t, multipartETag, strings.Trim(*version.ETag, "\""), "Version ETag should match multipart ETag")
|
|
assert.False(t, *version.IsLatest, "Multipart version should NOT be latest (delete marker is latest)")
|
|
|
|
// Should have 1 delete marker
|
|
require.Len(t, versionsResp.DeleteMarkers, 1, "Should have exactly 1 delete marker")
|
|
deleteMarker := versionsResp.DeleteMarkers[0]
|
|
assert.Equal(t, deleteMarkerVersionId, *deleteMarker.VersionId, "Delete marker version ID should match")
|
|
assert.True(t, *deleteMarker.IsLatest, "Delete marker should be latest")
|
|
|
|
t.Logf("ListObjectVersions: 1 version (ETag=%s), 1 delete marker", multipartETag)
|
|
|
|
// Access the specific version by version ID - should still work
|
|
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
VersionId: aws.String(multipartVersionId),
|
|
})
|
|
require.NoError(t, err, "Should be able to get object by version ID after delete marker")
|
|
defer getResp.Body.Close()
|
|
|
|
assert.Equal(t, multipartETag, strings.Trim(*getResp.ETag, "\""),
|
|
"GetObject with version ID should return correct ETag")
|
|
assert.Equal(t, int64(partSize*2), *getResp.ContentLength,
|
|
"GetObject with version ID should return correct size")
|
|
|
|
t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
|
|
|
|
// Delete the delete marker to "undelete" the object
|
|
undeleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
VersionId: aws.String(deleteMarkerVersionId),
|
|
})
|
|
require.NoError(t, err, "Failed to delete the delete marker")
|
|
require.NotNil(t, undeleteResp.DeleteMarker, "Deleting a delete marker version should report DeleteMarker=true")
|
|
assert.True(t, *undeleteResp.DeleteMarker, "Deleting a delete marker version should report DeleteMarker=true")
|
|
require.NotNil(t, undeleteResp.VersionId, "Deleting a delete marker version should echo the version ID")
|
|
assert.Equal(t, deleteMarkerVersionId, *undeleteResp.VersionId, "DeleteObject should report the deleted delete-marker version ID")
|
|
|
|
// ListObjectsV2 should show the object again
|
|
listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list objects after undelete")
|
|
require.Len(t, listAfterUndelete.Contents, 1, "Object should be visible again after removing delete marker")
|
|
assert.Equal(t, multipartETag, strings.Trim(*listAfterUndelete.Contents[0].ETag, "\""),
|
|
"Undeleted object should have correct multipart ETag")
|
|
|
|
t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
|
|
}
|
|
|
|
func TestVersioningCompleteMultipartUploadIsIdempotent(t *testing.T) {
|
|
client := getS3Client(t)
|
|
bucketName := getNewBucketName()
|
|
|
|
createBucket(t, client, bucketName)
|
|
defer deleteBucket(t, client, bucketName)
|
|
|
|
enableVersioning(t, client, bucketName)
|
|
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
|
|
|
objectKey := "multipart-idempotent-object"
|
|
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to create multipart upload")
|
|
|
|
partSize := 5 * 1024 * 1024
|
|
part1Data := bytes.Repeat([]byte("i"), partSize)
|
|
part2Data := bytes.Repeat([]byte("j"), partSize)
|
|
|
|
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
PartNumber: aws.Int32(1),
|
|
Body: bytes.NewReader(part1Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload first part")
|
|
|
|
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
PartNumber: aws.Int32(2),
|
|
Body: bytes.NewReader(part2Data),
|
|
})
|
|
require.NoError(t, err, "Failed to upload second part")
|
|
|
|
completeInput := &s3.CompleteMultipartUploadInput{
|
|
Bucket: aws.String(bucketName),
|
|
Key: aws.String(objectKey),
|
|
UploadId: createResp.UploadId,
|
|
MultipartUpload: &types.CompletedMultipartUpload{
|
|
Parts: []types.CompletedPart{
|
|
{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
|
|
{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
|
|
},
|
|
},
|
|
}
|
|
|
|
firstCompleteResp, err := client.CompleteMultipartUpload(context.TODO(), completeInput)
|
|
require.NoError(t, err, "First CompleteMultipartUpload should succeed")
|
|
require.NotNil(t, firstCompleteResp.ETag)
|
|
require.NotNil(t, firstCompleteResp.VersionId)
|
|
|
|
secondCompleteResp, err := client.CompleteMultipartUpload(context.TODO(), completeInput)
|
|
require.NoError(t, err, "Repeated CompleteMultipartUpload should return the existing object instead of creating a second version")
|
|
require.NotNil(t, secondCompleteResp.ETag)
|
|
require.NotNil(t, secondCompleteResp.VersionId, "Repeated complete should report the existing version ID")
|
|
assert.Equal(t, *firstCompleteResp.ETag, *secondCompleteResp.ETag, "Repeated complete should report the same ETag")
|
|
assert.Equal(t, *firstCompleteResp.VersionId, *secondCompleteResp.VersionId, "Repeated complete should report the same version ID")
|
|
|
|
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucketName),
|
|
Prefix: aws.String(objectKey),
|
|
})
|
|
require.NoError(t, err, "Failed to list object versions")
|
|
require.Len(t, versionsResp.Versions, 1, "Repeated completion must not create a duplicate version")
|
|
assert.Equal(t, *firstCompleteResp.VersionId, *versionsResp.Versions[0].VersionId, "The original multipart version should remain current")
|
|
assert.Empty(t, versionsResp.DeleteMarkers, "Repeated completion should not create delete markers")
|
|
}
|