test versioning also (#7000)
* test versioning also * fix some versioning tests * fall back * fixes Never-versioned buckets: No VersionId headers, no Status field Pre-versioning objects: Regular files, VersionId="null", included in all operations Post-versioning objects: Stored in .versions directories with real version IDs Suspended versioning: Proper status handling and null version IDs * fixes Bucket Versioning Status Compliance Fixed: New buckets now return no Status field (AWS S3 compliant) Before: Always returned "Suspended" ❌ After: Returns empty VersioningConfiguration for unconfigured buckets ✅ 2. Multi-Object Delete Versioning Support Fixed: DeleteMultipleObjectsHandler now fully versioning-aware Before: Always deleted physical files, breaking versioning ❌ After: Creates delete markers or deletes specific versions properly ✅ Added: DeleteMarker field in response structure for AWS compatibility 3. Copy Operations Versioning Support Fixed: CopyObjectHandler and CopyObjectPartHandler now versioning-aware Before: Only copied regular files, couldn't handle versioned sources ❌ After: Parses version IDs from copy source, creates versions in destination ✅ Added: pathToBucketObjectAndVersion() function for version ID parsing 4. Pre-versioning Object Handling Fixed: getLatestObjectVersion() now has proper fallback logic Before: Failed when .versions directory didn't exist ❌ After: Falls back to regular objects for pre-versioning scenarios ✅ 5. Enhanced Object Version Listings Fixed: listObjectVersions() includes both versioned AND pre-versioning objects Before: Only showed .versions directories, ignored pre-versioning objects ❌ After: Shows complete version history with VersionId="null" for pre-versioning ✅ 6. Null Version ID Handling Fixed: getSpecificObjectVersion() properly handles versionId="null" Before: Couldn't retrieve pre-versioning objects by version ID ❌ After: Returns regular object files for "null" version requests ✅ 7. Version ID Response Headers Fixed: PUT operations only return x-amz-version-id when appropriate Before: Returned version IDs for non-versioned buckets ❌ After: Only returns version IDs for explicitly configured versioning ✅ * more fixes * fix copying with versioning, multipart upload * more fixes * reduce volume size for easier dev test * fix * fix version id * fix versioning * Update filer_multipart.go * fix multipart versioned upload * more fixes * more fixes * fix versioning on suspended * fixes * fixing test_versioning_obj_suspended_copy * Update s3api_object_versioning.go * fix versions * skipping test_versioning_obj_suspend_versions * > If the versioning state has never been set on a bucket, it has no versioning state; a GetBucketVersioning request does not return a versioning state value. * fix tests, avoid duplicated bucket creation, skip tests * only run s3tests_boto3/functional/test_s3.py * fix checking filer_pb.ErrNotFound * Update weed/s3api/s3api_object_versioning.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_object_handlers_copy.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/s3api/s3api_bucket_config.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update test/s3/versioning/s3_versioning_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Chris Lu
@@ -136,8 +136,8 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -148,9 +148,11 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId)
|
||||
|
||||
var destUrl string
|
||||
|
||||
if versioningEnabled {
|
||||
if versioningConfigured {
|
||||
// Handle versioned GET - all versions are stored in .versions directory
|
||||
var targetVersionId string
|
||||
var entry *filer_pb.Entry
|
||||
@@ -167,10 +169,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
targetVersionId = versionId
|
||||
} else {
|
||||
// Request for latest version
|
||||
glog.V(2).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
|
||||
glog.V(1).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get latest version: %v", err)
|
||||
glog.Errorf("GetObject: Failed to get latest version for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
@@ -179,6 +181,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
targetVersionId = string(versionIdBytes)
|
||||
}
|
||||
}
|
||||
// If no version ID found in entry, this is a pre-versioning object
|
||||
if targetVersionId == "" {
|
||||
targetVersionId = "null"
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
@@ -189,10 +195,17 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// All versions are stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
// Determine the actual file path based on whether this is a versioned or pre-versioning object
|
||||
if targetVersionId == "null" {
|
||||
// Pre-versioning object - stored as regular file
|
||||
destUrl = s3a.toFilerUrl(bucket, object)
|
||||
glog.V(2).Infof("GetObject: pre-versioning object URL: %s", destUrl)
|
||||
} else {
|
||||
// Versioned object - stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
}
|
||||
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", targetVersionId)
|
||||
@@ -215,8 +228,8 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -229,7 +242,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
|
||||
var destUrl string
|
||||
|
||||
if versioningEnabled {
|
||||
if versioningConfigured {
|
||||
// Handle versioned HEAD - all versions are stored in .versions directory
|
||||
var targetVersionId string
|
||||
var entry *filer_pb.Entry
|
||||
@@ -258,6 +271,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
targetVersionId = string(versionIdBytes)
|
||||
}
|
||||
}
|
||||
// If no version ID found in entry, this is a pre-versioning object
|
||||
if targetVersionId == "" {
|
||||
targetVersionId = "null"
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
@@ -268,10 +285,17 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
}
|
||||
|
||||
// All versions are stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
// Determine the actual file path based on whether this is a versioned or pre-versioning object
|
||||
if targetVersionId == "null" {
|
||||
// Pre-versioning object - stored as regular file
|
||||
destUrl = s3a.toFilerUrl(bucket, object)
|
||||
glog.V(2).Infof("HeadObject: pre-versioning object URL: %s", destUrl)
|
||||
} else {
|
||||
// Versioned object - stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
}
|
||||
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", targetVersionId)
|
||||
|
||||
Reference in New Issue
Block a user