s3api: ensure MD5 is calculated or reused during CopyObject (#8163)

* s3api: ensure MD5 is calculated or reused during CopyObject

Fixes #8155
- Capture and reuse source MD5 for direct copies
- Calculate MD5 for small inline objects during copy

* s3api: refactor encryption logic and safe MD5 copying

- Extract duplicated bucket default encryption logic into helper
- Use safe append copy for MD5 slice to avoid shared modifications

* refactor

* avoids unnecessary MD5 recalculations for small files
This commit is contained in:
Chris Lu
2026-01-29 12:53:38 -08:00
committed by GitHub
parent 4d513a2b3d
commit 8b61fd77b5
2 changed files with 45 additions and 14 deletions

View File

@@ -155,6 +155,20 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
return
}
// Determine whether we can reuse the source MD5 (direct copy without encryption changes).
canReuseSourceMd5 := false
var sourceMd5 []byte
if entry.Attributes != nil && len(entry.Attributes.Md5) > 0 {
sourceMd5 = append([]byte(nil), entry.Attributes.Md5...)
srcPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, srcBucket, srcObject)
dstPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, dstBucket, dstObject)
state := DetectEncryptionStateWithEntry(entry, r, srcPath, dstPath)
s3a.applyCopyBucketDefaultEncryption(state, dstBucket)
if strategy, err := DetermineUnifiedCopyStrategy(state, entry.Extended, r); err == nil && strategy == CopyStrategyDirect {
canReuseSourceMd5 = true
}
}
// Create new entry for destination
dstEntry := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{
@@ -237,9 +251,17 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
}
}
if dstEntry.Attributes != nil {
if len(dstEntry.Attributes.Md5) == 0 && canReuseSourceMd5 {
dstEntry.Attributes.Md5 = append([]byte(nil), sourceMd5...)
} else if uint64(len(dstEntry.Content)) == dstEntry.Attributes.FileSize {
dstEntry.Attributes.Md5 = util.Md5(dstEntry.Content)
}
}
} else {
// Use unified copy strategy approach
dstChunks, dstMetadata, copyErr := s3a.executeUnifiedCopyStrategy(entry, r, dstBucket, srcObject, dstObject)
dstChunks, dstMetadata, copyErr := s3a.executeUnifiedCopyStrategy(entry, r, srcBucket, dstBucket, srcObject, dstObject)
if copyErr != nil {
glog.Errorf("CopyObjectHandler unified copy error: %v", copyErr)
// Map errors to appropriate S3 errors
@@ -257,6 +279,10 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
}
glog.V(2).Infof("Applied %d destination metadata entries for copy: %s", len(dstMetadata), r.URL.Path)
}
if dstEntry.Attributes != nil && len(dstEntry.Attributes.Md5) == 0 && canReuseSourceMd5 {
dstEntry.Attributes.Md5 = append([]byte(nil), sourceMd5...)
}
}
// Check if destination bucket has versioning enabled