fix: reduce N+1 queries in S3 versioned object list operations (#7814)

* fix: achieve single-scan efficiency for S3 versioned object listing

When listing objects in a versioning-enabled bucket, the original code
triggered multiple getEntry calls per versioned object (up to 12 with
retries), causing excessive 'find' operations visible in Grafana and
leading to high memory usage.

This fix achieves single-scan efficiency by caching list metadata
(size, ETag, mtime, owner) directly in the .versions directory:

1. Add new Extended keys for caching list metadata in .versions dir
2. Update upload/copy/multipart paths to cache metadata when creating versions
3. Update getLatestVersionEntryFromDirectoryEntry to use cached metadata
   (zero getEntry calls when cache is available)
4. Update updateLatestVersionAfterDeletion to maintain cache consistency

Performance improvement for N versioned objects:
- Before: N×1 to N×12 find operations per list request
- After: 0 extra find operations (all metadata from single scan)

This matches the efficiency of normal (non-versioned) object listing.

* Update s3api_object_versioning.go

* s3api: fix ETag handling for versioned objects and simplify delete marker creation

- Add Md5 attribute to synthetic logicalEntry for single-part uploads to ensure
  filer.ETag() returns correct value in ListObjects response
- Simplify delete marker creation by initializing entry directly in mkFile callback
- Add bytes and encoding/hex imports for ETag parsing

* s3api: preserve default attributes in delete marker mkFile callback

Only modify Mtime field instead of replacing the entire Attributes struct,
preserving default values like Crtime, FileMode, Uid, and Gid that mkFile
initializes.

* s3api: fix ETag handling in newListEntry for multipart uploads

Prioritize ExtETagKey from Extended attributes before falling back to
filer.ETag(). This properly handles multipart upload ETags (format: md5-parts)
for versioned objects, where the synthetic entry has cached ETag metadata
but no chunks to calculate from.

* s3api: reduce code duplication in delete marker creation

Extract deleteMarkerExtended map to be reused in both mkFile callback
and deleteMarkerEntry construction.

* test: add multipart upload versioning tests for ETag verification

Add tests to verify that multipart uploaded objects in versioned buckets
have correct ETags when listed:

- TestMultipartUploadVersioningListETag: Basic multipart upload with 2 parts
- TestMultipartUploadMultipleVersionsListETag: Multiple multipart versions
- TestMixedSingleAndMultipartVersionsListETag: Mix of single-part and multipart

These tests cover a bug where synthetic entries for versioned objects
didn't include proper ETag handling for multipart uploads.

* test: add delete marker test for multipart uploaded versioned objects

TestMultipartUploadDeleteMarkerListBehavior verifies:
- Delete marker creation hides object from ListObjectsV2
- ListObjectVersions shows both version and delete marker
- Version ETag (multipart format) is preserved after delete marker
- Object can be accessed by version ID after delete marker
- Removing delete marker restores object visibility

* refactor: address code review feedback

- test: use assert.ElementsMatch for ETag verification (more idiomatic)
- s3api: optimize newListEntry ETag logic (check ExtETagKey first)
- s3api: fix edge case in ETag parsing (>= 2 instead of > 2)

* s3api: prevent stale cached metadata and preserve existing extended attrs

- setCachedListMetadata: clear old cached keys before setting new values
  to prevent stale data when new version lacks certain fields (e.g., owner)
- createDeleteMarker: merge extended attributes instead of overwriting
  to preserve any existing metadata on the entry

* s3api: extract clearCachedVersionMetadata to reduce code duplication

- clearCachedVersionMetadata: clears only metadata fields (size, mtime, etag, owner, deleteMarker)
- clearCachedListMetadata: now reuses clearCachedVersionMetadata + clears ID/filename
- setCachedListMetadata: uses clearCachedVersionMetadata (not clearCachedListMetadata
  because caller has already set ID/filename)

* s3api: share timestamp between version entry and cache entry

Capture versionMtime once before mkFile and reuse for both:
- versionEntry.Attributes.Mtime in the mkFile callback
- versionEntryForCache.Attributes.Mtime for list caching

This keeps list vs. HEAD LastModified timestamps aligned.

* s3api: remove amzAccountId variable shadowing in multipart upload

Extract amzAccountId before mkFile callback and reuse in both places,
similar to how versionMtime is handled. Avoids confusion from
redeclaring the same variable.

weed/s3api/s3api_object_handlers_list.go

@@ -3,11 +3,11 @@ package s3api
 import (
 	"context"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -491,7 +491,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 	}
 
 	// Track .versions directories found in this directory for later processing
-	var versionsDirs []string
+	// Store the full entry to avoid additional getEntry calls (N+1 query optimization)
+	var versionsDirs []*filer_pb.Entry
 
 	for {
 		resp, recvErr := stream.Recv()
@@ -528,9 +529,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 			}
 
 			// Skip .versions directories in regular list operations but track them for logical object creation
+			// Store the full entry to avoid additional getEntry calls later
 			if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
 				glog.V(4).Infof("Found .versions directory: %s", entry.Name)
-				versionsDirs = append(versionsDirs, entry.Name)
+				versionsDirs = append(versionsDirs, entry)
 				continue
 			}
 
@@ -568,6 +570,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 
 	// After processing all regular entries, handle versioned objects
 	// Create logical entries for objects that have .versions directories
+	// OPTIMIZATION: Use the already-fetched .versions directory entry to avoid N+1 queries
 	for _, versionsDir := range versionsDirs {
 		if cursor.maxKeys <= 0 {
 			cursor.isTruncated = true
@@ -576,10 +579,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 
 		// Update nextMarker to ensure pagination advances past this .versions directory
 		// This is critical to prevent infinite loops when results are truncated
-		nextMarker = versionsDir
+		nextMarker = versionsDir.Name
 
 		// Extract object name from .versions directory name (remove .versions suffix)
-		baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder)
+		baseObjectName := strings.TrimSuffix(versionsDir.Name, s3_constants.VersionsFolder)
 
 		// Construct full object path relative to bucket
 		// dir is something like "/buckets/sea-test-1/Veeam/Backup/vbr/Config"
@@ -602,12 +605,17 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 		glog.V(4).Infof("Processing versioned object: baseObjectName=%s, bucketRelativePath=%s, fullObjectPath=%s",
 			baseObjectName, bucketRelativePath, fullObjectPath)
 
-		// Get the latest version information for this object
-		if latestVersionEntry, latestVersionErr := s3a.getLatestVersionEntryForListOperation(bucketName, fullObjectPath); latestVersionErr == nil {
+		// OPTIMIZATION: Use metadata from the already-fetched .versions directory entry
+		// This avoids additional getEntry calls which cause high "find" usage
+		if latestVersionEntry, err := s3a.getLatestVersionEntryFromDirectoryEntry(bucketName, fullObjectPath, versionsDir); err == nil {
 			glog.V(4).Infof("Creating logical entry for versioned object: %s", fullObjectPath)
 			eachEntryFn(dir, latestVersionEntry)
+		} else if errors.Is(err, ErrDeleteMarker) {
+			// Expected: latest version is a delete marker, object should not appear in list
+			glog.V(4).Infof("Skipping versioned object %s: delete marker", fullObjectPath)
 		} else {
-			glog.V(4).Infof("Failed to get latest version for %s: %v", fullObjectPath, latestVersionErr)
+			// Unexpected failure: missing metadata, fetch error, etc.
+			glog.V(3).Infof("Skipping versioned object %s due to error: %v", fullObjectPath, err)
 		}
 	}
 
@@ -712,36 +720,6 @@ func (s3a *S3ApiServer) ensureDirectoryAllEmpty(filerClient filer_pb.SeaweedFile
 	return true, nil
 }
 
-// getLatestVersionEntryForListOperation gets the latest version of an object and creates a logical entry for list operations
-// This is used to show versioned objects as logical object names in regular list operations
-func (s3a *S3ApiServer) getLatestVersionEntryForListOperation(bucket, object string) (*filer_pb.Entry, error) {
-	// Get the latest version entry
-	latestVersionEntry, err := s3a.getLatestObjectVersion(bucket, object)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get latest version: %w", err)
-	}
-
-	// Check if this is a delete marker (should not be shown in regular list)
-	if latestVersionEntry.Extended != nil {
-		if deleteMarker, exists := latestVersionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
-			return nil, fmt.Errorf("latest version is a delete marker")
-		}
-	}
-
-	// Create a logical entry that appears to be stored at the object path (not the versioned path)
-	// This allows the list operation to show the logical object name while preserving all metadata
-	// Use path.Base to get just the filename, since the entry.Name should be the local name only
-	// (the directory path is already included in the 'dir' parameter passed to eachEntryFn)
-	logicalEntry := &filer_pb.Entry{
-		Name:        path.Base(object),
-		IsDirectory: false,
-		Attributes:  latestVersionEntry.Attributes,
-		Extended:    latestVersionEntry.Extended,
-		Chunks:      latestVersionEntry.Chunks,
-	}
-
-	return logicalEntry, nil
-}
 
 // compareWithDelimiter compares two strings for sorting, treating the delimiter character
 // as having lower precedence than other characters to match AWS S3 behavior.