Fix s3 versioning listing bugs (#7705)

* fix: add pagination to list-object-versions for buckets with >1000 objects

The findVersionsRecursively() function used a fixed limit of 1000 entries
without pagination. This caused objects beyond the first 1000 entries
(sorted alphabetically) to never appear in list-object-versions responses.

Changes:
- Add pagination loop using filer.PaginationSize (1024)
- Use isLast flag from s3a.list() to detect end of pagination
- Track startFrom marker for each page

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: prevent infinite loop in ListObjects when processing .versions directories

The doListFilerEntries() function processes .versions directories in a
secondary loop after the main entry loop, but failed to update nextMarker.
This caused infinite pagination loops when results were truncated, as the
same .versions directories would be reprocessed on each page.

Bug introduced by: c196d03951
("fix listing object versions (#7006)")

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
jfburdet
2025-12-10 21:43:08 +01:00
committed by GitHub
parent c153420022
commit 27a28faf49
2 changed files with 201 additions and 186 deletions

View File

@@ -573,6 +573,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
break break
} }
// Update nextMarker to ensure pagination advances past this .versions directory
// This is critical to prevent infinite loops when results are truncated
nextMarker = versionsDir
// Extract object name from .versions directory name (remove .versions suffix) // Extract object name from .versions directory name (remove .versions suffix)
baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder) baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder)

View File

@@ -264,13 +264,18 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
// findVersionsRecursively searches for all .versions directories and regular files recursively // findVersionsRecursively searches for all .versions directories and regular files recursively
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error { func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
// List entries in current directory // List entries in current directory with pagination
entries, _, err := s3a.list(currentPath, "", "", false, 1000) startFrom := ""
for {
entries, isLast, err := s3a.list(currentPath, "", startFrom, false, filer.PaginationSize)
if err != nil { if err != nil {
return err return err
} }
for _, entry := range entries { for _, entry := range entries {
// Track last entry name for pagination
startFrom = entry.Name
entryPath := path.Join(relativePath, entry.Name) entryPath := path.Join(relativePath, entry.Name)
// Skip if this doesn't match the prefix filter // Skip if this doesn't match the prefix filter
@@ -486,6 +491,12 @@ func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string
} }
} }
// If we've reached the last page, stop pagination
if isLast {
break
}
}
return nil return nil
} }