Fix s3 versioning listing bugs (#7705)
* fix: add pagination to list-object-versions for buckets with >1000 objects
The findVersionsRecursively() function used a fixed limit of 1000 entries
without pagination. This caused objects beyond the first 1000 entries
(sorted alphabetically) to never appear in list-object-versions responses.
Changes:
- Add pagination loop using filer.PaginationSize (1024)
- Use isLast flag from s3a.list() to detect end of pagination
- Track startFrom marker for each page
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
* fix: prevent infinite loop in ListObjects when processing .versions directories
The doListFilerEntries() function processes .versions directories in a
secondary loop after the main entry loop, but failed to update nextMarker.
This caused infinite pagination loops when results were truncated, as the
same .versions directories would be reprocessed on each page.
Bug introduced by: c196d03951
("fix listing object versions (#7006)")
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -573,6 +573,10 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update nextMarker to ensure pagination advances past this .versions directory
|
||||||
|
// This is critical to prevent infinite loops when results are truncated
|
||||||
|
nextMarker = versionsDir
|
||||||
|
|
||||||
// Extract object name from .versions directory name (remove .versions suffix)
|
// Extract object name from .versions directory name (remove .versions suffix)
|
||||||
baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder)
|
baseObjectName := strings.TrimSuffix(versionsDir, s3_constants.VersionsFolder)
|
||||||
|
|
||||||
|
|||||||
@@ -264,13 +264,18 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
|
|||||||
|
|
||||||
// findVersionsRecursively searches for all .versions directories and regular files recursively
|
// findVersionsRecursively searches for all .versions directories and regular files recursively
|
||||||
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
|
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
|
||||||
// List entries in current directory
|
// List entries in current directory with pagination
|
||||||
entries, _, err := s3a.list(currentPath, "", "", false, 1000)
|
startFrom := ""
|
||||||
|
for {
|
||||||
|
entries, isLast, err := s3a.list(currentPath, "", startFrom, false, filer.PaginationSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
|
// Track last entry name for pagination
|
||||||
|
startFrom = entry.Name
|
||||||
|
|
||||||
entryPath := path.Join(relativePath, entry.Name)
|
entryPath := path.Join(relativePath, entry.Name)
|
||||||
|
|
||||||
// Skip if this doesn't match the prefix filter
|
// Skip if this doesn't match the prefix filter
|
||||||
@@ -486,6 +491,12 @@ func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If we've reached the last page, stop pagination
|
||||||
|
if isLast {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user