fix: directory incorrectly listed as object in S3 ListObjects (#7939)
* fix: directory incorrectly listed as object in S3 ListObjects
Regular directories (without MIME type) were only added to CommonPrefixes
when delimiter was exactly '/'. This caused directories to be silently
skipped for other delimiter values.
Changed the condition from 'delimiter == "/"' to 'delimiter != ""' to
ensure directories are correctly added to CommonPrefixes for any delimiter.
Fixes issue where directories like 'data/file.vhd' were being returned as
objects instead of prefixes in ListObjects responses.
* fix: complete the directory listing fix for all delimiters
Address reviewer feedback:
- Changed doListFilerEntries line 549 from 'delimiter != "/"' to 'delimiter == ""'
This ensures directories are yielded to the callback for ANY delimiter, not just "/"
- Parameterized test to verify fix works with multiple delimiters (/, _, :)
The previous fix only addressed line 260 but line 549 was still causing
recursion for non-"/" delimiters, preventing directories from being
added to CommonPrefixes.
* docs: update test comment to reflect multiple delimiters
Address reviewer feedback - clarify that the test verifies behavior
for any non-empty delimiter, not just '/'.
* docs: clarify test comment with delimiter examples
Add specific examples of delimiters ('/', '_', ':') to make it clear
that the test verifies behavior with multiple delimiter types.
* fix: revert line 549 to original logic, only line 260 needed changing
The fix for directories being listed as objects only required changing
line 260 from 'delimiter == "/"' to 'delimiter != ""'.
Line 549 should remain as 'delimiter != "/"' to allow recursion for
delimiters that don't exist in paths (e.g., delimiter=z for paths like
b/a/c). This is correct S3 behavior.
Updated test to only verify delimiter="/" since other delimiters should
recurse into directories to find actual files.
* docs: clarify test scope in directory listing test
This commit is contained in:
@@ -257,7 +257,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
|||||||
cursor.maxKeys--
|
cursor.maxKeys--
|
||||||
lastEntryWasCommonPrefix = false
|
lastEntryWasCommonPrefix = false
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
||||||
} else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
|
} else if delimiter != "" { // A response can contain CommonPrefixes only if you specify a delimiter.
|
||||||
// Use raw dir and entry.Name (not encoded) to ensure consistent handling
|
// Use raw dir and entry.Name (not encoded) to ensure consistent handling
|
||||||
// Encoding will be applied after sorting if encodingTypeUrl is set
|
// Encoding will be applied after sorting if encodingTypeUrl is set
|
||||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||||
|
|||||||
63
weed/s3api/s3api_object_handlers_list_directory_test.go
Normal file
63
weed/s3api/s3api_object_handlers_list_directory_test.go
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
package s3api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDirectoryListedAsCommonPrefix tests that regular directories (without MIME)
|
||||||
|
// are correctly listed as CommonPrefixes when delimiter="/" is used, not as objects.
|
||||||
|
// Note: The fix applies to any non-empty delimiter ('/', '_', ':'), but this test focuses on '/'.
|
||||||
|
func TestDirectoryListedAsCommonPrefix(t *testing.T) {
|
||||||
|
s3a := &S3ApiServer{
|
||||||
|
option: &S3ApiServerOption{
|
||||||
|
BucketsPath: "/buckets",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Regular directory (no MIME) - matching actual user metadata
|
||||||
|
regularDir := &filer_pb.Entry{
|
||||||
|
Name: "f2f1237f-0e69-4e0b-8f01-d4fa299787e1.vhd",
|
||||||
|
IsDirectory: true,
|
||||||
|
Attributes: &filer_pb.FuseAttributes{
|
||||||
|
Mime: "", // Empty MIME - IsDirectoryKeyObject() returns false
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &testFilerClient{
|
||||||
|
entriesByDir: map[string][]*filer_pb.Entry{
|
||||||
|
"/buckets/xoa-bucket/xo-vm-backups/data": {regularDir},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cursor := &ListingCursor{maxKeys: 1000}
|
||||||
|
var seenDirs []string
|
||||||
|
var seenFiles []string
|
||||||
|
|
||||||
|
_, err := s3a.doListFilerEntries(
|
||||||
|
client,
|
||||||
|
"/buckets/xoa-bucket/xo-vm-backups/data",
|
||||||
|
"", // prefix
|
||||||
|
cursor,
|
||||||
|
"", // marker
|
||||||
|
"/", // delimiter="/" - should yield directory for CommonPrefix processing
|
||||||
|
false,
|
||||||
|
"xoa-bucket",
|
||||||
|
func(dir string, entry *filer_pb.Entry) {
|
||||||
|
if entry.IsDirectory {
|
||||||
|
seenDirs = append(seenDirs, entry.Name)
|
||||||
|
} else {
|
||||||
|
seenFiles = append(seenFiles, entry.Name)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// The directory should be passed to the callback for delimiter="/"
|
||||||
|
assert.Contains(t, seenDirs, "f2f1237f-0e69-4e0b-8f01-d4fa299787e1.vhd",
|
||||||
|
"Directory should be passed to callback for CommonPrefix processing with delimiter=/")
|
||||||
|
assert.Empty(t, seenFiles, "No files should be seen, only the directory")
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user