fix(s3api): make ListObjectsV1 namespaced and prevent marker-echo pagination loops (#8409)

* fix(s3api): make ListObjectsV1 namespaced and stop marker-echo pagination loops

* test(s3api): harden marker-echo coverage and align V1 encoding tag

* test(s3api): cover encoded marker matching and trim redundant setup

* refactor(s3api): tighten V1 list helper visibility and test mock docs
This commit is contained in:
Plamen Nikolov
2026-02-24 09:45:08 +02:00
committed by GitHub
parent 2d65d7f499
commit ff84ef880d
2 changed files with 213 additions and 1 deletions

View File

@@ -47,6 +47,37 @@ type ListBucketResultV2 struct {
StartAfter string `xml:"StartAfter,omitempty"`
}
type listBucketResultV1 struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
Metadata []MetadataEntry `xml:"Metadata,omitempty"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker,omitempty"`
MaxKeys int `xml:"MaxKeys"`
Delimiter string `xml:"Delimiter,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
Contents []ListEntry `xml:"Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
EncodingType string `xml:"EncodingType,omitempty"`
}
func toListBucketResultV1(in ListBucketResult) listBucketResultV1 {
return listBucketResultV1{
Metadata: in.Metadata,
Name: in.Name,
Prefix: in.Prefix,
Marker: in.Marker,
NextMarker: in.NextMarker,
MaxKeys: in.MaxKeys,
Delimiter: in.Delimiter,
IsTruncated: in.IsTruncated,
Contents: in.Contents,
CommonPrefixes: in.CommonPrefixes,
EncodingType: in.EncodingType,
}
}
func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
@@ -148,6 +179,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
sanitizeV1MarkerEcho(&response, marker, encodingTypeUrl)
if len(response.Contents) == 0 {
if exists, existErr := s3a.bucketExists(bucket); existErr == nil && !exists {
@@ -157,7 +189,47 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
}
glog.V(3).Infof("ListObjectsV1Handler response: %+v", response)
writeSuccessResponseXML(w, r, response)
writeSuccessResponseXML(w, r, toListBucketResultV1(response))
}
func sanitizeV1MarkerEcho(response *ListBucketResult, marker string, encodingTypeUrl bool) {
if marker == "" {
return
}
markerCandidates := map[string]struct{}{
marker: {},
strings.TrimPrefix(marker, "/"): {},
}
if encodingTypeUrl {
escapedMarker := urlPathEscape(strings.TrimPrefix(marker, "/"))
markerCandidates[escapedMarker] = struct{}{}
}
matchesMarker := func(v string) bool {
if _, ok := markerCandidates[v]; ok {
return true
}
_, ok := markerCandidates[strings.TrimPrefix(v, "/")]
return ok
}
if len(response.Contents) > 0 {
filtered := response.Contents[:0]
for _, content := range response.Contents {
if matchesMarker(content.Key) {
continue
}
filtered = append(filtered, content)
}
response.Contents = filtered
}
// doListFilerEntries advances nextMarker to the last emitted entry and skips
// the marker in exclusive mode. So NextMarker==marker indicates no progress.
if matchesMarker(response.NextMarker) && len(response.Contents) == 0 && len(response.CommonPrefixes) == 0 {
response.NextMarker = ""
response.IsTruncated = false
}
}
func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys uint16, originalMarker string, delimiter string, encodingTypeUrl bool, fetchOwner bool) (response ListBucketResult, err error) {
@@ -543,6 +615,12 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
}
entry := resp.Entry
// listFilerEntries always calls doListFilerEntries with inclusiveStartFrom=false
// (S3 marker semantics are exclusive), but keep the guard explicit to preserve
// behavior if inclusive callers are introduced in the future.
if !inclusiveStartFrom && marker != "" && entry.Name == marker {
continue
}
if cursor.maxKeys <= 0 {
cursor.isTruncated = true