Fix S3 delete for non-empty directory markers (#8740)

* Fix S3 delete for non-empty directory markers

* Address review feedback on directory marker deletes

* Stabilize FUSE concurrent directory operations

weed/s3api/filer_util.go

@@ -11,6 +11,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
@@ -47,16 +49,32 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData,
 
 	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 
-		err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
-		if err != nil {
-			return err
-		}
-
-		return nil
+		return doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
 	})
 
 }
 
+func (s3a *S3ApiServer) rmObject(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
+
+	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		return deleteObjectEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
+	})
+
+}
+
+func deleteObjectEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
+	err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
+	if err == nil {
+		return nil
+	}
+	if !strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
+		return err
+	}
+
+	return demoteDirectoryMarkerToImplicitDirectory(client, parentDirectoryPath, entryName)
+}
+
 func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
 	request := &filer_pb.DeleteEntryRequest{
 		Directory:            parentDirectoryPath,
@@ -78,6 +96,71 @@ func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath strin
 	return nil
 }
 
+func demoteDirectoryMarkerToImplicitDirectory(client filer_pb.SeaweedFilerClient, parentDirectoryPath, entryName string) error {
+	resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: parentDirectoryPath,
+		Name:      entryName,
+	})
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return nil
+		}
+		return fmt.Errorf("lookup entry %s/%s: %w", parentDirectoryPath, entryName, err)
+	}
+	if resp.Entry == nil || !resp.Entry.IsDirectory {
+		return nil
+	}
+	if !resp.Entry.IsDirectoryKeyObject() {
+		return nil
+	}
+
+	clearDirectoryMarkerMetadata(resp.Entry)
+
+	if err := filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     resp.Entry,
+	}); err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound {
+			return nil
+		}
+		return fmt.Errorf("update entry %s/%s: %w", parentDirectoryPath, entryName, err)
+	}
+	return nil
+}
+
+func clearDirectoryMarkerMetadata(entry *filer_pb.Entry) {
+	if entry == nil {
+		return
+	}
+	if entry.Attributes == nil {
+		entry.Attributes = &filer_pb.FuseAttributes{}
+	}
+
+	entry.Attributes.Mime = ""
+	entry.Attributes.Md5 = nil
+	entry.Attributes.FileSize = 0
+	entry.Content = nil
+	entry.Chunks = nil
+
+	if len(entry.Extended) == 0 {
+		return
+	}
+
+	filtered := make(map[string][]byte)
+	for k, v := range entry.Extended {
+		lowerKey := strings.ToLower(k)
+		if strings.HasPrefix(lowerKey, "xattr-") || strings.HasPrefix(lowerKey, s3_constants.SeaweedFSInternalPrefix) {
+			filtered[k] = v
+		}
+	}
+
+	if len(filtered) == 0 {
+		entry.Extended = nil
+		return
+	}
+	entry.Extended = filtered
+}
+
 func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
 
 	return filer_pb.Exists(context.Background(), s3a, parentDirectoryPath, entryName, isDirectory)

weed/s3api/filer_util_delete_test.go

@@ -0,0 +1,152 @@
+package s3api
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type deleteObjectEntryTestClient struct {
+	filer_pb.SeaweedFilerClient
+
+	deleteResp *filer_pb.DeleteEntryResponse
+	deleteErr  error
+	lookupResp *filer_pb.LookupDirectoryEntryResponse
+	lookupErr  error
+	updateErr  error
+
+	deleteReq *filer_pb.DeleteEntryRequest
+	lookupReq *filer_pb.LookupDirectoryEntryRequest
+	updateReq *filer_pb.UpdateEntryRequest
+}
+
+func (c *deleteObjectEntryTestClient) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
+	c.deleteReq = req
+	if c.deleteResp == nil {
+		return &filer_pb.DeleteEntryResponse{}, c.deleteErr
+	}
+	return c.deleteResp, c.deleteErr
+}
+
+func (c *deleteObjectEntryTestClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	c.lookupReq = req
+	if c.lookupResp == nil {
+		return &filer_pb.LookupDirectoryEntryResponse{}, c.lookupErr
+	}
+	return c.lookupResp, c.lookupErr
+}
+
+func (c *deleteObjectEntryTestClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
+	c.updateReq = req
+	return &filer_pb.UpdateEntryResponse{}, c.updateErr
+}
+
+func TestDeleteObjectEntryDemotesNonEmptyDirectoryMarker(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes: &filer_pb.FuseAttributes{
+					Mime:     "application/octet-stream",
+					Md5:      []byte{1, 2, 3, 4},
+					FileSize: 4,
+				},
+				Content: []byte("test"),
+				Extended: map[string][]byte{
+					s3_constants.ExtETagKey:                   []byte("etag"),
+					s3_constants.ExtAmzOwnerKey:               []byte("owner"),
+					s3_constants.AmzUserMetaPrefix + "Color":  []byte("blue"),
+					s3_constants.AmzObjectTaggingPrefix + "k": []byte("v"),
+					"xattr-keep":           []byte("keep-me"),
+					"x-seaweedfs-internal": []byte("keep-me-too"),
+				},
+			},
+		},
+	}
+
+	err := deleteObjectEntry(client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	require.NotNil(t, client.updateReq)
+
+	updated := client.updateReq.Entry
+	require.NotNil(t, updated)
+	assert.False(t, updated.IsDirectoryKeyObject())
+	assert.Equal(t, "", updated.Attributes.Mime)
+	assert.Empty(t, updated.Attributes.Md5)
+	assert.Zero(t, updated.Attributes.FileSize)
+	assert.Nil(t, updated.Content)
+	assert.Nil(t, updated.Chunks)
+	assert.Equal(t, map[string][]byte{
+		"xattr-keep":           []byte("keep-me"),
+		"x-seaweedfs-internal": []byte("keep-me-too"),
+	}, updated.Extended)
+}
+
+func TestDeleteObjectEntryTreatsImplicitDirectoryAsSuccessfulNoop(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes:  &filer_pb.FuseAttributes{},
+			},
+		},
+	}
+
+	err := deleteObjectEntry(client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	assert.Nil(t, client.updateReq)
+}
+
+func TestDeleteObjectEntryIgnoresConcurrentUpdateNotFound(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes: &filer_pb.FuseAttributes{
+					Mime: "application/octet-stream",
+				},
+			},
+		},
+		updateErr: status.Error(codes.NotFound, "already removed"),
+	}
+
+	err := deleteObjectEntry(client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	require.NotNil(t, client.updateReq)
+}
+
+func TestDeleteObjectEntryPropagatesNonDirectoryDeleteErrors(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteErr: errors.New("boom"),
+	}
+
+	err := deleteObjectEntry(client, "/buckets/test", "photos", true, false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "boom")
+	assert.Nil(t, client.lookupReq)
+	assert.Nil(t, client.updateReq)
+}

weed/s3api/s3api_object_handlers_copy.go

@@ -372,7 +372,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
 
 		// Check if destination exists and remove it first (S3 copy overwrites)
 		if exists, _ := s3a.exists(dstDir, dstName, false); exists {
-			if err := s3a.rm(dstDir, dstName, false, false); err != nil {
+			if err := s3a.rmObject(dstDir, dstName, false, false); err != nil {
 				s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 				return
 			}

weed/s3api/s3api_object_handlers_delete.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -130,7 +129,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 		dir, name := target.DirAndName()
 
 		err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-			return doDeleteEntry(client, dir, name, true, false)
+			return deleteObjectEntry(client, dir, name, true, false)
 			// Note: Empty folder cleanup is now handled asynchronously by EmptyFolderCleaner
 			// which listens to metadata events and uses consistent hashing for coordination
 		})
@@ -345,11 +344,9 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 				parentDirectoryPath, entryName := target.DirAndName()
 				isDeleteData, isRecursive := true, false
 
-				err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
+				err := deleteObjectEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
 				if err == nil {
 					deletedObjects = append(deletedObjects, object)
-				} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
-					deletedObjects = append(deletedObjects, object)
 				} else {
 					deleteErrors = append(deleteErrors, DeleteError{
 						Code:      "",

weed/s3api/s3api_object_versioning.go

@@ -949,7 +949,7 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
 		}
 
 		// Delete the regular file
-		deleteErr := s3a.rm(bucketDir, normalizedObject, true, false)
+		deleteErr := s3a.rmObject(bucketDir, normalizedObject, true, false)
 		if deleteErr != nil {
 			// Check if file was already deleted by another process
 			if _, checkErr := s3a.getEntry(bucketDir, normalizedObject); checkErr != nil {