Fix S3 delete for non-empty directory markers (#8740)

* Fix S3 delete for non-empty directory markers

* Address review feedback on directory marker deletes

* Stabilize FUSE concurrent directory operations

weed/s3api/filer_util.go

@@ -11,6 +11,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 )
 
 func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
@@ -47,16 +49,32 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData,
 
 	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
 
-		err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
-		if err != nil {
-			return err
-		}
-
-		return nil
+		return doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
 	})
 
 }
 
+func (s3a *S3ApiServer) rmObject(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
+
+	return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		return deleteObjectEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
+	})
+
+}
+
+func deleteObjectEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
+	err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
+	if err == nil {
+		return nil
+	}
+	if !strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
+		return err
+	}
+
+	return demoteDirectoryMarkerToImplicitDirectory(client, parentDirectoryPath, entryName)
+}
+
 func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
 	request := &filer_pb.DeleteEntryRequest{
 		Directory:            parentDirectoryPath,
@@ -78,6 +96,71 @@ func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath strin
 	return nil
 }
 
+func demoteDirectoryMarkerToImplicitDirectory(client filer_pb.SeaweedFilerClient, parentDirectoryPath, entryName string) error {
+	resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: parentDirectoryPath,
+		Name:      entryName,
+	})
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return nil
+		}
+		return fmt.Errorf("lookup entry %s/%s: %w", parentDirectoryPath, entryName, err)
+	}
+	if resp.Entry == nil || !resp.Entry.IsDirectory {
+		return nil
+	}
+	if !resp.Entry.IsDirectoryKeyObject() {
+		return nil
+	}
+
+	clearDirectoryMarkerMetadata(resp.Entry)
+
+	if err := filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     resp.Entry,
+	}); err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound {
+			return nil
+		}
+		return fmt.Errorf("update entry %s/%s: %w", parentDirectoryPath, entryName, err)
+	}
+	return nil
+}
+
+func clearDirectoryMarkerMetadata(entry *filer_pb.Entry) {
+	if entry == nil {
+		return
+	}
+	if entry.Attributes == nil {
+		entry.Attributes = &filer_pb.FuseAttributes{}
+	}
+
+	entry.Attributes.Mime = ""
+	entry.Attributes.Md5 = nil
+	entry.Attributes.FileSize = 0
+	entry.Content = nil
+	entry.Chunks = nil
+
+	if len(entry.Extended) == 0 {
+		return
+	}
+
+	filtered := make(map[string][]byte)
+	for k, v := range entry.Extended {
+		lowerKey := strings.ToLower(k)
+		if strings.HasPrefix(lowerKey, "xattr-") || strings.HasPrefix(lowerKey, s3_constants.SeaweedFSInternalPrefix) {
+			filtered[k] = v
+		}
+	}
+
+	if len(filtered) == 0 {
+		entry.Extended = nil
+		return
+	}
+	entry.Extended = filtered
+}
+
 func (s3a *S3ApiServer) exists(parentDirectoryPath string, entryName string, isDirectory bool) (exists bool, err error) {
 
 	return filer_pb.Exists(context.Background(), s3a, parentDirectoryPath, entryName, isDirectory)