Switch empty-folder cleanup to bucket policy (#8292)

* Fix Spark _temporary cleanup and add issue #8285 regression test

* Generalize empty folder cleanup for Spark temp artifacts

* Revert synchronous folder pruning and add cleanup diagnostics

* Add actionable empty-folder cleanup diagnostics

* Fix Spark temp marker cleanup in async folder cleaner

* Fix Spark temp cleanup with implicit directory markers

* Keep explicit directory markers non-implicit

* logging

* more logs

* Switch empty-folder cleanup to bucket policy

* Seaweed-X-Amz-Allow-Empty-Folders

* less logs

* go vet

* less logs

* refactoring

weed/command/filer_remote_gateway_buckets.go

@@ -390,11 +390,10 @@ func (option *RemoteGatewayOptions) detectBucketInfo(actualDir string) (bucket u
 }
 
 func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
-	if !strings.HasPrefix(dir, bucketsDir+"/") {
-		return "", false
+	if bucketPath, ok := util.ExtractBucketPath(bucketsDir, dir, false); ok {
+		return util.FullPath(bucketPath), true
 	}
-	parts := strings.SplitN(dir[len(bucketsDir)+1:], "/", 2)
-	return util.FullPath(bucketsDir).Child(parts[0]), true
+	return "", false
 }
 
 func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {

weed/command/mount_std.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"os/user"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -26,6 +27,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 	"google.golang.org/grpc/reflection"
@@ -59,6 +61,63 @@ func runMount(cmd *Command, args []string) bool {
 	return RunMount(&mountOptions, os.FileMode(umask))
 }
 
+func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.FilerClient, mountRoot, bucketRootPath string) error {
+	bucketPath, isBucketRootMount := bucketPathForMountRoot(mountRoot, bucketRootPath)
+	if !isBucketRootMount {
+		return nil
+	}
+
+	entry, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
+	if err != nil {
+		return err
+	}
+	if entry == nil {
+		return fmt.Errorf("bucket %s not found", bucketPath)
+	}
+
+	if entry.Extended == nil {
+		entry.Extended = make(map[string][]byte)
+	}
+	if strings.EqualFold(strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])), "true") {
+		return nil
+	}
+
+	entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
+
+	bucketFullPath := util.FullPath(bucketPath)
+	parent, _ := bucketFullPath.DirAndName()
+	if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
+			Directory: parent,
+			Entry:     entry,
+		})
+	}); err != nil {
+		return err
+	}
+
+	glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
+	return nil
+}
+
+func bucketPathForMountRoot(mountRoot, bucketRootPath string) (string, bool) {
+	cleanPath := path.Clean("/" + strings.TrimPrefix(mountRoot, "/"))
+	cleanBucketRoot := path.Clean("/" + strings.TrimPrefix(bucketRootPath, "/"))
+	if cleanBucketRoot == "/" {
+		return "", false
+	}
+	prefix := cleanBucketRoot + "/"
+	if !strings.HasPrefix(cleanPath, prefix) {
+		return "", false
+	}
+	rest := strings.TrimPrefix(cleanPath, prefix)
+
+	bucketParts := strings.Split(rest, "/")
+	if len(bucketParts) != 1 || bucketParts[0] == "" {
+		return "", false
+	}
+	return cleanBucketRoot + "/" + bucketParts[0], true
+}
+
 func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	// basic checks
@@ -73,6 +132,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	util.LoadSecurityConfiguration()
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	var cipher bool
+	var bucketRootPath string
 	var err error
 	for i := 0; i < 10; i++ {
 		err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
@@ -81,6 +141,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 				return fmt.Errorf("get filer grpc address %v configuration: %w", filerAddresses, err)
 			}
 			cipher = resp.Cipher
+			bucketRootPath = resp.DirBuckets
 			return nil
 		})
 		if err != nil {
@@ -93,6 +154,9 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
 		return true
 	}
+	if bucketRootPath == "" {
+		bucketRootPath = "/buckets"
+	}
 
 	filerMountRootPath := *option.filerMountRootPath
 
@@ -287,6 +351,10 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		fmt.Printf("failed to create dir %s on filer %s: %v\n", mountRoot, filerAddresses, err)
 		return false
 	}
+	if err := ensureBucketAllowEmptyFolders(context.Background(), seaweedFileSystem, mountRoot, bucketRootPath); err != nil {
+		fmt.Printf("failed to set bucket auto-remove-empty-folders policy for %s: %v\n", mountRoot, err)
+		return false
+	}
 
 	server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
 	if err != nil {

weed/command/mount_std_test.go

@@ -0,0 +1,52 @@
+//go:build linux || darwin || freebsd
+// +build linux darwin freebsd
+
+package command
+
+import "testing"
+
+func Test_bucketPathForMountRoot(t *testing.T) {
+	tests := []struct {
+		name      string
+		mountRoot string
+		expected  string
+		ok        bool
+	}{
+		{
+			name:      "bucket root mount",
+			mountRoot: "/buckets/test",
+			expected:  "/buckets/test",
+			ok:        true,
+		},
+		{
+			name:      "bucket root with trailing slash",
+			mountRoot: "/buckets/test/",
+			expected:  "/buckets/test",
+			ok:        true,
+		},
+		{
+			name:      "subdirectory mount",
+			mountRoot: "/buckets/test/data",
+			expected:  "",
+			ok:        false,
+		},
+		{
+			name:      "non-bucket mount",
+			mountRoot: "/data/test",
+			expected:  "",
+			ok:        false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotPath, gotOK := bucketPathForMountRoot(tt.mountRoot, "/buckets")
+			if gotOK != tt.ok {
+				t.Fatalf("expected ok=%v, got %v", tt.ok, gotOK)
+			}
+			if gotPath != tt.expected {
+				t.Fatalf("expected path %q, got %q", tt.expected, gotPath)
+			}
+		})
+	}
+}