Worker set its working directory (#8461)

* set working directory

* consolidate to worker directory

* working directory

* correct directory name

* refactoring to use wildcard matcher

* simplify

* cleaning ec working directory

* fix reference

* clean

* adjust test

weed/worker/tasks/erasure_coding/detection.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
@@ -12,6 +11,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/placement"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -51,7 +51,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
 
 	var planner *ecPlacementPlanner
 
-	allowedCollections := ParseCollectionFilter(ecConfig.CollectionFilter)
+	allowedCollections := wildcard.CompileWildcardMatchers(ecConfig.CollectionFilter)
 
 	// Group metrics by VolumeID to handle replicas and select canonical server
 	volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
@@ -108,12 +108,9 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
 		}
 
 		// Check collection filter if specified
-		if len(allowedCollections) > 0 {
-			// Skip if volume's collection is not in the allowed list
-			if !allowedCollections[metric.Collection] {
-				skippedCollectionFilter++
-				continue
-			}
+		if len(allowedCollections) > 0 && !wildcard.MatchesAnyWildcard(allowedCollections, metric.Collection) {
+			skippedCollectionFilter++
+			continue
 		}
 
 		// Check quiet duration and fullness criteria
@@ -336,20 +333,6 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
 	return results, hasMore, nil
 }
 
-func ParseCollectionFilter(filter string) map[string]bool {
-	allowed := make(map[string]bool)
-	for _, collection := range strings.Split(filter, ",") {
-		trimmed := strings.TrimSpace(collection)
-		if trimmed != "" {
-			allowed[trimmed] = true
-		}
-	}
-	if len(allowed) == 0 {
-		return nil
-	}
-	return allowed
-}
-
 type ecDiskState struct {
 	baseAvailable      int64
 	reservedVolumes    int32

weed/worker/tasks/erasure_coding/ec_task.go

@@ -106,9 +106,10 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 	}
 
 	// Use the working directory from task parameters, or fall back to a default
-	baseWorkDir := t.workDir
-
-	// Create unique working directory for this task
+	baseWorkDir := ecParams.WorkingDir
+	if baseWorkDir == "" {
+		baseWorkDir = t.GetWorkingDir()
+	}
 	taskWorkDir := filepath.Join(baseWorkDir, fmt.Sprintf("vol_%d_%d", t.volumeID, time.Now().Unix()))
 	if err := os.MkdirAll(taskWorkDir, 0755); err != nil {
 		return fmt.Errorf("failed to create task working directory %s: %v", taskWorkDir, err)
@@ -119,9 +120,9 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 	t.workDir = taskWorkDir
 	glog.V(1).Infof("Task working directory configured: %s (logs will be written here)", taskWorkDir)
 
-	// Ensure cleanup of working directory (but preserve logs)
+	// Ensure cleanup of working directory
 	defer func() {
-		// Clean up volume files and EC shards, but preserve the directory structure and any logs
+		// Clean up volume files and EC shards
 		patterns := []string{"*.dat", "*.idx", "*.ec*", "*.vif"}
 		for _, pattern := range patterns {
 			matches, err := filepath.Glob(filepath.Join(taskWorkDir, pattern))
@@ -134,7 +135,12 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
 				}
 			}
 		}
-		glog.V(1).Infof("Cleaned up volume files from working directory: %s (logs preserved)", taskWorkDir)
+		// Remove the entire working directory
+		if err := os.RemoveAll(taskWorkDir); err != nil {
+			glog.V(2).Infof("Could not remove working directory %s: %v", taskWorkDir, err)
+		} else {
+			glog.V(1).Infof("Cleaned up working directory: %s", taskWorkDir)
+		}
 	}()
 
 	// Step 1: Mark volume readonly

weed/worker/tasks/vacuum/detection.go

@@ -98,10 +98,10 @@ func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterI
 // This function is moved from MaintenanceIntegration.createVacuumTaskParams to the detection logic
 func createVacuumTaskParams(task *types.TaskDetectionResult, metric *types.VolumeHealthMetrics, vacuumConfig *Config, clusterInfo *types.ClusterInfo) *worker_pb.TaskParams {
 	// Use configured values or defaults
-	garbageThreshold := 0.3                    // Default 30%
-	verifyChecksum := true                     // Default to verify
-	batchSize := int32(1000)                   // Default batch size
-	workingDir := "/tmp/seaweedfs_vacuum_work" // Default working directory
+	garbageThreshold := 0.3  // Default 30%
+	verifyChecksum := true   // Default to verify
+	batchSize := int32(1000) // Default batch size
+	workingDir := ""         // Use worker-provided default if empty
 
 	if vacuumConfig != nil {
 		garbageThreshold = vacuumConfig.GarbageThreshold

weed/worker/types/base/task.go

@@ -16,6 +16,7 @@ type BaseTask struct {
 	logger           types.Logger
 	cancelled        bool
 	currentStage     string
+	workingDir       string
 }
 
 // NewBaseTask creates a new base task
@@ -116,3 +117,13 @@ func (t *BaseTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
 	// Subclasses must implement this
 	return 0
 }
+
+// SetWorkingDir sets the task working directory
+func (t *BaseTask) SetWorkingDir(workingDir string) {
+	t.workingDir = workingDir
+}
+
+// GetWorkingDir returns the task working directory
+func (t *BaseTask) GetWorkingDir() string {
+	return t.workingDir
+}

weed/worker/types/task.go

@@ -29,6 +29,10 @@ type Task interface {
 	// Progress
 	GetProgress() float64
 	SetProgressCallback(func(float64, string))
+
+	// Working Directory
+	SetWorkingDir(string)
+	GetWorkingDir() string
 }
 
 // TaskWithLogging extends Task with logging capabilities
@@ -131,6 +135,7 @@ type UnifiedBaseTask struct {
 	logger           Logger
 	cancelled        bool
 	currentStage     string
+	workingDir       string
 }
 
 // NewBaseTask creates a new base task
@@ -206,3 +211,13 @@ func (t *UnifiedBaseTask) SetLogger(logger Logger) {
 func (t *UnifiedBaseTask) GetLogger() Logger {
 	return t.logger
 }
+
+// SetWorkingDir sets the task working directory
+func (t *UnifiedBaseTask) SetWorkingDir(workingDir string) {
+	t.workingDir = workingDir
+}
+
+// GetWorkingDir returns the task working directory
+func (t *UnifiedBaseTask) GetWorkingDir() string {
+	return t.workingDir
+}

weed/worker/worker.go

@@ -172,6 +172,13 @@ func NewWorker(config *types.WorkerConfig) (*Worker, error) {
 	// Use the global unified registry that already has all tasks registered
 	registry := tasks.GetGlobalTaskRegistry()
 
+	// Ensure the base working directory exists
+	if config.BaseWorkingDir != "" {
+		if err := os.MkdirAll(config.BaseWorkingDir, 0755); err != nil {
+			return nil, fmt.Errorf("failed to create base working directory %s: %v", config.BaseWorkingDir, err)
+		}
+	}
+
 	// Initialize task log handler
 	logDir := filepath.Join(config.BaseWorkingDir, "task_logs")
 	// Ensure the base task log directory exists to avoid errors when admin requests logs
@@ -677,6 +684,7 @@ func (w *Worker) executeTask(task *types.TaskInput) {
 
 	// Task execution uses the new unified Task interface
 	glog.V(2).Infof("Executing task %s in working directory: %s", task.ID, taskWorkingDir)
+	taskInstance.SetWorkingDir(taskWorkingDir)
 
 	// If we have a file logger, adapt it so task WithFields logs are captured into file
 	if fileLogger != nil {