admin: Refactor task destination planning (#7063)
* refactor planning into task detection * refactoring worker tasks * refactor * compiles, but only balance task is registered * compiles, but has nil exception * avoid nil logger * add back ec task * setting ec log directory * implement balance and vacuum tasks * EC tasks will no longer fail with "file not found" errors * Use ReceiveFile API to send locally generated shards * distributing shard files and ecx,ecj,vif files * generate .ecx files correctly * do not mount all possible EC shards (0-13) on every destination * use constants * delete all replicas * rename files * pass in volume size to tasks
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
@@ -39,6 +40,9 @@ func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterI
|
||||
Reason: "Volume has excessive garbage requiring vacuum",
|
||||
ScheduleAt: time.Now(),
|
||||
}
|
||||
|
||||
// Create typed parameters for vacuum task
|
||||
result.TypedParams = createVacuumTaskParams(result, metric, vacuumConfig)
|
||||
results = append(results, result)
|
||||
} else {
|
||||
// Debug why volume was not selected
|
||||
@@ -74,39 +78,36 @@ func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterI
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Scheduling implements the scheduling logic for vacuum tasks
|
||||
func Scheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
|
||||
vacuumConfig := config.(*Config)
|
||||
// createVacuumTaskParams creates typed parameters for vacuum tasks
|
||||
// This function is moved from MaintenanceIntegration.createVacuumTaskParams to the detection logic
|
||||
func createVacuumTaskParams(task *types.TaskDetectionResult, metric *types.VolumeHealthMetrics, vacuumConfig *Config) *worker_pb.TaskParams {
|
||||
// Use configured values or defaults
|
||||
garbageThreshold := 0.3 // Default 30%
|
||||
verifyChecksum := true // Default to verify
|
||||
batchSize := int32(1000) // Default batch size
|
||||
workingDir := "/tmp/seaweedfs_vacuum_work" // Default working directory
|
||||
|
||||
// Count running vacuum tasks
|
||||
runningVacuumCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeVacuum {
|
||||
runningVacuumCount++
|
||||
}
|
||||
if vacuumConfig != nil {
|
||||
garbageThreshold = vacuumConfig.GarbageThreshold
|
||||
// Note: VacuumTaskConfig has GarbageThreshold, MinVolumeAgeHours, MinIntervalSeconds
|
||||
// Other fields like VerifyChecksum, BatchSize, WorkingDir would need to be added
|
||||
// to the protobuf definition if they should be configurable
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningVacuumCount >= vacuumConfig.MaxConcurrent {
|
||||
return false
|
||||
// Create typed protobuf parameters
|
||||
return &worker_pb.TaskParams{
|
||||
VolumeId: task.VolumeID,
|
||||
Server: task.Server,
|
||||
Collection: task.Collection,
|
||||
VolumeSize: metric.Size, // Store original volume size for tracking changes
|
||||
TaskParams: &worker_pb.TaskParams_VacuumParams{
|
||||
VacuumParams: &worker_pb.VacuumTaskParams{
|
||||
GarbageThreshold: garbageThreshold,
|
||||
ForceVacuum: false,
|
||||
BatchSize: batchSize,
|
||||
WorkingDir: workingDir,
|
||||
VerifyChecksum: verifyChecksum,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Check for available workers with vacuum capability
|
||||
for _, worker := range availableWorkers {
|
||||
if worker.CurrentLoad < worker.MaxConcurrent {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeVacuum {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// CreateTask creates a new vacuum task instance
|
||||
func CreateTask(params types.TaskParams) (types.TaskInterface, error) {
|
||||
// Create and return the vacuum task using existing Task type
|
||||
return NewTask(params.Server, params.VolumeID), nil
|
||||
}
|
||||
|
||||
151
weed/worker/tasks/vacuum/monitoring.go
Normal file
151
weed/worker/tasks/vacuum/monitoring.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// VacuumMetrics contains vacuum-specific monitoring data
|
||||
type VacuumMetrics struct {
|
||||
// Execution metrics
|
||||
VolumesVacuumed int64 `json:"volumes_vacuumed"`
|
||||
TotalSpaceReclaimed int64 `json:"total_space_reclaimed"`
|
||||
TotalFilesProcessed int64 `json:"total_files_processed"`
|
||||
TotalGarbageCollected int64 `json:"total_garbage_collected"`
|
||||
LastVacuumTime time.Time `json:"last_vacuum_time"`
|
||||
|
||||
// Performance metrics
|
||||
AverageVacuumTime int64 `json:"average_vacuum_time_seconds"`
|
||||
AverageGarbageRatio float64 `json:"average_garbage_ratio"`
|
||||
SuccessfulOperations int64 `json:"successful_operations"`
|
||||
FailedOperations int64 `json:"failed_operations"`
|
||||
|
||||
// Current task metrics
|
||||
CurrentGarbageRatio float64 `json:"current_garbage_ratio"`
|
||||
VolumesPendingVacuum int `json:"volumes_pending_vacuum"`
|
||||
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewVacuumMetrics creates a new vacuum metrics instance
|
||||
func NewVacuumMetrics() *VacuumMetrics {
|
||||
return &VacuumMetrics{
|
||||
LastVacuumTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordVolumeVacuumed records a successful volume vacuum operation
|
||||
func (m *VacuumMetrics) RecordVolumeVacuumed(spaceReclaimed int64, filesProcessed int64, garbageCollected int64, vacuumTime time.Duration, garbageRatio float64) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
m.VolumesVacuumed++
|
||||
m.TotalSpaceReclaimed += spaceReclaimed
|
||||
m.TotalFilesProcessed += filesProcessed
|
||||
m.TotalGarbageCollected += garbageCollected
|
||||
m.SuccessfulOperations++
|
||||
m.LastVacuumTime = time.Now()
|
||||
|
||||
// Update average vacuum time
|
||||
if m.AverageVacuumTime == 0 {
|
||||
m.AverageVacuumTime = int64(vacuumTime.Seconds())
|
||||
} else {
|
||||
// Exponential moving average
|
||||
newTime := int64(vacuumTime.Seconds())
|
||||
m.AverageVacuumTime = (m.AverageVacuumTime*4 + newTime) / 5
|
||||
}
|
||||
|
||||
// Update average garbage ratio
|
||||
if m.AverageGarbageRatio == 0 {
|
||||
m.AverageGarbageRatio = garbageRatio
|
||||
} else {
|
||||
// Exponential moving average
|
||||
m.AverageGarbageRatio = 0.8*m.AverageGarbageRatio + 0.2*garbageRatio
|
||||
}
|
||||
}
|
||||
|
||||
// RecordFailure records a failed vacuum operation
|
||||
func (m *VacuumMetrics) RecordFailure() {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
m.FailedOperations++
|
||||
}
|
||||
|
||||
// UpdateCurrentGarbageRatio updates the current volume's garbage ratio
|
||||
func (m *VacuumMetrics) UpdateCurrentGarbageRatio(ratio float64) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
m.CurrentGarbageRatio = ratio
|
||||
}
|
||||
|
||||
// SetVolumesPendingVacuum sets the number of volumes pending vacuum
|
||||
func (m *VacuumMetrics) SetVolumesPendingVacuum(count int) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
m.VolumesPendingVacuum = count
|
||||
}
|
||||
|
||||
// GetMetrics returns a copy of the current metrics (without the mutex)
|
||||
func (m *VacuumMetrics) GetMetrics() VacuumMetrics {
|
||||
m.mutex.RLock()
|
||||
defer m.mutex.RUnlock()
|
||||
|
||||
// Create a copy without the mutex to avoid copying lock value
|
||||
return VacuumMetrics{
|
||||
VolumesVacuumed: m.VolumesVacuumed,
|
||||
TotalSpaceReclaimed: m.TotalSpaceReclaimed,
|
||||
TotalFilesProcessed: m.TotalFilesProcessed,
|
||||
TotalGarbageCollected: m.TotalGarbageCollected,
|
||||
LastVacuumTime: m.LastVacuumTime,
|
||||
AverageVacuumTime: m.AverageVacuumTime,
|
||||
AverageGarbageRatio: m.AverageGarbageRatio,
|
||||
SuccessfulOperations: m.SuccessfulOperations,
|
||||
FailedOperations: m.FailedOperations,
|
||||
CurrentGarbageRatio: m.CurrentGarbageRatio,
|
||||
VolumesPendingVacuum: m.VolumesPendingVacuum,
|
||||
}
|
||||
}
|
||||
|
||||
// GetSuccessRate returns the success rate as a percentage
|
||||
func (m *VacuumMetrics) GetSuccessRate() float64 {
|
||||
m.mutex.RLock()
|
||||
defer m.mutex.RUnlock()
|
||||
|
||||
total := m.SuccessfulOperations + m.FailedOperations
|
||||
if total == 0 {
|
||||
return 100.0
|
||||
}
|
||||
return float64(m.SuccessfulOperations) / float64(total) * 100.0
|
||||
}
|
||||
|
||||
// GetAverageSpaceReclaimed returns the average space reclaimed per volume
|
||||
func (m *VacuumMetrics) GetAverageSpaceReclaimed() float64 {
|
||||
m.mutex.RLock()
|
||||
defer m.mutex.RUnlock()
|
||||
|
||||
if m.VolumesVacuumed == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(m.TotalSpaceReclaimed) / float64(m.VolumesVacuumed)
|
||||
}
|
||||
|
||||
// Reset resets all metrics to zero
|
||||
func (m *VacuumMetrics) Reset() {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
|
||||
*m = VacuumMetrics{
|
||||
LastVacuumTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Global metrics instance for vacuum tasks
|
||||
var globalVacuumMetrics = NewVacuumMetrics()
|
||||
|
||||
// GetGlobalVacuumMetrics returns the global vacuum metrics instance
|
||||
func GetGlobalVacuumMetrics() *VacuumMetrics {
|
||||
return globalVacuumMetrics
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
@@ -35,9 +36,19 @@ func RegisterVacuumTask() {
|
||||
Icon: "fas fa-broom text-primary",
|
||||
Capabilities: []string{"vacuum", "storage"},
|
||||
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: CreateTask,
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
|
||||
if params == nil {
|
||||
return nil, fmt.Errorf("task parameters are required")
|
||||
}
|
||||
return NewVacuumTask(
|
||||
fmt.Sprintf("vacuum-%d", params.VolumeId),
|
||||
params.Server,
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
), nil
|
||||
},
|
||||
DetectionFunc: Detection,
|
||||
ScanInterval: 2 * time.Hour,
|
||||
SchedulingFunc: Scheduling,
|
||||
37
weed/worker/tasks/vacuum/scheduling.go
Normal file
37
weed/worker/tasks/vacuum/scheduling.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Scheduling implements the scheduling logic for vacuum tasks
|
||||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
|
||||
vacuumConfig := config.(*Config)
|
||||
|
||||
// Count running vacuum tasks
|
||||
runningVacuumCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeVacuum {
|
||||
runningVacuumCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningVacuumCount >= vacuumConfig.MaxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for available workers with vacuum capability
|
||||
for _, worker := range availableWorkers {
|
||||
if worker.CurrentLoad < worker.MaxConcurrent {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeVacuum {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,214 +0,0 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Task implements vacuum operation to reclaim disk space
|
||||
type Task struct {
|
||||
*tasks.BaseTask
|
||||
server string
|
||||
volumeID uint32
|
||||
garbageThreshold float64
|
||||
}
|
||||
|
||||
// NewTask creates a new vacuum task instance
|
||||
func NewTask(server string, volumeID uint32) *Task {
|
||||
task := &Task{
|
||||
BaseTask: tasks.NewBaseTask(types.TaskTypeVacuum),
|
||||
server: server,
|
||||
volumeID: volumeID,
|
||||
garbageThreshold: 0.3, // Default 30% threshold
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
// Execute performs the vacuum operation
|
||||
func (t *Task) Execute(params types.TaskParams) error {
|
||||
// Use BaseTask.ExecuteTask to handle logging initialization
|
||||
return t.ExecuteTask(context.Background(), params, t.executeImpl)
|
||||
}
|
||||
|
||||
// executeImpl is the actual vacuum implementation
|
||||
func (t *Task) executeImpl(ctx context.Context, params types.TaskParams) error {
|
||||
t.LogInfo("Starting vacuum for volume %d on server %s", t.volumeID, t.server)
|
||||
|
||||
// Parse garbage threshold from typed parameters
|
||||
if params.TypedParams != nil {
|
||||
if vacuumParams := params.TypedParams.GetVacuumParams(); vacuumParams != nil {
|
||||
t.garbageThreshold = vacuumParams.GarbageThreshold
|
||||
t.LogWithFields("INFO", "Using garbage threshold from parameters", map[string]interface{}{
|
||||
"threshold": t.garbageThreshold,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Convert server address to gRPC address and use proper dial option
|
||||
grpcAddress := pb.ServerToGrpcAddress(t.server)
|
||||
var dialOpt grpc.DialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
if params.GrpcDialOption != nil {
|
||||
dialOpt = params.GrpcDialOption
|
||||
}
|
||||
|
||||
conn, err := grpc.NewClient(grpcAddress, dialOpt)
|
||||
if err != nil {
|
||||
t.LogError("Failed to connect to volume server %s: %v", t.server, err)
|
||||
return fmt.Errorf("failed to connect to volume server %s: %v", t.server, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := volume_server_pb.NewVolumeServerClient(conn)
|
||||
|
||||
// Step 1: Check vacuum eligibility
|
||||
t.SetProgress(10.0)
|
||||
t.LogDebug("Checking vacuum eligibility for volume %d", t.volumeID)
|
||||
|
||||
checkResp, err := client.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum check failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum check failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Check if garbage ratio meets threshold
|
||||
if checkResp.GarbageRatio < t.garbageThreshold {
|
||||
t.LogWarning("Volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum",
|
||||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100)
|
||||
return fmt.Errorf("volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum",
|
||||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100)
|
||||
}
|
||||
|
||||
t.LogWithFields("INFO", "Volume eligible for vacuum", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"garbage_ratio": checkResp.GarbageRatio,
|
||||
"threshold": t.garbageThreshold,
|
||||
"garbage_percent": checkResp.GarbageRatio * 100,
|
||||
})
|
||||
|
||||
// Step 2: Compact volume
|
||||
t.SetProgress(30.0)
|
||||
t.LogInfo("Starting compact for volume %d", t.volumeID)
|
||||
|
||||
compactStream, err := client.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum compact failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum compact failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Process compact stream and track progress
|
||||
var processedBytes int64
|
||||
var totalBytes int64
|
||||
|
||||
for {
|
||||
resp, err := compactStream.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
t.LogError("Vacuum compact stream error for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum compact stream error for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
processedBytes = resp.ProcessedBytes
|
||||
if resp.LoadAvg_1M > 0 {
|
||||
totalBytes = int64(resp.LoadAvg_1M) // This is a rough approximation
|
||||
}
|
||||
|
||||
// Update progress based on processed bytes (30% to 70% of total progress)
|
||||
if totalBytes > 0 {
|
||||
compactProgress := float64(processedBytes) / float64(totalBytes)
|
||||
if compactProgress > 1.0 {
|
||||
compactProgress = 1.0
|
||||
}
|
||||
progress := 30.0 + (compactProgress * 40.0) // 30% to 70%
|
||||
t.SetProgress(progress)
|
||||
}
|
||||
|
||||
t.LogWithFields("DEBUG", "Volume compact progress", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"processed_bytes": processedBytes,
|
||||
"total_bytes": totalBytes,
|
||||
"compact_progress": fmt.Sprintf("%.1f%%", (float64(processedBytes)/float64(totalBytes))*100),
|
||||
})
|
||||
}
|
||||
|
||||
// Step 3: Commit vacuum changes
|
||||
t.SetProgress(80.0)
|
||||
t.LogInfo("Committing vacuum for volume %d", t.volumeID)
|
||||
|
||||
commitResp, err := client.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum commit failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum commit failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Step 4: Cleanup temporary files
|
||||
t.SetProgress(90.0)
|
||||
t.LogInfo("Cleaning up vacuum files for volume %d", t.volumeID)
|
||||
|
||||
_, err = client.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
// Log warning but don't fail the task
|
||||
t.LogWarning("Vacuum cleanup warning for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
t.SetProgress(100.0)
|
||||
|
||||
newVolumeSize := commitResp.VolumeSize
|
||||
t.LogWithFields("INFO", "Successfully completed vacuum", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"server": t.server,
|
||||
"new_volume_size": newVolumeSize,
|
||||
"garbage_reclaimed": true,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate validates the task parameters
|
||||
func (t *Task) Validate(params types.TaskParams) error {
|
||||
if params.VolumeID == 0 {
|
||||
return fmt.Errorf("volume_id is required")
|
||||
}
|
||||
if params.Server == "" {
|
||||
return fmt.Errorf("server is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateTime estimates the time needed for the task
|
||||
func (t *Task) EstimateTime(params types.TaskParams) time.Duration {
|
||||
// Base time for vacuum operations - varies by volume size and garbage ratio
|
||||
// Typically vacuum is faster than EC encoding
|
||||
baseTime := 5 * time.Minute
|
||||
|
||||
// Use default estimation since volume size is not available in typed params
|
||||
return baseTime
|
||||
}
|
||||
|
||||
// GetProgress returns the current progress
|
||||
func (t *Task) GetProgress() float64 {
|
||||
return t.BaseTask.GetProgress()
|
||||
}
|
||||
|
||||
// Cancel cancels the task
|
||||
func (t *Task) Cancel() error {
|
||||
return t.BaseTask.Cancel()
|
||||
}
|
||||
236
weed/worker/tasks/vacuum/vacuum_task.go
Normal file
236
weed/worker/tasks/vacuum/vacuum_task.go
Normal file
@@ -0,0 +1,236 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// VacuumTask implements the Task interface
|
||||
type VacuumTask struct {
|
||||
*base.BaseTask
|
||||
server string
|
||||
volumeID uint32
|
||||
collection string
|
||||
garbageThreshold float64
|
||||
progress float64
|
||||
}
|
||||
|
||||
// NewVacuumTask creates a new unified vacuum task instance
|
||||
func NewVacuumTask(id string, server string, volumeID uint32, collection string) *VacuumTask {
|
||||
return &VacuumTask{
|
||||
BaseTask: base.NewBaseTask(id, types.TaskTypeVacuum),
|
||||
server: server,
|
||||
volumeID: volumeID,
|
||||
collection: collection,
|
||||
garbageThreshold: 0.3, // Default 30% threshold
|
||||
}
|
||||
}
|
||||
|
||||
// Execute implements the UnifiedTask interface
|
||||
func (t *VacuumTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
vacuumParams := params.GetVacuumParams()
|
||||
if vacuumParams == nil {
|
||||
return fmt.Errorf("vacuum parameters are required")
|
||||
}
|
||||
|
||||
t.garbageThreshold = vacuumParams.GarbageThreshold
|
||||
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"server": t.server,
|
||||
"collection": t.collection,
|
||||
"garbage_threshold": t.garbageThreshold,
|
||||
}).Info("Starting vacuum task")
|
||||
|
||||
// Step 1: Check volume status and garbage ratio
|
||||
t.ReportProgress(10.0)
|
||||
t.GetLogger().Info("Checking volume status")
|
||||
eligible, currentGarbageRatio, err := t.checkVacuumEligibility()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check vacuum eligibility: %v", err)
|
||||
}
|
||||
|
||||
if !eligible {
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"current_garbage_ratio": currentGarbageRatio,
|
||||
"required_threshold": t.garbageThreshold,
|
||||
}).Info("Volume does not meet vacuum criteria, skipping")
|
||||
t.ReportProgress(100.0)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Step 2: Perform vacuum operation
|
||||
t.ReportProgress(50.0)
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"garbage_ratio": currentGarbageRatio,
|
||||
"threshold": t.garbageThreshold,
|
||||
}).Info("Performing vacuum operation")
|
||||
|
||||
if err := t.performVacuum(); err != nil {
|
||||
return fmt.Errorf("failed to perform vacuum: %v", err)
|
||||
}
|
||||
|
||||
// Step 3: Verify vacuum results
|
||||
t.ReportProgress(90.0)
|
||||
t.GetLogger().Info("Verifying vacuum results")
|
||||
if err := t.verifyVacuumResults(); err != nil {
|
||||
glog.Warningf("Vacuum verification failed: %v", err)
|
||||
// Don't fail the task - vacuum operation itself succeeded
|
||||
}
|
||||
|
||||
t.ReportProgress(100.0)
|
||||
glog.Infof("Vacuum task completed successfully: volume %d from %s (garbage ratio was %.2f%%)",
|
||||
t.volumeID, t.server, currentGarbageRatio*100)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate implements the UnifiedTask interface
|
||||
func (t *VacuumTask) Validate(params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
vacuumParams := params.GetVacuumParams()
|
||||
if vacuumParams == nil {
|
||||
return fmt.Errorf("vacuum parameters are required")
|
||||
}
|
||||
|
||||
if params.VolumeId != t.volumeID {
|
||||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
|
||||
}
|
||||
|
||||
if params.Server != t.server {
|
||||
return fmt.Errorf("source server mismatch: expected %s, got %s", t.server, params.Server)
|
||||
}
|
||||
|
||||
if vacuumParams.GarbageThreshold < 0 || vacuumParams.GarbageThreshold > 1.0 {
|
||||
return fmt.Errorf("invalid garbage threshold: %f (must be between 0.0 and 1.0)", vacuumParams.GarbageThreshold)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateTime implements the UnifiedTask interface
|
||||
func (t *VacuumTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
|
||||
// Basic estimate based on simulated steps
|
||||
return 14 * time.Second // Sum of all step durations
|
||||
}
|
||||
|
||||
// GetProgress returns current progress
|
||||
func (t *VacuumTask) GetProgress() float64 {
|
||||
return t.progress
|
||||
}
|
||||
|
||||
// Helper methods for real vacuum operations
|
||||
|
||||
// checkVacuumEligibility checks if the volume meets vacuum criteria
|
||||
func (t *VacuumTask) checkVacuumEligibility() (bool, float64, error) {
|
||||
var garbageRatio float64
|
||||
|
||||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check volume vacuum status: %v", err)
|
||||
}
|
||||
|
||||
garbageRatio = resp.GarbageRatio
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
|
||||
eligible := garbageRatio >= t.garbageThreshold
|
||||
glog.V(1).Infof("Volume %d garbage ratio: %.2f%%, threshold: %.2f%%, eligible: %v",
|
||||
t.volumeID, garbageRatio*100, t.garbageThreshold*100, eligible)
|
||||
|
||||
return eligible, garbageRatio, nil
|
||||
}
|
||||
|
||||
// performVacuum executes the actual vacuum operation
|
||||
func (t *VacuumTask) performVacuum() error {
|
||||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
// Step 1: Compact the volume
|
||||
t.GetLogger().Info("Compacting volume")
|
||||
stream, err := client.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("vacuum compact failed: %v", err)
|
||||
}
|
||||
|
||||
// Read compact progress
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return fmt.Errorf("vacuum compact stream error: %v", recvErr)
|
||||
}
|
||||
glog.V(2).Infof("Volume %d compact progress: %d bytes processed", t.volumeID, resp.ProcessedBytes)
|
||||
}
|
||||
|
||||
// Step 2: Commit the vacuum
|
||||
t.GetLogger().Info("Committing vacuum operation")
|
||||
_, err = client.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("vacuum commit failed: %v", err)
|
||||
}
|
||||
|
||||
// Step 3: Cleanup old files
|
||||
t.GetLogger().Info("Cleaning up vacuum files")
|
||||
_, err = client.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("vacuum cleanup failed: %v", err)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Volume %d vacuum operation completed successfully", t.volumeID)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// verifyVacuumResults checks the volume status after vacuum
|
||||
func (t *VacuumTask) verifyVacuumResults() error {
|
||||
return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
resp, err := client.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to verify vacuum results: %v", err)
|
||||
}
|
||||
|
||||
postVacuumGarbageRatio := resp.GarbageRatio
|
||||
|
||||
glog.V(1).Infof("Volume %d post-vacuum garbage ratio: %.2f%%",
|
||||
t.volumeID, postVacuumGarbageRatio*100)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user