Admin: misc improvements on admin server and workers. EC now works. (#7055)
* initial design * added simulation as tests * reorganized the codebase to move the simulation framework and tests into their own dedicated package * integration test. ec worker task * remove "enhanced" reference * start master, volume servers, filer Current Status ✅ Master: Healthy and running (port 9333) ✅ Filer: Healthy and running (port 8888) ✅ Volume Servers: All 6 servers running (ports 8080-8085) 🔄 Admin/Workers: Will start when dependencies are ready * generate write load * tasks are assigned * admin start wtih grpc port. worker has its own working directory * Update .gitignore * working worker and admin. Task detection is not working yet. * compiles, detection uses volumeSizeLimitMB from master * compiles * worker retries connecting to admin * build and restart * rendering pending tasks * skip task ID column * sticky worker id * test canScheduleTaskNow * worker reconnect to admin * clean up logs * worker register itself first * worker can run ec work and report status but: 1. one volume should not be repeatedly worked on. 2. ec shards needs to be distributed and source data should be deleted. * move ec task logic * listing ec shards * local copy, ec. Need to distribute. * ec is mostly working now * distribution of ec shards needs improvement * need configuration to enable ec * show ec volumes * interval field UI component * rename * integration test with vauuming * garbage percentage threshold * fix warning * display ec shard sizes * fix ec volumes list * Update ui.go * show default values * ensure correct default value * MaintenanceConfig use ConfigField * use schema defined defaults * config * reduce duplication * refactor to use BaseUIProvider * each task register its schema * checkECEncodingCandidate use ecDetector * use vacuumDetector * use volumeSizeLimitMB * remove remove * remove unused * refactor * use new framework * remove v2 reference * refactor * left menu can scroll now * The maintenance manager was not being initialized when no data directory was configured for persistent storage. * saving config * Update task_config_schema_templ.go * enable/disable tasks * protobuf encoded task configurations * fix system settings * use ui component * remove logs * interface{} Reduction * reduce interface{} * reduce interface{} * avoid from/to map * reduce interface{} * refactor * keep it DRY * added logging * debug messages * debug level * debug * show the log caller line * use configured task policy * log level * handle admin heartbeat response * Update worker.go * fix EC rack and dc count * Report task status to admin server * fix task logging, simplify interface checking, use erasure_coding constants * factor in empty volume server during task planning * volume.list adds disk id * track disk id also * fix locking scheduled and manual scanning * add active topology * simplify task detector * ec task completed, but shards are not showing up * implement ec in ec_typed.go * adjust log level * dedup * implementing ec copying shards and only ecx files * use disk id when distributing ec shards 🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk 📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId 🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest 💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId]) 📂 File System: EC shards and metadata land in the exact disk directory planned * Delete original volume from all locations * clean up existing shard locations * local encoding and distributing * Update docker/admin_integration/EC-TESTING-README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * check volume id range * simplify * fix tests * fix types * clean up logs and tests --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Chris Lu
190
weed/worker/tasks/vacuum/config.go
Normal file
190
weed/worker/tasks/vacuum/config.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
)
|
||||
|
||||
// Config extends BaseConfig with vacuum-specific settings
|
||||
type Config struct {
|
||||
base.BaseConfig
|
||||
GarbageThreshold float64 `json:"garbage_threshold"`
|
||||
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"`
|
||||
MinIntervalSeconds int `json:"min_interval_seconds"`
|
||||
}
|
||||
|
||||
// NewDefaultConfig creates a new default vacuum configuration
|
||||
func NewDefaultConfig() *Config {
|
||||
return &Config{
|
||||
BaseConfig: base.BaseConfig{
|
||||
Enabled: true,
|
||||
ScanIntervalSeconds: 2 * 60 * 60, // 2 hours
|
||||
MaxConcurrent: 2,
|
||||
},
|
||||
GarbageThreshold: 0.3, // 30%
|
||||
MinVolumeAgeSeconds: 24 * 60 * 60, // 24 hours
|
||||
MinIntervalSeconds: 7 * 24 * 60 * 60, // 7 days
|
||||
}
|
||||
}
|
||||
|
||||
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
||||
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
||||
return &worker_pb.TaskPolicy{
|
||||
Enabled: c.Enabled,
|
||||
MaxConcurrent: int32(c.MaxConcurrent),
|
||||
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
|
||||
VacuumConfig: &worker_pb.VacuumTaskConfig{
|
||||
GarbageThreshold: float64(c.GarbageThreshold),
|
||||
MinVolumeAgeHours: int32(c.MinVolumeAgeSeconds / 3600), // Convert seconds to hours
|
||||
MinIntervalSeconds: int32(c.MinIntervalSeconds),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
||||
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
if policy == nil {
|
||||
return fmt.Errorf("policy is nil")
|
||||
}
|
||||
|
||||
// Set general TaskPolicy fields
|
||||
c.Enabled = policy.Enabled
|
||||
c.MaxConcurrent = int(policy.MaxConcurrent)
|
||||
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
||||
|
||||
// Set vacuum-specific fields from the task config
|
||||
if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
|
||||
c.GarbageThreshold = float64(vacuumConfig.GarbageThreshold)
|
||||
c.MinVolumeAgeSeconds = int(vacuumConfig.MinVolumeAgeHours * 3600) // Convert hours to seconds
|
||||
c.MinIntervalSeconds = int(vacuumConfig.MinIntervalSeconds)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
||||
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Try to load from persistence if available
|
||||
if persistence, ok := configPersistence.(interface {
|
||||
LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error)
|
||||
}); ok {
|
||||
if policy, err := persistence.LoadVacuumTaskPolicy(); err == nil && policy != nil {
|
||||
if err := config.FromTaskPolicy(policy); err == nil {
|
||||
glog.V(1).Infof("Loaded vacuum configuration from persistence")
|
||||
return config
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Using default vacuum configuration")
|
||||
return config
|
||||
}
|
||||
|
||||
// GetConfigSpec returns the configuration schema for vacuum tasks
|
||||
func GetConfigSpec() base.ConfigSpec {
|
||||
return base.ConfigSpec{
|
||||
Fields: []*config.Field{
|
||||
{
|
||||
Name: "enabled",
|
||||
JSONName: "enabled",
|
||||
Type: config.FieldTypeBool,
|
||||
DefaultValue: true,
|
||||
Required: false,
|
||||
DisplayName: "Enable Vacuum Tasks",
|
||||
Description: "Whether vacuum tasks should be automatically created",
|
||||
HelpText: "Toggle this to enable or disable automatic vacuum task generation",
|
||||
InputType: "checkbox",
|
||||
CSSClasses: "form-check-input",
|
||||
},
|
||||
{
|
||||
Name: "scan_interval_seconds",
|
||||
JSONName: "scan_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 2 * 60 * 60,
|
||||
MinValue: 10 * 60,
|
||||
MaxValue: 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Scan Interval",
|
||||
Description: "How often to scan for volumes needing vacuum",
|
||||
HelpText: "The system will check for volumes that need vacuuming at this interval",
|
||||
Placeholder: "2",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "max_concurrent",
|
||||
JSONName: "max_concurrent",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 2,
|
||||
MinValue: 1,
|
||||
MaxValue: 10,
|
||||
Required: true,
|
||||
DisplayName: "Max Concurrent Tasks",
|
||||
Description: "Maximum number of vacuum tasks that can run simultaneously",
|
||||
HelpText: "Limits the number of vacuum operations running at the same time to control system load",
|
||||
Placeholder: "2 (default)",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "garbage_threshold",
|
||||
JSONName: "garbage_threshold",
|
||||
Type: config.FieldTypeFloat,
|
||||
DefaultValue: 0.3,
|
||||
MinValue: 0.0,
|
||||
MaxValue: 1.0,
|
||||
Required: true,
|
||||
DisplayName: "Garbage Percentage Threshold",
|
||||
Description: "Trigger vacuum when garbage ratio exceeds this percentage",
|
||||
HelpText: "Volumes with more deleted content than this threshold will be vacuumed",
|
||||
Placeholder: "0.30 (30%)",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "min_volume_age_seconds",
|
||||
JSONName: "min_volume_age_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 24 * 60 * 60,
|
||||
MinValue: 1 * 60 * 60,
|
||||
MaxValue: 7 * 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Minimum Volume Age",
|
||||
Description: "Only vacuum volumes older than this duration",
|
||||
HelpText: "Prevents vacuuming of recently created volumes that may still be actively written to",
|
||||
Placeholder: "24",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "min_interval_seconds",
|
||||
JSONName: "min_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 7 * 24 * 60 * 60,
|
||||
MinValue: 1 * 24 * 60 * 60,
|
||||
MaxValue: 30 * 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Minimum Interval",
|
||||
Description: "Minimum time between vacuum operations on the same volume",
|
||||
HelpText: "Prevents excessive vacuuming of the same volume by enforcing a minimum wait time",
|
||||
Placeholder: "7",
|
||||
Unit: config.UnitDays,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
112
weed/worker/tasks/vacuum/detection.go
Normal file
112
weed/worker/tasks/vacuum/detection.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Detection implements the detection logic for vacuum tasks
|
||||
func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
|
||||
if !config.IsEnabled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
vacuumConfig := config.(*Config)
|
||||
var results []*types.TaskDetectionResult
|
||||
minVolumeAge := time.Duration(vacuumConfig.MinVolumeAgeSeconds) * time.Second
|
||||
|
||||
debugCount := 0
|
||||
skippedDueToGarbage := 0
|
||||
skippedDueToAge := 0
|
||||
|
||||
for _, metric := range metrics {
|
||||
// Check if volume needs vacuum
|
||||
if metric.GarbageRatio >= vacuumConfig.GarbageThreshold && metric.Age >= minVolumeAge {
|
||||
priority := types.TaskPriorityNormal
|
||||
if metric.GarbageRatio > 0.6 {
|
||||
priority = types.TaskPriorityHigh
|
||||
}
|
||||
|
||||
result := &types.TaskDetectionResult{
|
||||
TaskType: types.TaskTypeVacuum,
|
||||
VolumeID: metric.VolumeID,
|
||||
Server: metric.Server,
|
||||
Collection: metric.Collection,
|
||||
Priority: priority,
|
||||
Reason: "Volume has excessive garbage requiring vacuum",
|
||||
ScheduleAt: time.Now(),
|
||||
}
|
||||
results = append(results, result)
|
||||
} else {
|
||||
// Debug why volume was not selected
|
||||
if debugCount < 5 { // Limit debug output to first 5 volumes
|
||||
if metric.GarbageRatio < vacuumConfig.GarbageThreshold {
|
||||
skippedDueToGarbage++
|
||||
}
|
||||
if metric.Age < minVolumeAge {
|
||||
skippedDueToAge++
|
||||
}
|
||||
}
|
||||
debugCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Log debug summary if no tasks were created
|
||||
if len(results) == 0 && len(metrics) > 0 {
|
||||
totalVolumes := len(metrics)
|
||||
glog.Infof("VACUUM: No tasks created for %d volumes. Threshold=%.2f%%, MinAge=%s. Skipped: %d (garbage<threshold), %d (age<minimum)",
|
||||
totalVolumes, vacuumConfig.GarbageThreshold*100, minVolumeAge, skippedDueToGarbage, skippedDueToAge)
|
||||
|
||||
// Show details for first few volumes
|
||||
for i, metric := range metrics {
|
||||
if i >= 3 { // Limit to first 3 volumes
|
||||
break
|
||||
}
|
||||
glog.Infof("VACUUM: Volume %d: garbage=%.2f%% (need ≥%.2f%%), age=%s (need ≥%s)",
|
||||
metric.VolumeID, metric.GarbageRatio*100, vacuumConfig.GarbageThreshold*100,
|
||||
metric.Age.Truncate(time.Minute), minVolumeAge.Truncate(time.Minute))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// Scheduling implements the scheduling logic for vacuum tasks
|
||||
func Scheduling(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker, config base.TaskConfig) bool {
|
||||
vacuumConfig := config.(*Config)
|
||||
|
||||
// Count running vacuum tasks
|
||||
runningVacuumCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeVacuum {
|
||||
runningVacuumCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningVacuumCount >= vacuumConfig.MaxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for available workers with vacuum capability
|
||||
for _, worker := range availableWorkers {
|
||||
if worker.CurrentLoad < worker.MaxConcurrent {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeVacuum {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// CreateTask creates a new vacuum task instance
|
||||
func CreateTask(params types.TaskParams) (types.TaskInterface, error) {
|
||||
// Create and return the vacuum task using existing Task type
|
||||
return NewTask(params.Server, params.VolumeID), nil
|
||||
}
|
||||
@@ -1,314 +0,0 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// UIProvider provides the UI for vacuum task configuration
|
||||
type UIProvider struct {
|
||||
detector *VacuumDetector
|
||||
scheduler *VacuumScheduler
|
||||
}
|
||||
|
||||
// NewUIProvider creates a new vacuum UI provider
|
||||
func NewUIProvider(detector *VacuumDetector, scheduler *VacuumScheduler) *UIProvider {
|
||||
return &UIProvider{
|
||||
detector: detector,
|
||||
scheduler: scheduler,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTaskType returns the task type
|
||||
func (ui *UIProvider) GetTaskType() types.TaskType {
|
||||
return types.TaskTypeVacuum
|
||||
}
|
||||
|
||||
// GetDisplayName returns the human-readable name
|
||||
func (ui *UIProvider) GetDisplayName() string {
|
||||
return "Volume Vacuum"
|
||||
}
|
||||
|
||||
// GetDescription returns a description of what this task does
|
||||
func (ui *UIProvider) GetDescription() string {
|
||||
return "Reclaims disk space by removing deleted files from volumes"
|
||||
}
|
||||
|
||||
// GetIcon returns the icon CSS class for this task type
|
||||
func (ui *UIProvider) GetIcon() string {
|
||||
return "fas fa-broom text-primary"
|
||||
}
|
||||
|
||||
// VacuumConfig represents the vacuum configuration
|
||||
type VacuumConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
GarbageThreshold float64 `json:"garbage_threshold"`
|
||||
ScanIntervalSeconds int `json:"scan_interval_seconds"`
|
||||
MaxConcurrent int `json:"max_concurrent"`
|
||||
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"`
|
||||
MinIntervalSeconds int `json:"min_interval_seconds"`
|
||||
}
|
||||
|
||||
// Helper functions for duration conversion
|
||||
func secondsToDuration(seconds int) time.Duration {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
func durationToSeconds(d time.Duration) int {
|
||||
return int(d.Seconds())
|
||||
}
|
||||
|
||||
// formatDurationForUser formats seconds as a user-friendly duration string
|
||||
func formatDurationForUser(seconds int) string {
|
||||
d := secondsToDuration(seconds)
|
||||
if d < time.Minute {
|
||||
return fmt.Sprintf("%ds", seconds)
|
||||
}
|
||||
if d < time.Hour {
|
||||
return fmt.Sprintf("%.0fm", d.Minutes())
|
||||
}
|
||||
if d < 24*time.Hour {
|
||||
return fmt.Sprintf("%.1fh", d.Hours())
|
||||
}
|
||||
return fmt.Sprintf("%.1fd", d.Hours()/24)
|
||||
}
|
||||
|
||||
// RenderConfigForm renders the configuration form HTML
|
||||
func (ui *UIProvider) RenderConfigForm(currentConfig interface{}) (template.HTML, error) {
|
||||
config := ui.getCurrentVacuumConfig()
|
||||
|
||||
// Build form using the FormBuilder helper
|
||||
form := types.NewFormBuilder()
|
||||
|
||||
// Detection Settings
|
||||
form.AddCheckboxField(
|
||||
"enabled",
|
||||
"Enable Vacuum Tasks",
|
||||
"Whether vacuum tasks should be automatically created",
|
||||
config.Enabled,
|
||||
)
|
||||
|
||||
form.AddNumberField(
|
||||
"garbage_threshold",
|
||||
"Garbage Threshold (%)",
|
||||
"Trigger vacuum when garbage ratio exceeds this percentage (0.0-1.0)",
|
||||
config.GarbageThreshold,
|
||||
true,
|
||||
)
|
||||
|
||||
form.AddDurationField(
|
||||
"scan_interval",
|
||||
"Scan Interval",
|
||||
"How often to scan for volumes needing vacuum",
|
||||
secondsToDuration(config.ScanIntervalSeconds),
|
||||
true,
|
||||
)
|
||||
|
||||
form.AddDurationField(
|
||||
"min_volume_age",
|
||||
"Minimum Volume Age",
|
||||
"Only vacuum volumes older than this duration",
|
||||
secondsToDuration(config.MinVolumeAgeSeconds),
|
||||
true,
|
||||
)
|
||||
|
||||
// Scheduling Settings
|
||||
form.AddNumberField(
|
||||
"max_concurrent",
|
||||
"Max Concurrent Tasks",
|
||||
"Maximum number of vacuum tasks that can run simultaneously",
|
||||
float64(config.MaxConcurrent),
|
||||
true,
|
||||
)
|
||||
|
||||
form.AddDurationField(
|
||||
"min_interval",
|
||||
"Minimum Interval",
|
||||
"Minimum time between vacuum operations on the same volume",
|
||||
secondsToDuration(config.MinIntervalSeconds),
|
||||
true,
|
||||
)
|
||||
|
||||
// Generate organized form sections using Bootstrap components
|
||||
html := `
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="card mb-4">
|
||||
<div class="card-header">
|
||||
<h5 class="mb-0">
|
||||
<i class="fas fa-search me-2"></i>
|
||||
Detection Settings
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
` + string(form.Build()) + `
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function resetForm() {
|
||||
if (confirm('Reset all vacuum settings to defaults?')) {
|
||||
// Reset to default values
|
||||
document.querySelector('input[name="enabled"]').checked = true;
|
||||
document.querySelector('input[name="garbage_threshold"]').value = '0.3';
|
||||
document.querySelector('input[name="scan_interval"]').value = '30m';
|
||||
document.querySelector('input[name="min_volume_age"]').value = '1h';
|
||||
document.querySelector('input[name="max_concurrent"]').value = '2';
|
||||
document.querySelector('input[name="min_interval"]').value = '6h';
|
||||
}
|
||||
}
|
||||
</script>
|
||||
`
|
||||
|
||||
return template.HTML(html), nil
|
||||
}
|
||||
|
||||
// ParseConfigForm parses form data into configuration
|
||||
func (ui *UIProvider) ParseConfigForm(formData map[string][]string) (interface{}, error) {
|
||||
config := &VacuumConfig{}
|
||||
|
||||
// Parse enabled checkbox
|
||||
config.Enabled = len(formData["enabled"]) > 0 && formData["enabled"][0] == "on"
|
||||
|
||||
// Parse garbage threshold
|
||||
if thresholdStr := formData["garbage_threshold"]; len(thresholdStr) > 0 {
|
||||
if threshold, err := strconv.ParseFloat(thresholdStr[0], 64); err != nil {
|
||||
return nil, fmt.Errorf("invalid garbage threshold: %w", err)
|
||||
} else if threshold < 0 || threshold > 1 {
|
||||
return nil, fmt.Errorf("garbage threshold must be between 0.0 and 1.0")
|
||||
} else {
|
||||
config.GarbageThreshold = threshold
|
||||
}
|
||||
}
|
||||
|
||||
// Parse scan interval
|
||||
if intervalStr := formData["scan_interval"]; len(intervalStr) > 0 {
|
||||
if interval, err := time.ParseDuration(intervalStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid scan interval: %w", err)
|
||||
} else {
|
||||
config.ScanIntervalSeconds = durationToSeconds(interval)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse min volume age
|
||||
if ageStr := formData["min_volume_age"]; len(ageStr) > 0 {
|
||||
if age, err := time.ParseDuration(ageStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid min volume age: %w", err)
|
||||
} else {
|
||||
config.MinVolumeAgeSeconds = durationToSeconds(age)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse max concurrent
|
||||
if concurrentStr := formData["max_concurrent"]; len(concurrentStr) > 0 {
|
||||
if concurrent, err := strconv.Atoi(concurrentStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid max concurrent: %w", err)
|
||||
} else if concurrent < 1 {
|
||||
return nil, fmt.Errorf("max concurrent must be at least 1")
|
||||
} else {
|
||||
config.MaxConcurrent = concurrent
|
||||
}
|
||||
}
|
||||
|
||||
// Parse min interval
|
||||
if intervalStr := formData["min_interval"]; len(intervalStr) > 0 {
|
||||
if interval, err := time.ParseDuration(intervalStr[0]); err != nil {
|
||||
return nil, fmt.Errorf("invalid min interval: %w", err)
|
||||
} else {
|
||||
config.MinIntervalSeconds = durationToSeconds(interval)
|
||||
}
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// GetCurrentConfig returns the current configuration
|
||||
func (ui *UIProvider) GetCurrentConfig() interface{} {
|
||||
return ui.getCurrentVacuumConfig()
|
||||
}
|
||||
|
||||
// ApplyConfig applies the new configuration
|
||||
func (ui *UIProvider) ApplyConfig(config interface{}) error {
|
||||
vacuumConfig, ok := config.(*VacuumConfig)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid config type, expected *VacuumConfig")
|
||||
}
|
||||
|
||||
// Apply to detector
|
||||
if ui.detector != nil {
|
||||
ui.detector.SetEnabled(vacuumConfig.Enabled)
|
||||
ui.detector.SetGarbageThreshold(vacuumConfig.GarbageThreshold)
|
||||
ui.detector.SetScanInterval(secondsToDuration(vacuumConfig.ScanIntervalSeconds))
|
||||
ui.detector.SetMinVolumeAge(secondsToDuration(vacuumConfig.MinVolumeAgeSeconds))
|
||||
}
|
||||
|
||||
// Apply to scheduler
|
||||
if ui.scheduler != nil {
|
||||
ui.scheduler.SetEnabled(vacuumConfig.Enabled)
|
||||
ui.scheduler.SetMaxConcurrent(vacuumConfig.MaxConcurrent)
|
||||
ui.scheduler.SetMinInterval(secondsToDuration(vacuumConfig.MinIntervalSeconds))
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Applied vacuum configuration: enabled=%v, threshold=%.1f%%, scan_interval=%s, max_concurrent=%d",
|
||||
vacuumConfig.Enabled, vacuumConfig.GarbageThreshold*100, formatDurationForUser(vacuumConfig.ScanIntervalSeconds), vacuumConfig.MaxConcurrent)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getCurrentVacuumConfig gets the current configuration from detector and scheduler
|
||||
func (ui *UIProvider) getCurrentVacuumConfig() *VacuumConfig {
|
||||
config := &VacuumConfig{
|
||||
// Default values (fallback if detectors/schedulers are nil)
|
||||
Enabled: true,
|
||||
GarbageThreshold: 0.3,
|
||||
ScanIntervalSeconds: 30 * 60,
|
||||
MinVolumeAgeSeconds: 1 * 60 * 60,
|
||||
MaxConcurrent: 2,
|
||||
MinIntervalSeconds: 6 * 60 * 60,
|
||||
}
|
||||
|
||||
// Get current values from detector
|
||||
if ui.detector != nil {
|
||||
config.Enabled = ui.detector.IsEnabled()
|
||||
config.GarbageThreshold = ui.detector.GetGarbageThreshold()
|
||||
config.ScanIntervalSeconds = durationToSeconds(ui.detector.ScanInterval())
|
||||
config.MinVolumeAgeSeconds = durationToSeconds(ui.detector.GetMinVolumeAge())
|
||||
}
|
||||
|
||||
// Get current values from scheduler
|
||||
if ui.scheduler != nil {
|
||||
config.MaxConcurrent = ui.scheduler.GetMaxConcurrent()
|
||||
config.MinIntervalSeconds = durationToSeconds(ui.scheduler.GetMinInterval())
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
// RegisterUI registers the vacuum UI provider with the UI registry
|
||||
func RegisterUI(uiRegistry *types.UIRegistry, detector *VacuumDetector, scheduler *VacuumScheduler) {
|
||||
uiProvider := NewUIProvider(detector, scheduler)
|
||||
uiRegistry.RegisterUI(uiProvider)
|
||||
|
||||
glog.V(1).Infof("✅ Registered vacuum task UI provider")
|
||||
}
|
||||
|
||||
// Example: How to get the UI provider for external use
|
||||
func GetUIProvider(uiRegistry *types.UIRegistry) *UIProvider {
|
||||
provider := uiRegistry.GetProvider(types.TaskTypeVacuum)
|
||||
if provider == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if vacuumProvider, ok := provider.(*UIProvider); ok {
|
||||
return vacuumProvider
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,60 +1,184 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Task implements vacuum operation to reclaim disk space
|
||||
type Task struct {
|
||||
*tasks.BaseTask
|
||||
server string
|
||||
volumeID uint32
|
||||
server string
|
||||
volumeID uint32
|
||||
garbageThreshold float64
|
||||
}
|
||||
|
||||
// NewTask creates a new vacuum task instance
|
||||
func NewTask(server string, volumeID uint32) *Task {
|
||||
task := &Task{
|
||||
BaseTask: tasks.NewBaseTask(types.TaskTypeVacuum),
|
||||
server: server,
|
||||
volumeID: volumeID,
|
||||
BaseTask: tasks.NewBaseTask(types.TaskTypeVacuum),
|
||||
server: server,
|
||||
volumeID: volumeID,
|
||||
garbageThreshold: 0.3, // Default 30% threshold
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
// Execute executes the vacuum task
|
||||
// Execute performs the vacuum operation
|
||||
func (t *Task) Execute(params types.TaskParams) error {
|
||||
glog.Infof("Starting vacuum task for volume %d on server %s", t.volumeID, t.server)
|
||||
// Use BaseTask.ExecuteTask to handle logging initialization
|
||||
return t.ExecuteTask(context.Background(), params, t.executeImpl)
|
||||
}
|
||||
|
||||
// Simulate vacuum operation with progress updates
|
||||
steps := []struct {
|
||||
name string
|
||||
duration time.Duration
|
||||
progress float64
|
||||
}{
|
||||
{"Scanning volume", 1 * time.Second, 20},
|
||||
{"Identifying deleted files", 2 * time.Second, 50},
|
||||
{"Compacting data", 3 * time.Second, 80},
|
||||
{"Finalizing vacuum", 1 * time.Second, 100},
|
||||
// executeImpl is the actual vacuum implementation
|
||||
func (t *Task) executeImpl(ctx context.Context, params types.TaskParams) error {
|
||||
t.LogInfo("Starting vacuum for volume %d on server %s", t.volumeID, t.server)
|
||||
|
||||
// Parse garbage threshold from typed parameters
|
||||
if params.TypedParams != nil {
|
||||
if vacuumParams := params.TypedParams.GetVacuumParams(); vacuumParams != nil {
|
||||
t.garbageThreshold = vacuumParams.GarbageThreshold
|
||||
t.LogWithFields("INFO", "Using garbage threshold from parameters", map[string]interface{}{
|
||||
"threshold": t.garbageThreshold,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for _, step := range steps {
|
||||
if t.IsCancelled() {
|
||||
return fmt.Errorf("vacuum task cancelled")
|
||||
// Convert server address to gRPC address and use proper dial option
|
||||
grpcAddress := pb.ServerToGrpcAddress(t.server)
|
||||
var dialOpt grpc.DialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
if params.GrpcDialOption != nil {
|
||||
dialOpt = params.GrpcDialOption
|
||||
}
|
||||
|
||||
conn, err := grpc.NewClient(grpcAddress, dialOpt)
|
||||
if err != nil {
|
||||
t.LogError("Failed to connect to volume server %s: %v", t.server, err)
|
||||
return fmt.Errorf("failed to connect to volume server %s: %v", t.server, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := volume_server_pb.NewVolumeServerClient(conn)
|
||||
|
||||
// Step 1: Check vacuum eligibility
|
||||
t.SetProgress(10.0)
|
||||
t.LogDebug("Checking vacuum eligibility for volume %d", t.volumeID)
|
||||
|
||||
checkResp, err := client.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum check failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum check failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Check if garbage ratio meets threshold
|
||||
if checkResp.GarbageRatio < t.garbageThreshold {
|
||||
t.LogWarning("Volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum",
|
||||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100)
|
||||
return fmt.Errorf("volume %d garbage ratio %.2f%% is below threshold %.2f%%, skipping vacuum",
|
||||
t.volumeID, checkResp.GarbageRatio*100, t.garbageThreshold*100)
|
||||
}
|
||||
|
||||
t.LogWithFields("INFO", "Volume eligible for vacuum", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"garbage_ratio": checkResp.GarbageRatio,
|
||||
"threshold": t.garbageThreshold,
|
||||
"garbage_percent": checkResp.GarbageRatio * 100,
|
||||
})
|
||||
|
||||
// Step 2: Compact volume
|
||||
t.SetProgress(30.0)
|
||||
t.LogInfo("Starting compact for volume %d", t.volumeID)
|
||||
|
||||
compactStream, err := client.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum compact failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum compact failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Process compact stream and track progress
|
||||
var processedBytes int64
|
||||
var totalBytes int64
|
||||
|
||||
for {
|
||||
resp, err := compactStream.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
t.LogError("Vacuum compact stream error for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum compact stream error for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Vacuum task step: %s", step.name)
|
||||
t.SetProgress(step.progress)
|
||||
processedBytes = resp.ProcessedBytes
|
||||
if resp.LoadAvg_1M > 0 {
|
||||
totalBytes = int64(resp.LoadAvg_1M) // This is a rough approximation
|
||||
}
|
||||
|
||||
// Simulate work
|
||||
time.Sleep(step.duration)
|
||||
// Update progress based on processed bytes (30% to 70% of total progress)
|
||||
if totalBytes > 0 {
|
||||
compactProgress := float64(processedBytes) / float64(totalBytes)
|
||||
if compactProgress > 1.0 {
|
||||
compactProgress = 1.0
|
||||
}
|
||||
progress := 30.0 + (compactProgress * 40.0) // 30% to 70%
|
||||
t.SetProgress(progress)
|
||||
}
|
||||
|
||||
t.LogWithFields("DEBUG", "Volume compact progress", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"processed_bytes": processedBytes,
|
||||
"total_bytes": totalBytes,
|
||||
"compact_progress": fmt.Sprintf("%.1f%%", (float64(processedBytes)/float64(totalBytes))*100),
|
||||
})
|
||||
}
|
||||
|
||||
glog.Infof("Vacuum task completed for volume %d on server %s", t.volumeID, t.server)
|
||||
// Step 3: Commit vacuum changes
|
||||
t.SetProgress(80.0)
|
||||
t.LogInfo("Committing vacuum for volume %d", t.volumeID)
|
||||
|
||||
commitResp, err := client.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.LogError("Vacuum commit failed for volume %d: %v", t.volumeID, err)
|
||||
return fmt.Errorf("vacuum commit failed for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
// Step 4: Cleanup temporary files
|
||||
t.SetProgress(90.0)
|
||||
t.LogInfo("Cleaning up vacuum files for volume %d", t.volumeID)
|
||||
|
||||
_, err = client.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
// Log warning but don't fail the task
|
||||
t.LogWarning("Vacuum cleanup warning for volume %d: %v", t.volumeID, err)
|
||||
}
|
||||
|
||||
t.SetProgress(100.0)
|
||||
|
||||
newVolumeSize := commitResp.VolumeSize
|
||||
t.LogWithFields("INFO", "Successfully completed vacuum", map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"server": t.server,
|
||||
"new_volume_size": newVolumeSize,
|
||||
"garbage_reclaimed": true,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,9 +195,20 @@ func (t *Task) Validate(params types.TaskParams) error {
|
||||
|
||||
// EstimateTime estimates the time needed for the task
|
||||
func (t *Task) EstimateTime(params types.TaskParams) time.Duration {
|
||||
// Base time for vacuum operation
|
||||
baseTime := 25 * time.Second
|
||||
// Base time for vacuum operations - varies by volume size and garbage ratio
|
||||
// Typically vacuum is faster than EC encoding
|
||||
baseTime := 5 * time.Minute
|
||||
|
||||
// Could adjust based on volume size or usage patterns
|
||||
// Use default estimation since volume size is not available in typed params
|
||||
return baseTime
|
||||
}
|
||||
|
||||
// GetProgress returns the current progress
|
||||
func (t *Task) GetProgress() float64 {
|
||||
return t.BaseTask.GetProgress()
|
||||
}
|
||||
|
||||
// Cancel cancels the task
|
||||
func (t *Task) Cancel() error {
|
||||
return t.BaseTask.Cancel()
|
||||
}
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// VacuumDetector implements vacuum task detection using code instead of schemas
|
||||
type VacuumDetector struct {
|
||||
enabled bool
|
||||
garbageThreshold float64
|
||||
minVolumeAge time.Duration
|
||||
scanInterval time.Duration
|
||||
}
|
||||
|
||||
// Compile-time interface assertions
|
||||
var (
|
||||
_ types.TaskDetector = (*VacuumDetector)(nil)
|
||||
_ types.PolicyConfigurableDetector = (*VacuumDetector)(nil)
|
||||
)
|
||||
|
||||
// NewVacuumDetector creates a new simple vacuum detector
|
||||
func NewVacuumDetector() *VacuumDetector {
|
||||
return &VacuumDetector{
|
||||
enabled: true,
|
||||
garbageThreshold: 0.3,
|
||||
minVolumeAge: 24 * time.Hour,
|
||||
scanInterval: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTaskType returns the task type
|
||||
func (d *VacuumDetector) GetTaskType() types.TaskType {
|
||||
return types.TaskTypeVacuum
|
||||
}
|
||||
|
||||
// ScanForTasks scans for volumes that need vacuum operations
|
||||
func (d *VacuumDetector) ScanForTasks(volumeMetrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo) ([]*types.TaskDetectionResult, error) {
|
||||
if !d.enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var results []*types.TaskDetectionResult
|
||||
|
||||
for _, metric := range volumeMetrics {
|
||||
// Check if volume needs vacuum
|
||||
if metric.GarbageRatio >= d.garbageThreshold && metric.Age >= d.minVolumeAge {
|
||||
// Higher priority for volumes with more garbage
|
||||
priority := types.TaskPriorityNormal
|
||||
if metric.GarbageRatio > 0.6 {
|
||||
priority = types.TaskPriorityHigh
|
||||
}
|
||||
|
||||
result := &types.TaskDetectionResult{
|
||||
TaskType: types.TaskTypeVacuum,
|
||||
VolumeID: metric.VolumeID,
|
||||
Server: metric.Server,
|
||||
Collection: metric.Collection,
|
||||
Priority: priority,
|
||||
Reason: "Volume has excessive garbage requiring vacuum",
|
||||
Parameters: map[string]interface{}{
|
||||
"garbage_ratio": metric.GarbageRatio,
|
||||
"volume_age": metric.Age.String(),
|
||||
},
|
||||
ScheduleAt: time.Now(),
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Vacuum detector found %d volumes needing vacuum", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// ScanInterval returns how often this detector should scan
|
||||
func (d *VacuumDetector) ScanInterval() time.Duration {
|
||||
return d.scanInterval
|
||||
}
|
||||
|
||||
// IsEnabled returns whether this detector is enabled
|
||||
func (d *VacuumDetector) IsEnabled() bool {
|
||||
return d.enabled
|
||||
}
|
||||
|
||||
// Configuration setters
|
||||
|
||||
func (d *VacuumDetector) SetEnabled(enabled bool) {
|
||||
d.enabled = enabled
|
||||
}
|
||||
|
||||
func (d *VacuumDetector) SetGarbageThreshold(threshold float64) {
|
||||
d.garbageThreshold = threshold
|
||||
}
|
||||
|
||||
func (d *VacuumDetector) SetScanInterval(interval time.Duration) {
|
||||
d.scanInterval = interval
|
||||
}
|
||||
|
||||
func (d *VacuumDetector) SetMinVolumeAge(age time.Duration) {
|
||||
d.minVolumeAge = age
|
||||
}
|
||||
|
||||
// GetGarbageThreshold returns the current garbage threshold
|
||||
func (d *VacuumDetector) GetGarbageThreshold() float64 {
|
||||
return d.garbageThreshold
|
||||
}
|
||||
|
||||
// GetMinVolumeAge returns the minimum volume age
|
||||
func (d *VacuumDetector) GetMinVolumeAge() time.Duration {
|
||||
return d.minVolumeAge
|
||||
}
|
||||
|
||||
// GetScanInterval returns the scan interval
|
||||
func (d *VacuumDetector) GetScanInterval() time.Duration {
|
||||
return d.scanInterval
|
||||
}
|
||||
|
||||
// ConfigureFromPolicy configures the detector based on the maintenance policy
|
||||
func (d *VacuumDetector) ConfigureFromPolicy(policy interface{}) {
|
||||
// Type assert to the maintenance policy type we expect
|
||||
if maintenancePolicy, ok := policy.(interface {
|
||||
GetVacuumEnabled() bool
|
||||
GetVacuumGarbageRatio() float64
|
||||
}); ok {
|
||||
d.SetEnabled(maintenancePolicy.GetVacuumEnabled())
|
||||
d.SetGarbageThreshold(maintenancePolicy.GetVacuumGarbageRatio())
|
||||
} else {
|
||||
glog.V(1).Infof("Could not configure vacuum detector from policy: unsupported policy type")
|
||||
}
|
||||
}
|
||||
@@ -2,80 +2,71 @@ package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Factory creates vacuum task instances
|
||||
type Factory struct {
|
||||
*tasks.BaseTaskFactory
|
||||
}
|
||||
|
||||
// NewFactory creates a new vacuum task factory
|
||||
func NewFactory() *Factory {
|
||||
return &Factory{
|
||||
BaseTaskFactory: tasks.NewBaseTaskFactory(
|
||||
types.TaskTypeVacuum,
|
||||
[]string{"vacuum", "storage"},
|
||||
"Vacuum operation to reclaim disk space by removing deleted files",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Create creates a new vacuum task instance
|
||||
func (f *Factory) Create(params types.TaskParams) (types.TaskInterface, error) {
|
||||
// Validate parameters
|
||||
if params.VolumeID == 0 {
|
||||
return nil, fmt.Errorf("volume_id is required")
|
||||
}
|
||||
if params.Server == "" {
|
||||
return nil, fmt.Errorf("server is required")
|
||||
}
|
||||
|
||||
task := NewTask(params.Server, params.VolumeID)
|
||||
task.SetEstimatedDuration(task.EstimateTime(params))
|
||||
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// Shared detector and scheduler instances
|
||||
var (
|
||||
sharedDetector *VacuumDetector
|
||||
sharedScheduler *VacuumScheduler
|
||||
)
|
||||
|
||||
// getSharedInstances returns the shared detector and scheduler instances
|
||||
func getSharedInstances() (*VacuumDetector, *VacuumScheduler) {
|
||||
if sharedDetector == nil {
|
||||
sharedDetector = NewVacuumDetector()
|
||||
}
|
||||
if sharedScheduler == nil {
|
||||
sharedScheduler = NewVacuumScheduler()
|
||||
}
|
||||
return sharedDetector, sharedScheduler
|
||||
}
|
||||
|
||||
// GetSharedInstances returns the shared detector and scheduler instances (public access)
|
||||
func GetSharedInstances() (*VacuumDetector, *VacuumScheduler) {
|
||||
return getSharedInstances()
|
||||
}
|
||||
// Global variable to hold the task definition for configuration updates
|
||||
var globalTaskDef *base.TaskDefinition
|
||||
|
||||
// Auto-register this task when the package is imported
|
||||
func init() {
|
||||
factory := NewFactory()
|
||||
tasks.AutoRegister(types.TaskTypeVacuum, factory)
|
||||
RegisterVacuumTask()
|
||||
|
||||
// Get shared instances for all registrations
|
||||
detector, scheduler := getSharedInstances()
|
||||
|
||||
// Register with types registry
|
||||
tasks.AutoRegisterTypes(func(registry *types.TaskRegistry) {
|
||||
registry.RegisterTask(detector, scheduler)
|
||||
})
|
||||
|
||||
// Register with UI registry using the same instances
|
||||
tasks.AutoRegisterUI(func(uiRegistry *types.UIRegistry) {
|
||||
RegisterUI(uiRegistry, detector, scheduler)
|
||||
})
|
||||
// Register config updater
|
||||
tasks.AutoRegisterConfigUpdater(types.TaskTypeVacuum, UpdateConfigFromPersistence)
|
||||
}
|
||||
|
||||
// RegisterVacuumTask registers the vacuum task with the new architecture
|
||||
func RegisterVacuumTask() {
|
||||
// Create configuration instance
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Create complete task definition
|
||||
taskDef := &base.TaskDefinition{
|
||||
Type: types.TaskTypeVacuum,
|
||||
Name: "vacuum",
|
||||
DisplayName: "Volume Vacuum",
|
||||
Description: "Reclaims disk space by removing deleted files from volumes",
|
||||
Icon: "fas fa-broom text-primary",
|
||||
Capabilities: []string{"vacuum", "storage"},
|
||||
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: CreateTask,
|
||||
DetectionFunc: Detection,
|
||||
ScanInterval: 2 * time.Hour,
|
||||
SchedulingFunc: Scheduling,
|
||||
MaxConcurrent: 2,
|
||||
RepeatInterval: 7 * 24 * time.Hour,
|
||||
}
|
||||
|
||||
// Store task definition globally for configuration updates
|
||||
globalTaskDef = taskDef
|
||||
|
||||
// Register everything with a single function call!
|
||||
base.RegisterTask(taskDef)
|
||||
}
|
||||
|
||||
// UpdateConfigFromPersistence updates the vacuum configuration from persistence
|
||||
func UpdateConfigFromPersistence(configPersistence interface{}) error {
|
||||
if globalTaskDef == nil {
|
||||
return fmt.Errorf("vacuum task not registered")
|
||||
}
|
||||
|
||||
// Load configuration from persistence
|
||||
newConfig := LoadConfigFromPersistence(configPersistence)
|
||||
if newConfig == nil {
|
||||
return fmt.Errorf("failed to load configuration from persistence")
|
||||
}
|
||||
|
||||
// Update the task definition's config
|
||||
globalTaskDef.Config = newConfig
|
||||
|
||||
glog.V(1).Infof("Updated vacuum task configuration from persistence")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// VacuumScheduler implements vacuum task scheduling using code instead of schemas
|
||||
type VacuumScheduler struct {
|
||||
enabled bool
|
||||
maxConcurrent int
|
||||
minInterval time.Duration
|
||||
}
|
||||
|
||||
// Compile-time interface assertions
|
||||
var (
|
||||
_ types.TaskScheduler = (*VacuumScheduler)(nil)
|
||||
)
|
||||
|
||||
// NewVacuumScheduler creates a new simple vacuum scheduler
|
||||
func NewVacuumScheduler() *VacuumScheduler {
|
||||
return &VacuumScheduler{
|
||||
enabled: true,
|
||||
maxConcurrent: 2,
|
||||
minInterval: 6 * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTaskType returns the task type
|
||||
func (s *VacuumScheduler) GetTaskType() types.TaskType {
|
||||
return types.TaskTypeVacuum
|
||||
}
|
||||
|
||||
// CanScheduleNow determines if a vacuum task can be scheduled right now
|
||||
func (s *VacuumScheduler) CanScheduleNow(task *types.Task, runningTasks []*types.Task, availableWorkers []*types.Worker) bool {
|
||||
// Check if scheduler is enabled
|
||||
if !s.enabled {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check concurrent limit
|
||||
runningVacuumCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeVacuum {
|
||||
runningVacuumCount++
|
||||
}
|
||||
}
|
||||
|
||||
if runningVacuumCount >= s.maxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if there's an available worker with vacuum capability
|
||||
for _, worker := range availableWorkers {
|
||||
if worker.CurrentLoad < worker.MaxConcurrent {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeVacuum {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetPriority returns the priority for this task
|
||||
func (s *VacuumScheduler) GetPriority(task *types.Task) types.TaskPriority {
|
||||
// Could adjust priority based on task parameters
|
||||
if params, ok := task.Parameters["garbage_ratio"].(float64); ok {
|
||||
if params > 0.8 {
|
||||
return types.TaskPriorityHigh
|
||||
}
|
||||
}
|
||||
return task.Priority
|
||||
}
|
||||
|
||||
// GetMaxConcurrent returns max concurrent tasks of this type
|
||||
func (s *VacuumScheduler) GetMaxConcurrent() int {
|
||||
return s.maxConcurrent
|
||||
}
|
||||
|
||||
// GetDefaultRepeatInterval returns the default interval to wait before repeating vacuum tasks
|
||||
func (s *VacuumScheduler) GetDefaultRepeatInterval() time.Duration {
|
||||
return s.minInterval
|
||||
}
|
||||
|
||||
// IsEnabled returns whether this scheduler is enabled
|
||||
func (s *VacuumScheduler) IsEnabled() bool {
|
||||
return s.enabled
|
||||
}
|
||||
|
||||
// Configuration setters
|
||||
|
||||
func (s *VacuumScheduler) SetEnabled(enabled bool) {
|
||||
s.enabled = enabled
|
||||
}
|
||||
|
||||
func (s *VacuumScheduler) SetMaxConcurrent(max int) {
|
||||
s.maxConcurrent = max
|
||||
}
|
||||
|
||||
func (s *VacuumScheduler) SetMinInterval(interval time.Duration) {
|
||||
s.minInterval = interval
|
||||
}
|
||||
|
||||
// GetMinInterval returns the minimum interval
|
||||
func (s *VacuumScheduler) GetMinInterval() time.Duration {
|
||||
return s.minInterval
|
||||
}
|
||||
Reference in New Issue
Block a user