* initial design * added simulation as tests * reorganized the codebase to move the simulation framework and tests into their own dedicated package * integration test. ec worker task * remove "enhanced" reference * start master, volume servers, filer Current Status ✅ Master: Healthy and running (port 9333) ✅ Filer: Healthy and running (port 8888) ✅ Volume Servers: All 6 servers running (ports 8080-8085) 🔄 Admin/Workers: Will start when dependencies are ready * generate write load * tasks are assigned * admin start wtih grpc port. worker has its own working directory * Update .gitignore * working worker and admin. Task detection is not working yet. * compiles, detection uses volumeSizeLimitMB from master * compiles * worker retries connecting to admin * build and restart * rendering pending tasks * skip task ID column * sticky worker id * test canScheduleTaskNow * worker reconnect to admin * clean up logs * worker register itself first * worker can run ec work and report status but: 1. one volume should not be repeatedly worked on. 2. ec shards needs to be distributed and source data should be deleted. * move ec task logic * listing ec shards * local copy, ec. Need to distribute. * ec is mostly working now * distribution of ec shards needs improvement * need configuration to enable ec * show ec volumes * interval field UI component * rename * integration test with vauuming * garbage percentage threshold * fix warning * display ec shard sizes * fix ec volumes list * Update ui.go * show default values * ensure correct default value * MaintenanceConfig use ConfigField * use schema defined defaults * config * reduce duplication * refactor to use BaseUIProvider * each task register its schema * checkECEncodingCandidate use ecDetector * use vacuumDetector * use volumeSizeLimitMB * remove remove * remove unused * refactor * use new framework * remove v2 reference * refactor * left menu can scroll now * The maintenance manager was not being initialized when no data directory was configured for persistent storage. * saving config * Update task_config_schema_templ.go * enable/disable tasks * protobuf encoded task configurations * fix system settings * use ui component * remove logs * interface{} Reduction * reduce interface{} * reduce interface{} * avoid from/to map * reduce interface{} * refactor * keep it DRY * added logging * debug messages * debug level * debug * show the log caller line * use configured task policy * log level * handle admin heartbeat response * Update worker.go * fix EC rack and dc count * Report task status to admin server * fix task logging, simplify interface checking, use erasure_coding constants * factor in empty volume server during task planning * volume.list adds disk id * track disk id also * fix locking scheduled and manual scanning * add active topology * simplify task detector * ec task completed, but shards are not showing up * implement ec in ec_typed.go * adjust log level * dedup * implementing ec copying shards and only ecx files * use disk id when distributing ec shards 🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk 📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId 🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest 💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId]) 📂 File System: EC shards and metadata land in the exact disk directory planned * Delete original volume from all locations * clean up existing shard locations * local encoding and distributing * Update docker/admin_integration/EC-TESTING-README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * check volume id range * simplify * fix tests * fix types * clean up logs and tests --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
156 lines
4.6 KiB
Go
156 lines
4.6 KiB
Go
package base
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
|
)
|
|
|
|
// GenericFactory creates task instances using a TaskDefinition
|
|
type GenericFactory struct {
|
|
*tasks.BaseTaskFactory
|
|
taskDef *TaskDefinition
|
|
}
|
|
|
|
// NewGenericFactory creates a generic task factory
|
|
func NewGenericFactory(taskDef *TaskDefinition) *GenericFactory {
|
|
return &GenericFactory{
|
|
BaseTaskFactory: tasks.NewBaseTaskFactory(
|
|
taskDef.Type,
|
|
taskDef.Capabilities,
|
|
taskDef.Description,
|
|
),
|
|
taskDef: taskDef,
|
|
}
|
|
}
|
|
|
|
// Create creates a task instance using the task definition
|
|
func (f *GenericFactory) Create(params types.TaskParams) (types.TaskInterface, error) {
|
|
if f.taskDef.CreateTask == nil {
|
|
return nil, fmt.Errorf("no task creation function defined for %s", f.taskDef.Type)
|
|
}
|
|
return f.taskDef.CreateTask(params)
|
|
}
|
|
|
|
// GenericSchemaProvider provides config schema from TaskDefinition
|
|
type GenericSchemaProvider struct {
|
|
taskDef *TaskDefinition
|
|
}
|
|
|
|
// GetConfigSchema returns the schema from task definition
|
|
func (p *GenericSchemaProvider) GetConfigSchema() *tasks.TaskConfigSchema {
|
|
return &tasks.TaskConfigSchema{
|
|
TaskName: string(p.taskDef.Type),
|
|
DisplayName: p.taskDef.DisplayName,
|
|
Description: p.taskDef.Description,
|
|
Icon: p.taskDef.Icon,
|
|
Schema: config.Schema{
|
|
Fields: p.taskDef.ConfigSpec.Fields,
|
|
},
|
|
}
|
|
}
|
|
|
|
// GenericUIProvider provides UI functionality from TaskDefinition
|
|
type GenericUIProvider struct {
|
|
taskDef *TaskDefinition
|
|
}
|
|
|
|
// GetTaskType returns the task type
|
|
func (ui *GenericUIProvider) GetTaskType() types.TaskType {
|
|
return ui.taskDef.Type
|
|
}
|
|
|
|
// GetDisplayName returns the human-readable name
|
|
func (ui *GenericUIProvider) GetDisplayName() string {
|
|
return ui.taskDef.DisplayName
|
|
}
|
|
|
|
// GetDescription returns a description of what this task does
|
|
func (ui *GenericUIProvider) GetDescription() string {
|
|
return ui.taskDef.Description
|
|
}
|
|
|
|
// GetIcon returns the icon CSS class for this task type
|
|
func (ui *GenericUIProvider) GetIcon() string {
|
|
return ui.taskDef.Icon
|
|
}
|
|
|
|
// GetCurrentConfig returns current config as TaskConfig
|
|
func (ui *GenericUIProvider) GetCurrentConfig() types.TaskConfig {
|
|
return ui.taskDef.Config
|
|
}
|
|
|
|
// ApplyTaskPolicy applies protobuf TaskPolicy configuration
|
|
func (ui *GenericUIProvider) ApplyTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
|
return ui.taskDef.Config.FromTaskPolicy(policy)
|
|
}
|
|
|
|
// ApplyTaskConfig applies TaskConfig interface configuration
|
|
func (ui *GenericUIProvider) ApplyTaskConfig(config types.TaskConfig) error {
|
|
taskPolicy := config.ToTaskPolicy()
|
|
return ui.taskDef.Config.FromTaskPolicy(taskPolicy)
|
|
}
|
|
|
|
// RegisterTask registers a complete task definition with all registries
|
|
func RegisterTask(taskDef *TaskDefinition) {
|
|
// Validate task definition
|
|
if err := validateTaskDefinition(taskDef); err != nil {
|
|
glog.Errorf("Invalid task definition for %s: %v", taskDef.Type, err)
|
|
return
|
|
}
|
|
|
|
// Create and register factory
|
|
factory := NewGenericFactory(taskDef)
|
|
tasks.AutoRegister(taskDef.Type, factory)
|
|
|
|
// Create and register detector/scheduler
|
|
detector := NewGenericDetector(taskDef)
|
|
scheduler := NewGenericScheduler(taskDef)
|
|
|
|
tasks.AutoRegisterTypes(func(registry *types.TaskRegistry) {
|
|
registry.RegisterTask(detector, scheduler)
|
|
})
|
|
|
|
// Create and register schema provider
|
|
schemaProvider := &GenericSchemaProvider{taskDef: taskDef}
|
|
tasks.RegisterTaskConfigSchema(string(taskDef.Type), schemaProvider)
|
|
|
|
// Create and register UI provider
|
|
uiProvider := &GenericUIProvider{taskDef: taskDef}
|
|
tasks.AutoRegisterUI(func(uiRegistry *types.UIRegistry) {
|
|
baseUIProvider := tasks.NewBaseUIProvider(
|
|
taskDef.Type,
|
|
taskDef.DisplayName,
|
|
taskDef.Description,
|
|
taskDef.Icon,
|
|
schemaProvider.GetConfigSchema,
|
|
uiProvider.GetCurrentConfig,
|
|
uiProvider.ApplyTaskPolicy,
|
|
uiProvider.ApplyTaskConfig,
|
|
)
|
|
uiRegistry.RegisterUI(baseUIProvider)
|
|
})
|
|
|
|
glog.V(1).Infof("✅ Registered complete task definition: %s", taskDef.Type)
|
|
}
|
|
|
|
// validateTaskDefinition ensures the task definition is complete
|
|
func validateTaskDefinition(taskDef *TaskDefinition) error {
|
|
if taskDef.Type == "" {
|
|
return fmt.Errorf("task type is required")
|
|
}
|
|
if taskDef.Name == "" {
|
|
return fmt.Errorf("task name is required")
|
|
}
|
|
if taskDef.Config == nil {
|
|
return fmt.Errorf("task config is required")
|
|
}
|
|
// CreateTask is optional for tasks that use the typed task system
|
|
// The typed system registers tasks separately via types.RegisterGlobalTypedTask()
|
|
return nil
|
|
}
|