diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index 5a8d6c553..a6ca7eac9 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -212,8 +212,9 @@ To enable workers, add the following to your values.yaml: worker: enabled: true replicas: 2 # Scale based on workload - capabilities: "vacuum,balance,erasure_coding" # Tasks this worker can handle - maxConcurrent: 3 # Maximum concurrent tasks per worker + jobType: "vacuum,volume_balance,erasure_coding" # Job types this worker can handle + maxDetect: 1 # Maximum concurrent detection requests + maxExecute: 4 # Maximum concurrent execution jobs per worker # Working directory for task execution # Default: "/tmp/seaweedfs-worker" @@ -248,14 +249,14 @@ worker: memory: "2Gi" ``` -### Worker Capabilities +### Worker Job Types -Workers can be configured with different capabilities: +Workers can be configured with different job types: - **vacuum**: Reclaim deleted file space -- **balance**: Balance volumes across volume servers +- **volume_balance**: Balance volumes across volume servers - **erasure_coding**: Handle erasure coding operations -You can configure workers with all capabilities or create specialized worker pools with specific capabilities. +You can configure workers with all job types or create specialized worker pools with specific job types. ### Worker Deployment Strategy @@ -264,11 +265,11 @@ For production deployments, consider: 1. **Multiple Workers**: Deploy 2+ worker replicas for high availability 2. **Resource Allocation**: Workers need sufficient CPU/memory for maintenance tasks 3. **Storage**: Workers need temporary storage for vacuum and balance operations (size depends on volume size) -4. **Specialized Workers**: Create separate worker deployments for different capabilities if needed +4. **Specialized Workers**: Create separate worker deployments for different job types if needed Example specialized worker configuration: -For specialized worker pools, deploy separate Helm releases with different capabilities: +For specialized worker pools, deploy separate Helm releases with different job types: **values-worker-vacuum.yaml** (for vacuum operations): ```yaml @@ -287,8 +288,8 @@ admin: worker: enabled: true replicas: 2 - capabilities: "vacuum" - maxConcurrent: 2 + jobType: "vacuum" + maxExecute: 2 # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": @@ -313,8 +314,8 @@ admin: worker: enabled: true replicas: 1 - capabilities: "balance" - maxConcurrent: 1 + jobType: "volume_balance" + maxExecute: 1 # REQUIRED: Point to the admin service of your main SeaweedFS release # Replace with the namespace where your main seaweedfs is deployed # Example: If deploying in namespace "production": diff --git a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml index 23617df56..37c62dc98 100644 --- a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml @@ -136,11 +136,15 @@ spec: {{- else }} -admin={{ template "seaweedfs.fullname" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \ {{- end }} - -capabilities={{ .Values.worker.capabilities }} \ - -maxConcurrent={{ .Values.worker.maxConcurrent }} \ - -workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }} + -jobType={{ .Values.worker.jobType }} \ + -maxDetect={{ .Values.worker.maxDetect }} \ + -maxExecute={{ .Values.worker.maxExecute }} \ + -workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }} {{- if .Values.worker.metricsPort }} - -metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }} + -metricsPort={{ .Values.worker.metricsPort }}{{- if or .Values.worker.metricsIp .Values.worker.extraArgs }} \{{ end }} + {{- end }} + {{- if .Values.worker.metricsIp }} + -metricsIp={{ .Values.worker.metricsIp }}{{- if .Values.worker.extraArgs }} \{{ end }} {{- end }} {{- range $index, $arg := .Values.worker.extraArgs }} {{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 7e6c9b41b..8ce556b13 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1270,17 +1270,20 @@ worker: replicas: 1 loggingOverrideLevel: null metricsPort: 9327 + metricsIp: "" # If empty, defaults to 0.0.0.0 # Admin server to connect to adminServer: "" - # Worker capabilities - comma-separated list - # Available: vacuum, balance, erasure_coding - # Default: "vacuum,balance,erasure_coding" (all capabilities) - capabilities: "vacuum,balance,erasure_coding" + # Worker job types - comma-separated list + # Available: vacuum, volume_balance, erasure_coding + jobType: "vacuum,volume_balance,erasure_coding" - # Maximum number of concurrent tasks - maxConcurrent: 3 + # Maximum number of concurrent detection requests + maxDetect: 1 + + # Maximum number of concurrent execution jobs + maxExecute: 4 # Working directory for task execution workingDir: "/tmp/seaweedfs-worker" diff --git a/test/erasure_coding/admin_dockertest/ec_integration_test.go b/test/erasure_coding/admin_dockertest/ec_integration_test.go index 9f8c44c82..21f872ea1 100644 --- a/test/erasure_coding/admin_dockertest/ec_integration_test.go +++ b/test/erasure_coding/admin_dockertest/ec_integration_test.go @@ -2,13 +2,11 @@ package admin_dockertest import ( "bytes" + crand "crypto/rand" "encoding/json" "fmt" "io" - "io/ioutil" - "math/rand" "net/http" - "net/url" "os" "os/exec" "path/filepath" @@ -161,129 +159,95 @@ func TestEcEndToEnd(t *testing.T) { client := &http.Client{} - // 1. Configure Global Maintenance (Scan Interval = 1s) via API - t.Log("Configuring Global Maintenance via API...") + // 1. Configure plugin job types for fast EC detection/execution. + t.Log("Configuring plugin job types via API...") - // 1.1 Fetch current config - req, _ := http.NewRequest("GET", AdminUrl+"/api/maintenance/config", nil) + // Disable volume balance to reduce interference for this EC-focused test. + balanceConfig := map[string]interface{}{ + "job_type": "volume_balance", + "admin_runtime": map[string]interface{}{ + "enabled": false, + }, + } + jsonBody, err := json.Marshal(balanceConfig) + if err != nil { + t.Fatalf("Failed to marshal volume_balance config: %v", err) + } + req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody)) + if err != nil { + t.Fatalf("Failed to create volume_balance config request: %v", err) + } + req.Header.Set("Content-Type", "application/json") resp, err := client.Do(req) if err != nil { - t.Fatalf("Failed to get global config: %v", err) + t.Fatalf("Failed to update volume_balance config: %v", err) } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - t.Fatalf("Failed to get global config (status %d): %s", resp.StatusCode, string(body)) - } - - var globalConfig map[string]interface{} - if err := json.NewDecoder(resp.Body).Decode(&globalConfig); err != nil { - t.Fatalf("Failed to decode global config: %v", err) + t.Fatalf("Failed to update volume_balance config (status %d): %s", resp.StatusCode, string(body)) } resp.Body.Close() - // 1.2 Modify config - globalConfig["enabled"] = true - globalConfig["scan_interval_seconds"] = 1 - - // Ensure policy structure exists - if globalConfig["policy"] == nil { - globalConfig["policy"] = map[string]interface{}{} + ecConfig := map[string]interface{}{ + "job_type": "erasure_coding", + "admin_runtime": map[string]interface{}{ + "enabled": true, + "detection_interval_seconds": 1, + "global_execution_concurrency": 4, + "per_worker_execution_concurrency": 4, + "max_jobs_per_detection": 100, + }, + "worker_config_values": map[string]interface{}{ + "quiet_for_seconds": map[string]interface{}{ + "int64_value": "1", + }, + "min_interval_seconds": map[string]interface{}{ + "int64_value": "1", + }, + "min_size_mb": map[string]interface{}{ + "int64_value": "1", + }, + "fullness_ratio": map[string]interface{}{ + "double_value": 0.0001, + }, + }, } - policy, _ := globalConfig["policy"].(map[string]interface{}) - - // Ensure task_policies structure exists - if policy["task_policies"] == nil { - policy["task_policies"] = map[string]interface{}{} + jsonBody, err = json.Marshal(ecConfig) + if err != nil { + t.Fatalf("Failed to marshal erasure_coding config: %v", err) } - taskPolicies, _ := policy["task_policies"].(map[string]interface{}) - - // Disable balance tasks to avoid interference with EC test - if taskPolicies["balance"] == nil { - taskPolicies["balance"] = map[string]interface{}{} + req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/erasure_coding/config", bytes.NewBuffer(jsonBody)) + if err != nil { + t.Fatalf("Failed to create erasure_coding config request: %v", err) } - balancePolicy, _ := taskPolicies["balance"].(map[string]interface{}) - balancePolicy["enabled"] = false - - // Set global max concurrent - policy["global_max_concurrent"] = 4 - globalConfig["policy"] = policy - - // Explicitly set required fields - requiredFields := map[string]float64{ - "worker_timeout_seconds": 300, - "task_timeout_seconds": 7200, - "retry_delay_seconds": 900, - "cleanup_interval_seconds": 86400, - "task_retention_seconds": 604800, - "max_retries": 3, - } - for field, val := range requiredFields { - if _, ok := globalConfig[field]; !ok || globalConfig[field] == 0 { - globalConfig[field] = val - } - } - - // 1.3 Update config - jsonBody, _ := json.Marshal(globalConfig) - req, _ = http.NewRequest("PUT", AdminUrl+"/api/maintenance/config", bytes.NewBuffer(jsonBody)) req.Header.Set("Content-Type", "application/json") resp, err = client.Do(req) if err != nil { - t.Fatalf("Failed to update global config: %v", err) + t.Fatalf("Failed to update erasure_coding config: %v", err) } if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) - t.Fatalf("Failed to update global config (status %d): %s", resp.StatusCode, string(body)) + t.Fatalf("Failed to update erasure_coding config (status %d): %s", resp.StatusCode, string(body)) } resp.Body.Close() - // 2. Configure EC Task (Short intervals) via Form API - t.Log("Configuring EC Task via Form API...") - formData := url.Values{} - formData.Set("enabled", "true") - formData.Set("scan_interval_seconds", "1") - formData.Set("repeat_interval_seconds", "1") - formData.Set("check_interval_seconds", "1") - formData.Set("max_concurrent", "4") - formData.Set("quiet_for_seconds_value", "1") - formData.Set("quiet_for_seconds_unit", "seconds") - formData.Set("min_size_mb", "1") - formData.Set("fullness_ratio", "0.0001") - - req, _ = http.NewRequest("POST", AdminUrl+"/maintenance/config/erasure_coding", strings.NewReader(formData.Encode())) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - resp, err = client.Do(req) - if err != nil { - t.Fatalf("Failed to update EC config: %v", err) - } - if resp.StatusCode != 200 && resp.StatusCode != 303 { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("Failed to update EC config (status %d): %s", resp.StatusCode, string(body)) - } - resp.Body.Close() - t.Log("EC Task Configuration updated") - - // 3. Restart Admin to pick up Global Config (Scan Interval) - if len(runningCmds) > 0 { - adminCmd := runningCmds[len(runningCmds)-1] - t.Log("Restarting Admin Server to apply configuration...") - stopWeed(t, adminCmd) - time.Sleep(10 * time.Second) - startWeed(t, "admin_restarted", "admin", "-master=localhost:9333", "-port=23646", "-port.grpc=33646", "-dataDir=./tmp/admin") - waitForUrl(t, AdminUrl+"/health", 60) - } - - // 4. Upload a file + // 2. Upload a file fileSize := 5 * 1024 * 1024 data := make([]byte, fileSize) - rand.Read(data) + crand.Read(data) fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix()) t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName) uploadUrl := FilerUrl + "/" + fileName var uploadErr error for i := 0; i < 10; i++ { - req, _ := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data)) + req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data)) + if err != nil { + uploadErr = err + t.Logf("Upload attempt %d failed to create request: %v", i+1, err) + time.Sleep(2 * time.Second) + continue + } resp, err := client.Do(req) if err == nil { if resp.StatusCode == 201 { @@ -306,17 +270,17 @@ func TestEcEndToEnd(t *testing.T) { } t.Log("Upload successful") - // 5. Verify EC Encoding + // 3. Verify EC Encoding t.Log("Waiting for EC encoding (checking Master topology)...") startTime := time.Now() ecVerified := false var lastBody []byte for time.Since(startTime) < 300*time.Second { - // 5.1 Check Master Topology + // 3.1 Check Master Topology resp, err := http.Get(MasterUrl + "/dir/status") if err == nil { - lastBody, _ = ioutil.ReadAll(resp.Body) + lastBody, _ = io.ReadAll(resp.Body) resp.Body.Close() // Check total EC shards @@ -336,8 +300,8 @@ func TestEcEndToEnd(t *testing.T) { } } - // 5.2 Debug: Check workers and tasks - wResp, wErr := http.Get(AdminUrl + "/api/maintenance/workers") + // 3.2 Debug: Check workers and jobs + wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers") workerCount := 0 if wErr == nil { var workers []interface{} @@ -346,7 +310,7 @@ func TestEcEndToEnd(t *testing.T) { workerCount = len(workers) } - tResp, tErr := http.Get(AdminUrl + "/api/maintenance/tasks") + tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000") taskCount := 0 if tErr == nil { var tasks []interface{} diff --git a/unmaintained/stress/ec_stress_runner.go b/unmaintained/stress/ec_stress_runner.go new file mode 100644 index 000000000..3acd36c0d --- /dev/null +++ b/unmaintained/stress/ec_stress_runner.go @@ -0,0 +1,515 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + mrand "math/rand" + "net/http" + "net/url" + "os/signal" + "path" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type config struct { + MasterAddresses []string + FilerURL string + PathPrefix string + Collection string + FileSizeBytes int64 + BatchSize int + WriteInterval time.Duration + CleanupInterval time.Duration + EcMinAge time.Duration + MaxCleanupPerCycle int + RequestTimeout time.Duration + MaxRuntime time.Duration + DryRun bool +} + +type runner struct { + cfg config + + httpClient *http.Client + grpcDialOption grpc.DialOption + + mu sync.Mutex + sequence int64 + ecFirstSeenAt map[uint32]time.Time + rng *mrand.Rand +} + +type ecVolumeInfo struct { + Collection string + NodeShards map[pb.ServerAddress][]uint32 +} + +type ecCleanupCandidate struct { + VolumeID uint32 + FirstSeenAt time.Time + Info *ecVolumeInfo +} + +func main() { + cfg, err := loadConfig() + if err != nil { + log.Fatalf("invalid flags: %v", err) + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + if cfg.MaxRuntime > 0 { + runCtx, cancel := context.WithTimeout(ctx, cfg.MaxRuntime) + defer cancel() + ctx = runCtx + } + + r := &runner{ + cfg: cfg, + httpClient: &http.Client{Timeout: cfg.RequestTimeout}, + grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + ecFirstSeenAt: make(map[uint32]time.Time), + rng: mrand.New(mrand.NewSource(time.Now().UnixNano())), + } + + log.Printf( + "starting EC stress runner: masters=%s filer=%s prefix=%s collection=%s file_size=%d batch=%d write_interval=%s cleanup_interval=%s ec_min_age=%s max_cleanup=%d dry_run=%v", + strings.Join(cfg.MasterAddresses, ","), + cfg.FilerURL, + cfg.PathPrefix, + cfg.Collection, + cfg.FileSizeBytes, + cfg.BatchSize, + cfg.WriteInterval, + cfg.CleanupInterval, + cfg.EcMinAge, + cfg.MaxCleanupPerCycle, + cfg.DryRun, + ) + + if err := r.run(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + log.Fatalf("runner stopped with error: %v", err) + } + + log.Printf("runner stopped") +} + +func loadConfig() (config, error) { + var masters string + cfg := config{} + + flag.StringVar(&masters, "masters", "127.0.0.1:9333", "comma-separated master server addresses") + flag.StringVar(&cfg.FilerURL, "filer", "http://127.0.0.1:8888", "filer base URL") + flag.StringVar(&cfg.PathPrefix, "path_prefix", "/tmp/ec-stress", "filer path prefix for generated files") + flag.StringVar(&cfg.Collection, "collection", "ec_stress", "target collection for stress data") + + fileSizeMB := flag.Int("file_size_mb", 8, "size per generated file in MB") + flag.IntVar(&cfg.BatchSize, "batch_size", 4, "files generated per write cycle") + flag.DurationVar(&cfg.WriteInterval, "write_interval", 5*time.Second, "interval between write cycles") + flag.DurationVar(&cfg.CleanupInterval, "cleanup_interval", 2*time.Minute, "interval between EC cleanup cycles") + flag.DurationVar(&cfg.EcMinAge, "ec_min_age", 30*time.Minute, "minimum observed EC age before deletion") + flag.IntVar(&cfg.MaxCleanupPerCycle, "max_cleanup_per_cycle", 4, "maximum EC volumes deleted per cleanup cycle") + flag.DurationVar(&cfg.RequestTimeout, "request_timeout", 20*time.Second, "HTTP/gRPC request timeout") + flag.DurationVar(&cfg.MaxRuntime, "max_runtime", 0, "maximum run duration; 0 means run until interrupted") + flag.BoolVar(&cfg.DryRun, "dry_run", false, "log actions without deleting EC shards") + flag.Parse() + + cfg.MasterAddresses = splitNonEmpty(masters) + cfg.FileSizeBytes = int64(*fileSizeMB) * 1024 * 1024 + + if len(cfg.MasterAddresses) == 0 { + return cfg, fmt.Errorf("at least one master is required") + } + if cfg.FileSizeBytes <= 0 { + return cfg, fmt.Errorf("file_size_mb must be positive") + } + if cfg.BatchSize <= 0 { + return cfg, fmt.Errorf("batch_size must be positive") + } + if cfg.WriteInterval <= 0 { + return cfg, fmt.Errorf("write_interval must be positive") + } + if cfg.CleanupInterval <= 0 { + return cfg, fmt.Errorf("cleanup_interval must be positive") + } + if cfg.EcMinAge < 0 { + return cfg, fmt.Errorf("ec_min_age must be zero or positive") + } + // Note: EcMinAge == 0 intentionally disables the age guard, making EC volumes eligible for cleanup immediately. + if cfg.MaxCleanupPerCycle <= 0 { + return cfg, fmt.Errorf("max_cleanup_per_cycle must be positive") + } + if cfg.RequestTimeout <= 0 { + return cfg, fmt.Errorf("request_timeout must be positive") + } + + cfg.PathPrefix = ensureLeadingSlash(strings.TrimSpace(cfg.PathPrefix)) + cfg.Collection = strings.TrimSpace(cfg.Collection) + + cfg.FilerURL = strings.TrimRight(strings.TrimSpace(cfg.FilerURL), "/") + if cfg.FilerURL == "" { + return cfg, fmt.Errorf("filer URL is required") + } + + if _, err := url.ParseRequestURI(cfg.FilerURL); err != nil { + return cfg, fmt.Errorf("invalid filer URL %q: %w", cfg.FilerURL, err) + } + + return cfg, nil +} + +func (r *runner) run(ctx context.Context) error { + writeTicker := time.NewTicker(r.cfg.WriteInterval) + defer writeTicker.Stop() + + cleanupTicker := time.NewTicker(r.cfg.CleanupInterval) + defer cleanupTicker.Stop() + + r.runWriteCycle(ctx) + r.runCleanupCycle(ctx) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-writeTicker.C: + r.runWriteCycle(ctx) + case <-cleanupTicker.C: + r.runCleanupCycle(ctx) + } + } +} + +func (r *runner) runWriteCycle(ctx context.Context) { + for i := 0; i < r.cfg.BatchSize; i++ { + if ctx.Err() != nil { + return + } + if err := r.uploadOneFile(ctx); err != nil { + log.Printf("upload failed: %v", err) + } + } +} + +func (r *runner) uploadOneFile(ctx context.Context) error { + sequence := r.nextSequence() + filePath := path.Join(r.cfg.PathPrefix, fmt.Sprintf("ec-stress-%d-%d.bin", time.Now().UnixNano(), sequence)) + fileURL := r.cfg.FilerURL + filePath + if r.cfg.Collection != "" { + fileURL += "?collection=" + url.QueryEscape(r.cfg.Collection) + } + + uploadCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout) + defer cancel() + + body := io.LimitReader(r.rng, r.cfg.FileSizeBytes) + request, err := http.NewRequestWithContext(uploadCtx, http.MethodPut, fileURL, body) + if err != nil { + return err + } + request.ContentLength = r.cfg.FileSizeBytes + request.Header.Set("Content-Type", "application/octet-stream") + + response, err := r.httpClient.Do(request) + if err != nil { + return err + } + defer response.Body.Close() + + io.Copy(io.Discard, response.Body) + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("upload %s returned %s", filePath, response.Status) + } + + log.Printf("uploaded %s size=%d", filePath, r.cfg.FileSizeBytes) + return nil +} + +func (r *runner) runCleanupCycle(ctx context.Context) { + volumeList, err := r.fetchVolumeList(ctx) + if err != nil { + log.Printf("cleanup skipped: fetch volume list failed: %v", err) + return + } + if volumeList == nil || volumeList.TopologyInfo == nil { + log.Printf("cleanup skipped: topology is empty") + return + } + + ecVolumes := collectEcVolumes(volumeList.TopologyInfo, r.cfg.Collection) + candidates := r.selectCleanupCandidates(ecVolumes) + if len(candidates) == 0 { + log.Printf("cleanup: no EC volume candidate aged >= %s in collection=%q", r.cfg.EcMinAge, r.cfg.Collection) + return + } + + log.Printf("cleanup: deleting up to %d EC volumes (found=%d)", r.cfg.MaxCleanupPerCycle, len(candidates)) + deleted := 0 + for _, candidate := range candidates { + if ctx.Err() != nil { + return + } + + if r.cfg.DryRun { + log.Printf( + "cleanup dry-run: would delete EC volume=%d collection=%q seen_for=%s nodes=%d", + candidate.VolumeID, + candidate.Info.Collection, + time.Since(candidate.FirstSeenAt).Round(time.Second), + len(candidate.Info.NodeShards), + ) + continue + } + + if err := r.deleteEcVolume(ctx, candidate.VolumeID, candidate.Info); err != nil { + log.Printf("cleanup volume=%d failed: %v", candidate.VolumeID, err) + continue + } + + deleted++ + r.mu.Lock() + delete(r.ecFirstSeenAt, candidate.VolumeID) + r.mu.Unlock() + log.Printf("cleanup volume=%d completed", candidate.VolumeID) + } + + log.Printf("cleanup finished: deleted=%d attempted=%d", deleted, len(candidates)) +} + +func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResponse, error) { + var lastErr error + for _, master := range r.cfg.MasterAddresses { + masterAddress := strings.TrimSpace(master) + if masterAddress == "" { + continue + } + + var response *master_pb.VolumeListResponse + err := pb.WithMasterClient(false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error { + callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout) + defer cancel() + + resp, callErr := client.VolumeList(callCtx, &master_pb.VolumeListRequest{}) + if callErr != nil { + return callErr + } + response = resp + return nil + }) + if err == nil { + return response, nil + } + + lastErr = err + } + + if lastErr == nil { + lastErr = fmt.Errorf("no valid master address") + } + return nil, lastErr +} + +func collectEcVolumes(topology *master_pb.TopologyInfo, collection string) map[uint32]*ecVolumeInfo { + normalizedCollection := strings.TrimSpace(collection) + volumeShardSets := make(map[uint32]map[pb.ServerAddress]map[uint32]struct{}) + volumeCollection := make(map[uint32]string) + + for _, dc := range topology.GetDataCenterInfos() { + for _, rack := range dc.GetRackInfos() { + for _, node := range rack.GetDataNodeInfos() { + server := pb.NewServerAddressFromDataNode(node) + for _, disk := range node.GetDiskInfos() { + for _, shardInfo := range disk.GetEcShardInfos() { + if shardInfo == nil || shardInfo.Id == 0 { + continue + } + if normalizedCollection != "" && strings.TrimSpace(shardInfo.Collection) != normalizedCollection { + continue + } + + shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).IdsUint32() + if len(shards) == 0 { + continue + } + + perVolume := volumeShardSets[shardInfo.Id] + if perVolume == nil { + perVolume = make(map[pb.ServerAddress]map[uint32]struct{}) + volumeShardSets[shardInfo.Id] = perVolume + } + perNode := perVolume[server] + if perNode == nil { + perNode = make(map[uint32]struct{}) + perVolume[server] = perNode + } + for _, shardID := range shards { + perNode[shardID] = struct{}{} + } + volumeCollection[shardInfo.Id] = shardInfo.Collection + } + } + } + } + } + + result := make(map[uint32]*ecVolumeInfo, len(volumeShardSets)) + for volumeID, perNode := range volumeShardSets { + info := &ecVolumeInfo{ + Collection: volumeCollection[volumeID], + NodeShards: make(map[pb.ServerAddress][]uint32, len(perNode)), + } + for server, shardSet := range perNode { + shardIDs := make([]uint32, 0, len(shardSet)) + for shardID := range shardSet { + shardIDs = append(shardIDs, shardID) + } + sort.Slice(shardIDs, func(i, j int) bool { return shardIDs[i] < shardIDs[j] }) + info.NodeShards[server] = shardIDs + } + result[volumeID] = info + } + + return result +} + +func (r *runner) selectCleanupCandidates(ecVolumes map[uint32]*ecVolumeInfo) []ecCleanupCandidate { + now := time.Now() + + r.mu.Lock() + defer r.mu.Unlock() + + for volumeID := range ecVolumes { + if _, exists := r.ecFirstSeenAt[volumeID]; !exists { + r.ecFirstSeenAt[volumeID] = now + } + } + for volumeID := range r.ecFirstSeenAt { + if _, exists := ecVolumes[volumeID]; !exists { + delete(r.ecFirstSeenAt, volumeID) + } + } + + candidates := make([]ecCleanupCandidate, 0, len(ecVolumes)) + for volumeID, info := range ecVolumes { + firstSeenAt := r.ecFirstSeenAt[volumeID] + if r.cfg.EcMinAge > 0 && now.Sub(firstSeenAt) < r.cfg.EcMinAge { + continue + } + candidates = append(candidates, ecCleanupCandidate{ + VolumeID: volumeID, + FirstSeenAt: firstSeenAt, + Info: info, + }) + } + + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].FirstSeenAt.Equal(candidates[j].FirstSeenAt) { + return candidates[i].VolumeID < candidates[j].VolumeID + } + return candidates[i].FirstSeenAt.Before(candidates[j].FirstSeenAt) + }) + + if len(candidates) > r.cfg.MaxCleanupPerCycle { + candidates = candidates[:r.cfg.MaxCleanupPerCycle] + } + return candidates +} + +func (r *runner) deleteEcVolume(ctx context.Context, volumeID uint32, info *ecVolumeInfo) error { + if info == nil { + return fmt.Errorf("ec volume %d has no topology info", volumeID) + } + + failureCount := 0 + for server, shardIDs := range info.NodeShards { + err := pb.WithVolumeServerClient(false, server, r.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + unmountCtx, unmountCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout) + defer unmountCancel() + if _, err := client.VolumeEcShardsUnmount(unmountCtx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: volumeID, + ShardIds: shardIDs, + }); err != nil { + log.Printf("volume %d ec shards unmount on %s failed: %v", volumeID, server, err) + } + + if len(shardIDs) > 0 { + deleteCtx, deleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout) + defer deleteCancel() + if _, err := client.VolumeEcShardsDelete(deleteCtx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: volumeID, + Collection: r.cfg.Collection, + ShardIds: shardIDs, + }); err != nil { + return err + } + } + + finalDeleteCtx, finalDeleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout) + defer finalDeleteCancel() + if _, err := client.VolumeDelete(finalDeleteCtx, &volume_server_pb.VolumeDeleteRequest{ + VolumeId: volumeID, + }); err != nil { + log.Printf("volume %d delete on %s failed: %v", volumeID, server, err) + } + return nil + }) + + if err != nil { + failureCount++ + log.Printf("cleanup volume=%d server=%s shards=%v failed: %v", volumeID, server, shardIDs, err) + } + } + + if failureCount == len(info.NodeShards) && failureCount > 0 { + return fmt.Errorf("all shard deletions failed for volume %d", volumeID) + } + if failureCount > 0 { + return fmt.Errorf("partial shard deletion failure for volume %d", volumeID) + } + return nil +} + +func (r *runner) nextSequence() int64 { + r.mu.Lock() + defer r.mu.Unlock() + r.sequence++ + return r.sequence +} + +func splitNonEmpty(value string) []string { + parts := strings.Split(value, ",") + result := make([]string, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +func ensureLeadingSlash(value string) string { + if value == "" { + return "/" + } + if strings.HasPrefix(value, "/") { + return value + } + return "/" + value +} diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 346e717d9..a9aa050b0 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" + adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin" "github.com/seaweedfs/seaweedfs/weed/cluster" "github.com/seaweedfs/seaweedfs/weed/credential" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -17,6 +18,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" @@ -33,14 +35,9 @@ import ( ) const ( - maxAssignmentHistoryDisplay = 50 - maxLogMessageLength = 2000 - maxLogFields = 20 - maxRelatedTasksDisplay = 50 - maxRecentTasksDisplay = 10 - defaultCacheTimeout = 10 * time.Second - defaultFilerCacheTimeout = 30 * time.Second - defaultStatsCacheTimeout = 30 * time.Second + defaultCacheTimeout = 10 * time.Second + defaultFilerCacheTimeout = 30 * time.Second + defaultStatsCacheTimeout = 30 * time.Second ) // FilerConfig holds filer configuration needed for bucket operations @@ -101,6 +98,7 @@ type AdminServer struct { // Maintenance system maintenanceManager *maintenance.MaintenanceManager + plugin *adminplugin.Plugin // Topic retention purger topicRetentionPurger *TopicRetentionPurger @@ -226,6 +224,28 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, }() } + plugin, err := adminplugin.New(adminplugin.Options{ + DataDir: dataDir, + ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) { + return server.buildDefaultPluginClusterContext(), nil + }, + }) + if err != nil && dataDir != "" { + glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err) + plugin, err = adminplugin.New(adminplugin.Options{ + DataDir: "", + ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) { + return server.buildDefaultPluginClusterContext(), nil + }, + }) + } + if err != nil { + glog.Errorf("Failed to initialize plugin: %v", err) + } else { + server.plugin = plugin + glog.V(0).Infof("Plugin enabled") + } + return server } @@ -795,751 +815,6 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) { // VacuumVolume method moved to volume_management.go -// ShowMaintenanceQueue displays the maintenance queue page -func (as *AdminServer) ShowMaintenanceQueue(c *gin.Context) { - data, err := as.GetMaintenanceQueueData() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // This should not render HTML template, it should use the component approach - c.JSON(http.StatusOK, data) -} - -// ShowMaintenanceWorkers displays the maintenance workers page -func (as *AdminServer) ShowMaintenanceWorkers(c *gin.Context) { - workers, err := as.getMaintenanceWorkers() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // Create worker details data - workersData := make([]*WorkerDetailsData, 0, len(workers)) - for _, worker := range workers { - details, err := as.getMaintenanceWorkerDetails(worker.ID) - if err != nil { - // Create basic worker details if we can't get full details - details = &WorkerDetailsData{ - Worker: worker, - CurrentTasks: []*MaintenanceTask{}, - RecentTasks: []*MaintenanceTask{}, - Performance: &WorkerPerformance{ - TasksCompleted: 0, - TasksFailed: 0, - AverageTaskTime: 0, - Uptime: 0, - SuccessRate: 0, - }, - LastUpdated: time.Now(), - } - } - workersData = append(workersData, details) - } - - c.JSON(http.StatusOK, gin.H{ - "workers": workersData, - "title": "Maintenance Workers", - }) -} - -// ShowMaintenanceConfig displays the maintenance configuration page -func (as *AdminServer) ShowMaintenanceConfig(c *gin.Context) { - config, err := as.getMaintenanceConfig() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // This should not render HTML template, it should use the component approach - c.JSON(http.StatusOK, config) -} - -// UpdateMaintenanceConfig updates maintenance configuration from form -func (as *AdminServer) UpdateMaintenanceConfig(c *gin.Context) { - var config MaintenanceConfig - if err := c.ShouldBind(&config); err != nil { - c.HTML(http.StatusBadRequest, "error.html", gin.H{"error": err.Error()}) - return - } - - err := as.updateMaintenanceConfig(&config) - if err != nil { - c.HTML(http.StatusInternalServerError, "error.html", gin.H{"error": err.Error()}) - return - } - - c.Redirect(http.StatusSeeOther, "/maintenance/config") -} - -// TriggerMaintenanceScan triggers a maintenance scan -func (as *AdminServer) TriggerMaintenanceScan(c *gin.Context) { - err := as.triggerMaintenanceScan() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"success": false, "error": err.Error()}) - return - } - - c.JSON(http.StatusOK, gin.H{"success": true, "message": "Maintenance scan triggered"}) -} - -// GetMaintenanceTasks returns all maintenance tasks -func (as *AdminServer) GetMaintenanceTasks(c *gin.Context) { - tasks, err := as.GetAllMaintenanceTasks() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.JSON(http.StatusOK, tasks) -} - -// GetMaintenanceTask returns a specific maintenance task -func (as *AdminServer) GetMaintenanceTask(c *gin.Context) { - taskID := c.Param("id") - task, err := as.getMaintenanceTask(taskID) - if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Task not found"}) - return - } - - c.JSON(http.StatusOK, task) -} - -// GetMaintenanceTaskDetailAPI returns detailed task information via API -func (as *AdminServer) GetMaintenanceTaskDetailAPI(c *gin.Context) { - taskID := c.Param("id") - taskDetail, err := as.GetMaintenanceTaskDetail(taskID) - if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Task detail not found", "details": err.Error()}) - return - } - - c.JSON(http.StatusOK, taskDetail) -} - -// ShowMaintenanceTaskDetail renders the task detail page -func (as *AdminServer) ShowMaintenanceTaskDetail(c *gin.Context) { - username := c.GetString("username") - if username == "" { - username = "admin" // Default fallback - } - - taskID := c.Param("id") - taskDetail, err := as.GetMaintenanceTaskDetail(taskID) - if err != nil { - c.HTML(http.StatusNotFound, "error.html", gin.H{ - "error": "Task not found", - "details": err.Error(), - }) - return - } - - // Prepare data for template - data := gin.H{ - "username": username, - "task": taskDetail.Task, - "taskDetail": taskDetail, - "title": fmt.Sprintf("Task Detail - %s", taskID), - } - - c.HTML(http.StatusOK, "task_detail.html", data) -} - -// CancelMaintenanceTask cancels a pending maintenance task -func (as *AdminServer) CancelMaintenanceTask(c *gin.Context) { - taskID := c.Param("id") - err := as.cancelMaintenanceTask(taskID) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"success": false, "error": err.Error()}) - return - } - - c.JSON(http.StatusOK, gin.H{"success": true, "message": "Task cancelled"}) -} - -// cancelMaintenanceTask cancels a pending maintenance task -func (as *AdminServer) cancelMaintenanceTask(taskID string) error { - if as.maintenanceManager == nil { - return fmt.Errorf("maintenance manager not initialized") - } - - return as.maintenanceManager.CancelTask(taskID) -} - -// GetMaintenanceWorkersAPI returns all maintenance workers -func (as *AdminServer) GetMaintenanceWorkersAPI(c *gin.Context) { - workers, err := as.getMaintenanceWorkers() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.JSON(http.StatusOK, workers) -} - -// GetMaintenanceWorker returns a specific maintenance worker -func (as *AdminServer) GetMaintenanceWorker(c *gin.Context) { - workerID := c.Param("id") - worker, err := as.getMaintenanceWorkerDetails(workerID) - if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Worker not found"}) - return - } - - c.JSON(http.StatusOK, worker) -} - -// GetMaintenanceStats returns maintenance statistics -func (as *AdminServer) GetMaintenanceStats(c *gin.Context) { - stats, err := as.getMaintenanceStats() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.JSON(http.StatusOK, stats) -} - -// GetMaintenanceConfigAPI returns maintenance configuration -func (as *AdminServer) GetMaintenanceConfigAPI(c *gin.Context) { - config, err := as.getMaintenanceConfig() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.JSON(http.StatusOK, config) -} - -// UpdateMaintenanceConfigAPI updates maintenance configuration via API -func (as *AdminServer) UpdateMaintenanceConfigAPI(c *gin.Context) { - // Parse JSON into a generic map first to handle type conversions - var jsonConfig map[string]interface{} - if err := c.ShouldBindJSON(&jsonConfig); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - // Convert JSON map to protobuf configuration - config, err := convertJSONToMaintenanceConfig(jsonConfig) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse configuration: " + err.Error()}) - return - } - - err = as.updateMaintenanceConfig(config) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.JSON(http.StatusOK, gin.H{"success": true, "message": "Configuration updated"}) -} - -// GetMaintenanceConfigData returns maintenance configuration data (public wrapper) -func (as *AdminServer) GetMaintenanceConfigData() (*maintenance.MaintenanceConfigData, error) { - return as.getMaintenanceConfig() -} - -// UpdateMaintenanceConfigData updates maintenance configuration (public wrapper) -func (as *AdminServer) UpdateMaintenanceConfigData(config *maintenance.MaintenanceConfig) error { - return as.updateMaintenanceConfig(config) -} - -// Helper methods for maintenance operations - -// GetMaintenanceQueueData returns data for the maintenance queue UI -func (as *AdminServer) GetMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) { - tasks, err := as.GetAllMaintenanceTasks() - if err != nil { - return nil, err - } - - workers, err := as.getMaintenanceWorkers() - if err != nil { - return nil, err - } - - stats, err := as.getMaintenanceQueueStats() - if err != nil { - return nil, err - } - - return &maintenance.MaintenanceQueueData{ - Tasks: tasks, - Workers: workers, - Stats: stats, - LastUpdated: time.Now(), - }, nil -} - -// GetMaintenanceQueueStats returns statistics for the maintenance queue (exported for handlers) -func (as *AdminServer) GetMaintenanceQueueStats() (*maintenance.QueueStats, error) { - return as.getMaintenanceQueueStats() -} - -// getMaintenanceQueueStats returns statistics for the maintenance queue -func (as *AdminServer) getMaintenanceQueueStats() (*maintenance.QueueStats, error) { - if as.maintenanceManager == nil { - return &maintenance.QueueStats{ - PendingTasks: 0, - RunningTasks: 0, - CompletedToday: 0, - FailedToday: 0, - TotalTasks: 0, - }, nil - } - - // Get real statistics from maintenance manager - stats := as.maintenanceManager.GetStats() - - // Convert MaintenanceStats to QueueStats - queueStats := &maintenance.QueueStats{ - PendingTasks: stats.TasksByStatus[maintenance.TaskStatusPending], - RunningTasks: stats.TasksByStatus[maintenance.TaskStatusAssigned] + stats.TasksByStatus[maintenance.TaskStatusInProgress], - CompletedToday: stats.CompletedToday, - FailedToday: stats.FailedToday, - TotalTasks: stats.TotalTasks, - } - - return queueStats, nil -} - -// GetAllMaintenanceTasks returns all maintenance tasks -func (as *AdminServer) GetAllMaintenanceTasks() ([]*maintenance.MaintenanceTask, error) { - if as.maintenanceManager == nil { - return []*maintenance.MaintenanceTask{}, nil - } - - // 1. Collect all tasks from memory - tasksMap := make(map[string]*maintenance.MaintenanceTask) - - // Collect from memory via GetTasks loop to ensure we catch everything - statuses := []maintenance.MaintenanceTaskStatus{ - maintenance.TaskStatusPending, - maintenance.TaskStatusAssigned, - maintenance.TaskStatusInProgress, - maintenance.TaskStatusCompleted, - maintenance.TaskStatusFailed, - maintenance.TaskStatusCancelled, - } - - for _, status := range statuses { - tasks := as.maintenanceManager.GetTasks(status, "", 0) - for _, t := range tasks { - tasksMap[t.ID] = t - } - } - - // 2. Merge persisted tasks - if as.configPersistence != nil { - persistedTasks, err := as.configPersistence.LoadAllTaskStates() - if err == nil { - for _, t := range persistedTasks { - if _, exists := tasksMap[t.ID]; !exists { - tasksMap[t.ID] = t - } - } - } - } - - // 3. Bucketize buckets - var pendingTasks, activeTasks, finishedTasks []*maintenance.MaintenanceTask - - for _, t := range tasksMap { - switch t.Status { - case maintenance.TaskStatusPending: - pendingTasks = append(pendingTasks, t) - case maintenance.TaskStatusAssigned, maintenance.TaskStatusInProgress: - activeTasks = append(activeTasks, t) - case maintenance.TaskStatusCompleted, maintenance.TaskStatusFailed, maintenance.TaskStatusCancelled: - finishedTasks = append(finishedTasks, t) - default: - // Treat unknown as finished/archived? Or pending? - // Safest to add to finished so they appear somewhere - finishedTasks = append(finishedTasks, t) - } - } - - // 4. Sort buckets - // Pending: Newest Created First - sort.Slice(pendingTasks, func(i, j int) bool { - return pendingTasks[i].CreatedAt.After(pendingTasks[j].CreatedAt) - }) - - // Active: Newest Created First (or StartedAt?) - sort.Slice(activeTasks, func(i, j int) bool { - return activeTasks[i].CreatedAt.After(activeTasks[j].CreatedAt) - }) - - // Finished: Newest Completed First - sort.Slice(finishedTasks, func(i, j int) bool { - t1 := finishedTasks[i].CompletedAt - t2 := finishedTasks[j].CompletedAt - - // Handle nil completion times - if t1 == nil && t2 == nil { - // Both nil, fallback to CreatedAt - if !finishedTasks[i].CreatedAt.Equal(finishedTasks[j].CreatedAt) { - return finishedTasks[i].CreatedAt.After(finishedTasks[j].CreatedAt) - } - return finishedTasks[i].ID > finishedTasks[j].ID - } - if t1 == nil { - return false // t1 (nil) goes to bottom - } - if t2 == nil { - return true // t2 (nil) goes to bottom - } - - // Compare completion times - if !t1.Equal(*t2) { - return t1.After(*t2) - } - - // Fallback to CreatedAt if completion times are identical - if !finishedTasks[i].CreatedAt.Equal(finishedTasks[j].CreatedAt) { - return finishedTasks[i].CreatedAt.After(finishedTasks[j].CreatedAt) - } - - // Final tie-breaker: ID - return finishedTasks[i].ID > finishedTasks[j].ID - }) - - // 5. Recombine - allTasks := make([]*maintenance.MaintenanceTask, 0, len(tasksMap)) - allTasks = append(allTasks, pendingTasks...) - allTasks = append(allTasks, activeTasks...) - allTasks = append(allTasks, finishedTasks...) - - return allTasks, nil -} - -// getMaintenanceTask returns a specific maintenance task -func (as *AdminServer) getMaintenanceTask(taskID string) (*maintenance.MaintenanceTask, error) { - if as.maintenanceManager == nil { - return nil, fmt.Errorf("maintenance manager not initialized") - } - - // Search for the task across all statuses since we don't know which status it has - statuses := []maintenance.MaintenanceTaskStatus{ - maintenance.TaskStatusPending, - maintenance.TaskStatusAssigned, - maintenance.TaskStatusInProgress, - maintenance.TaskStatusCompleted, - maintenance.TaskStatusFailed, - maintenance.TaskStatusCancelled, - } - - // First, search for the task in memory across all statuses - for _, status := range statuses { - tasks := as.maintenanceManager.GetTasks(status, "", 0) // Get all tasks with this status - for _, task := range tasks { - if task.ID == taskID { - return task, nil - } - } - } - - // If not found in memory, try to load from persistent storage - if as.configPersistence != nil { - task, err := as.configPersistence.LoadTaskState(taskID) - if err == nil { - glog.V(2).Infof("Loaded task %s from persistent storage", taskID) - return task, nil - } - glog.V(2).Infof("Task %s not found in persistent storage: %v", taskID, err) - } - - return nil, fmt.Errorf("task %s not found", taskID) -} - -// GetMaintenanceTaskDetail returns comprehensive task details including logs and assignment history -func (as *AdminServer) GetMaintenanceTaskDetail(taskID string) (*maintenance.TaskDetailData, error) { - // Get basic task information - task, err := as.getMaintenanceTask(taskID) - if err != nil { - return nil, err - } - - // Copy task and truncate assignment history for display - displayTask := *task - displayTask.AssignmentHistory = nil // History is provided separately in taskDetail - - // Create task detail structure from the loaded task - taskDetail := &maintenance.TaskDetailData{ - Task: &displayTask, - AssignmentHistory: task.AssignmentHistory, // Use assignment history from persisted task - ExecutionLogs: []*maintenance.TaskExecutionLog{}, - RelatedTasks: []*maintenance.MaintenanceTask{}, - LastUpdated: time.Now(), - } - - // Truncate assignment history if it's too long (display last N only) - if len(taskDetail.AssignmentHistory) > maxAssignmentHistoryDisplay { - startIdx := len(taskDetail.AssignmentHistory) - maxAssignmentHistoryDisplay - taskDetail.AssignmentHistory = taskDetail.AssignmentHistory[startIdx:] - } - - if taskDetail.AssignmentHistory == nil { - taskDetail.AssignmentHistory = []*maintenance.TaskAssignmentRecord{} - } - - // Get worker information if task is assigned - if task.WorkerID != "" { - workers := as.maintenanceManager.GetWorkers() - for _, worker := range workers { - if worker.ID == task.WorkerID { - taskDetail.WorkerInfo = worker - break - } - } - } - - // Load execution logs from disk - if as.configPersistence != nil { - logs, err := as.configPersistence.LoadTaskExecutionLogs(taskID) - if err == nil { - taskDetail.ExecutionLogs = logs - } else { - glog.V(2).Infof("No execution logs found on disk for task %s", taskID) - } - } - - // Get related tasks (other tasks on same volume/server) - if task.VolumeID != 0 || task.Server != "" { - allTasks := as.maintenanceManager.GetTasks("", "", maxRelatedTasksDisplay) // Get recent tasks - for _, relatedTask := range allTasks { - if relatedTask.ID != taskID && - (relatedTask.VolumeID == task.VolumeID || relatedTask.Server == task.Server) { - taskDetail.RelatedTasks = append(taskDetail.RelatedTasks, relatedTask) - } - } - } - - // Save updated task detail to disk - if err := as.configPersistence.SaveTaskDetail(taskID, taskDetail); err != nil { - glog.V(1).Infof("Failed to save task detail for %s: %v", taskID, err) - } - - return taskDetail, nil -} - -// getMaintenanceWorkers returns all maintenance workers -func (as *AdminServer) getMaintenanceWorkers() ([]*maintenance.MaintenanceWorker, error) { - if as.maintenanceManager == nil { - return []*MaintenanceWorker{}, nil - } - return as.maintenanceManager.GetWorkers(), nil -} - -// getMaintenanceWorkerDetails returns detailed information about a worker -func (as *AdminServer) getMaintenanceWorkerDetails(workerID string) (*WorkerDetailsData, error) { - if as.maintenanceManager == nil { - return nil, fmt.Errorf("maintenance manager not initialized") - } - - workers := as.maintenanceManager.GetWorkers() - var targetWorker *MaintenanceWorker - for _, worker := range workers { - if worker.ID == workerID { - targetWorker = worker - break - } - } - - if targetWorker == nil { - return nil, fmt.Errorf("worker %s not found", workerID) - } - - // Get current tasks for this worker - currentTasks := as.maintenanceManager.GetTasks(TaskStatusInProgress, "", 0) - var workerCurrentTasks []*MaintenanceTask - for _, task := range currentTasks { - if task.WorkerID == workerID { - workerCurrentTasks = append(workerCurrentTasks, task) - } - } - - // Get recent tasks for this worker - recentTasks := as.maintenanceManager.GetTasks(TaskStatusCompleted, "", maxRecentTasksDisplay) - var workerRecentTasks []*MaintenanceTask - for _, task := range recentTasks { - if task.WorkerID == workerID { - workerRecentTasks = append(workerRecentTasks, task) - } - } - - // Calculate performance metrics - var totalDuration time.Duration - var completedTasks, failedTasks int - for _, task := range workerRecentTasks { - switch task.Status { - case TaskStatusCompleted: - completedTasks++ - if task.StartedAt != nil && task.CompletedAt != nil { - totalDuration += task.CompletedAt.Sub(*task.StartedAt) - } - case TaskStatusFailed: - failedTasks++ - } - } - - var averageTaskTime time.Duration - var successRate float64 - if completedTasks+failedTasks > 0 { - if completedTasks > 0 { - averageTaskTime = totalDuration / time.Duration(completedTasks) - } - successRate = float64(completedTasks) / float64(completedTasks+failedTasks) * 100 - } - - return &WorkerDetailsData{ - Worker: targetWorker, - CurrentTasks: workerCurrentTasks, - RecentTasks: workerRecentTasks, - Performance: &WorkerPerformance{ - TasksCompleted: completedTasks, - TasksFailed: failedTasks, - AverageTaskTime: averageTaskTime, - Uptime: time.Since(targetWorker.LastHeartbeat), // This should be tracked properly - SuccessRate: successRate, - }, - LastUpdated: time.Now(), - }, nil -} - -// GetWorkerLogs fetches logs from a specific worker for a task (now reads from disk) -func (as *AdminServer) GetWorkerLogs(c *gin.Context) { - workerID := c.Param("id") - taskID := c.Query("taskId") - - // Check config persistence first - if as.configPersistence == nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Config persistence not available"}) - return - } - - // Load logs strictly from disk to avoid timeouts and network dependency - // This matches the behavior of the Task Detail page - logs, err := as.configPersistence.LoadTaskExecutionLogs(taskID) - if err != nil { - glog.V(2).Infof("No execution logs found on disk for task %s: %v", taskID, err) - logs = []*maintenance.TaskExecutionLog{} - } - - // Filter logs by workerID if strictly needed, but usually task logs are what we want - // The persistent logs struct (TaskExecutionLog) matches what the frontend expects for the detail view - // ensuring consistent display. - - c.JSON(http.StatusOK, gin.H{"worker_id": workerID, "task_id": taskID, "logs": logs, "count": len(logs)}) -} - -// getMaintenanceStats returns maintenance statistics -func (as *AdminServer) getMaintenanceStats() (*MaintenanceStats, error) { - if as.maintenanceManager == nil { - return &MaintenanceStats{ - TotalTasks: 0, - TasksByStatus: make(map[MaintenanceTaskStatus]int), - TasksByType: make(map[MaintenanceTaskType]int), - ActiveWorkers: 0, - }, nil - } - return as.maintenanceManager.GetStats(), nil -} - -// getMaintenanceConfig returns maintenance configuration -func (as *AdminServer) getMaintenanceConfig() (*maintenance.MaintenanceConfigData, error) { - // Load configuration from persistent storage - config, err := as.configPersistence.LoadMaintenanceConfig() - if err != nil { - // Fallback to default configuration - config = maintenance.DefaultMaintenanceConfig() - } - - // Note: Do NOT apply schema defaults to existing config as it overrides saved values - // Only apply defaults when creating new configs or handling fallback cases - // The schema defaults should only be used in the UI for new installations - - // Get system stats from maintenance manager if available - var systemStats *MaintenanceStats - if as.maintenanceManager != nil { - systemStats = as.maintenanceManager.GetStats() - } else { - // Fallback stats - systemStats = &MaintenanceStats{ - TotalTasks: 0, - TasksByStatus: map[MaintenanceTaskStatus]int{ - TaskStatusPending: 0, - TaskStatusInProgress: 0, - TaskStatusCompleted: 0, - TaskStatusFailed: 0, - }, - TasksByType: make(map[MaintenanceTaskType]int), - ActiveWorkers: 0, - CompletedToday: 0, - FailedToday: 0, - AverageTaskTime: 0, - LastScanTime: time.Now().Add(-time.Hour), - NextScanTime: time.Now().Add(time.Duration(config.ScanIntervalSeconds) * time.Second), - } - } - - configData := &MaintenanceConfigData{ - Config: config, - IsEnabled: config.Enabled, - LastScanTime: systemStats.LastScanTime, - NextScanTime: systemStats.NextScanTime, - SystemStats: systemStats, - MenuItems: maintenance.BuildMaintenanceMenuItems(), - } - - return configData, nil -} - -// updateMaintenanceConfig updates maintenance configuration -func (as *AdminServer) updateMaintenanceConfig(config *maintenance.MaintenanceConfig) error { - // Use ConfigField validation instead of standalone validation - if err := maintenance.ValidateMaintenanceConfigWithSchema(config); err != nil { - return fmt.Errorf("configuration validation failed: %v", err) - } - - // Save configuration to persistent storage - if err := as.configPersistence.SaveMaintenanceConfig(config); err != nil { - return fmt.Errorf("failed to save maintenance configuration: %w", err) - } - - // Update maintenance manager if available - if as.maintenanceManager != nil { - if err := as.maintenanceManager.UpdateConfig(config); err != nil { - glog.Errorf("Failed to update maintenance manager config: %v", err) - // Don't return error here, just log it - } - } - - glog.V(1).Infof("Updated maintenance configuration (enabled: %v, scan interval: %ds)", - config.Enabled, config.ScanIntervalSeconds) - return nil -} - -// triggerMaintenanceScan triggers a maintenance scan -func (as *AdminServer) triggerMaintenanceScan() error { - if as.maintenanceManager == nil { - return fmt.Errorf("maintenance manager not initialized") - } - - glog.V(1).Infof("Triggering maintenance scan") - err := as.maintenanceManager.TriggerScan() - if err != nil { - glog.Errorf("Failed to trigger maintenance scan: %v", err) - return err - } - glog.V(1).Infof("Maintenance scan triggered successfully") - return nil -} - // TriggerTopicRetentionPurgeAPI triggers topic retention purge via HTTP API func (as *AdminServer) TriggerTopicRetentionPurgeAPI(c *gin.Context) { err := as.TriggerTopicRetentionPurge() @@ -1576,56 +851,6 @@ func (as *AdminServer) GetConfigInfo(c *gin.Context) { }) } -// GetMaintenanceWorkersData returns workers data for the maintenance workers page -func (as *AdminServer) GetMaintenanceWorkersData() (*MaintenanceWorkersData, error) { - workers, err := as.getMaintenanceWorkers() - if err != nil { - return nil, err - } - - // Create worker details data - workersData := make([]*WorkerDetailsData, 0, len(workers)) - activeWorkers := 0 - busyWorkers := 0 - totalLoad := 0 - - for _, worker := range workers { - details, err := as.getMaintenanceWorkerDetails(worker.ID) - if err != nil { - // Create basic worker details if we can't get full details - details = &WorkerDetailsData{ - Worker: worker, - CurrentTasks: []*MaintenanceTask{}, - RecentTasks: []*MaintenanceTask{}, - Performance: &WorkerPerformance{ - TasksCompleted: 0, - TasksFailed: 0, - AverageTaskTime: 0, - Uptime: 0, - SuccessRate: 0, - }, - LastUpdated: time.Now(), - } - } - workersData = append(workersData, details) - - if worker.Status == "active" { - activeWorkers++ - } else if worker.Status == "busy" { - busyWorkers++ - } - totalLoad += worker.CurrentLoad - } - - return &MaintenanceWorkersData{ - Workers: workersData, - ActiveWorkers: activeWorkers, - BusyWorkers: busyWorkers, - TotalLoad: totalLoad, - LastUpdated: time.Now(), - }, nil -} - // StartWorkerGrpcServer starts the worker gRPC server func (s *AdminServer) StartWorkerGrpcServer(grpcPort int) error { if s.workerGrpcServer != nil { @@ -1651,6 +876,166 @@ func (s *AdminServer) GetWorkerGrpcServer() *WorkerGrpcServer { return s.workerGrpcServer } +// GetWorkerGrpcPort returns the worker gRPC listen port, or 0 when unavailable. +func (s *AdminServer) GetWorkerGrpcPort() int { + if s.workerGrpcServer == nil { + return 0 + } + return s.workerGrpcServer.ListenPort() +} + +// GetPlugin returns the plugin instance when enabled. +func (s *AdminServer) GetPlugin() *adminplugin.Plugin { + return s.plugin +} + +// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor. +func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.RequestConfigSchema(ctx, jobType, forceRefresh) +} + +// LoadPluginJobTypeDescriptor loads persisted descriptor for one job type. +func (s *AdminServer) LoadPluginJobTypeDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.LoadDescriptor(jobType) +} + +// SavePluginJobTypeConfig persists plugin job type config in admin data dir. +func (s *AdminServer) SavePluginJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error { + if s.plugin == nil { + return fmt.Errorf("plugin is not enabled") + } + return s.plugin.SaveJobTypeConfig(config) +} + +// LoadPluginJobTypeConfig loads plugin job type config from persistence. +func (s *AdminServer) LoadPluginJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.LoadJobTypeConfig(jobType) +} + +// RunPluginDetection triggers one detection pass for a job type and returns proposed jobs. +func (s *AdminServer) RunPluginDetection( + ctx context.Context, + jobType string, + clusterContext *plugin_pb.ClusterContext, + maxResults int32, +) ([]*plugin_pb.JobProposal, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults) +} + +// FilterPluginProposalsWithActiveJobs drops proposals already represented by assigned/running jobs. +func (s *AdminServer) FilterPluginProposalsWithActiveJobs( + jobType string, + proposals []*plugin_pb.JobProposal, +) ([]*plugin_pb.JobProposal, int, error) { + if s.plugin == nil { + return nil, 0, fmt.Errorf("plugin is not enabled") + } + filtered, skipped := s.plugin.FilterProposalsWithActiveJobs(jobType, proposals) + return filtered, skipped, nil +} + +// RunPluginDetectionWithReport triggers one detection pass and returns request metadata and proposals. +func (s *AdminServer) RunPluginDetectionWithReport( + ctx context.Context, + jobType string, + clusterContext *plugin_pb.ClusterContext, + maxResults int32, +) (*adminplugin.DetectionReport, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults) +} + +// ExecutePluginJob dispatches one job to a capable worker and waits for completion. +func (s *AdminServer) ExecutePluginJob( + ctx context.Context, + job *plugin_pb.JobSpec, + clusterContext *plugin_pb.ClusterContext, + attempt int32, +) (*plugin_pb.JobCompleted, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt) +} + +// GetPluginRunHistory returns the bounded run history (last 10 success + last 10 error). +func (s *AdminServer) GetPluginRunHistory(jobType string) (*adminplugin.JobTypeRunHistory, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.LoadRunHistory(jobType) +} + +// ListPluginJobTypes returns known plugin job types from connected worker registry and persisted data. +func (s *AdminServer) ListPluginJobTypes() ([]string, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.ListKnownJobTypes() +} + +// GetPluginWorkers returns currently connected plugin workers. +func (s *AdminServer) GetPluginWorkers() []*adminplugin.WorkerSession { + if s.plugin == nil { + return nil + } + return s.plugin.ListWorkers() +} + +// ListPluginJobs returns tracked plugin jobs for monitoring. +func (s *AdminServer) ListPluginJobs(jobType, state string, limit int) []adminplugin.TrackedJob { + if s.plugin == nil { + return nil + } + return s.plugin.ListTrackedJobs(jobType, state, limit) +} + +// GetPluginJob returns one tracked plugin job by ID. +func (s *AdminServer) GetPluginJob(jobID string) (*adminplugin.TrackedJob, bool) { + if s.plugin == nil { + return nil, false + } + return s.plugin.GetTrackedJob(jobID) +} + +// GetPluginJobDetail returns detailed plugin job information with activity timeline. +func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLimit int) (*adminplugin.JobDetail, bool, error) { + if s.plugin == nil { + return nil, false, fmt.Errorf("plugin is not enabled") + } + return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit) +} + +// ListPluginActivities returns plugin job activities for monitoring. +func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity { + if s.plugin == nil { + return nil + } + return s.plugin.ListActivities(jobType, limit) +} + +// ListPluginSchedulerStates returns per-job-type scheduler state. +func (s *AdminServer) ListPluginSchedulerStates() ([]adminplugin.SchedulerJobTypeState, error) { + if s.plugin == nil { + return nil, fmt.Errorf("plugin is not enabled") + } + return s.plugin.ListSchedulerStates() +} + // Maintenance system integration methods // InitMaintenanceManager initializes the maintenance manager @@ -1852,6 +1237,10 @@ func (s *AdminServer) Shutdown() { // Stop maintenance manager s.StopMaintenanceManager() + if s.plugin != nil { + s.plugin.Shutdown() + } + // Stop worker gRPC server if err := s.StopWorkerGrpcServer(); err != nil { glog.Errorf("Failed to stop worker gRPC server: %v", err) diff --git a/weed/admin/dash/plugin_api.go b/weed/admin/dash/plugin_api.go new file mode 100644 index 000000000..c4e9aa789 --- /dev/null +++ b/weed/admin/dash/plugin_api.go @@ -0,0 +1,735 @@ +package dash + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/seaweedfs/seaweedfs/weed/admin/plugin" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultPluginDetectionTimeout = 45 * time.Second + defaultPluginExecutionTimeout = 90 * time.Second + maxPluginDetectionTimeout = 5 * time.Minute + maxPluginExecutionTimeout = 10 * time.Minute + defaultPluginRunTimeout = 5 * time.Minute + maxPluginRunTimeout = 30 * time.Minute +) + +// GetPluginStatusAPI returns plugin status. +func (s *AdminServer) GetPluginStatusAPI(c *gin.Context) { + plugin := s.GetPlugin() + if plugin == nil { + c.JSON(http.StatusOK, gin.H{ + "enabled": false, + "worker_grpc_port": s.GetWorkerGrpcPort(), + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "enabled": true, + "configured": plugin.IsConfigured(), + "base_dir": plugin.BaseDir(), + "worker_count": len(plugin.ListWorkers()), + "worker_grpc_port": s.GetWorkerGrpcPort(), + }) +} + +// GetPluginWorkersAPI returns currently connected plugin workers. +func (s *AdminServer) GetPluginWorkersAPI(c *gin.Context) { + workers := s.GetPluginWorkers() + if workers == nil { + c.JSON(http.StatusOK, []interface{}{}) + return + } + c.JSON(http.StatusOK, workers) +} + +// GetPluginJobTypesAPI returns known plugin job types from workers and persisted data. +func (s *AdminServer) GetPluginJobTypesAPI(c *gin.Context) { + jobTypes, err := s.ListPluginJobTypes() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if jobTypes == nil { + c.JSON(http.StatusOK, []interface{}{}) + return + } + c.JSON(http.StatusOK, jobTypes) +} + +// GetPluginJobsAPI returns tracked jobs for monitoring. +func (s *AdminServer) GetPluginJobsAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Query("job_type")) + state := strings.TrimSpace(c.Query("state")) + limit := parsePositiveInt(c.Query("limit"), 200) + jobs := s.ListPluginJobs(jobType, state, limit) + if jobs == nil { + c.JSON(http.StatusOK, []interface{}{}) + return + } + c.JSON(http.StatusOK, jobs) +} + +// GetPluginJobAPI returns one tracked job. +func (s *AdminServer) GetPluginJobAPI(c *gin.Context) { + jobID := strings.TrimSpace(c.Param("jobId")) + if jobID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobId is required"}) + return + } + + job, found := s.GetPluginJob(jobID) + if !found { + c.JSON(http.StatusNotFound, gin.H{"error": "job not found"}) + return + } + c.JSON(http.StatusOK, job) +} + +// GetPluginJobDetailAPI returns detailed information for one tracked plugin job. +func (s *AdminServer) GetPluginJobDetailAPI(c *gin.Context) { + jobID := strings.TrimSpace(c.Param("jobId")) + if jobID == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobId is required"}) + return + } + + activityLimit := parsePositiveInt(c.Query("activity_limit"), 500) + relatedLimit := parsePositiveInt(c.Query("related_limit"), 20) + + detail, found, err := s.GetPluginJobDetail(jobID, activityLimit, relatedLimit) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if !found || detail == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "job detail not found"}) + return + } + + c.JSON(http.StatusOK, detail) +} + +// GetPluginActivitiesAPI returns recent plugin activities. +func (s *AdminServer) GetPluginActivitiesAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Query("job_type")) + limit := parsePositiveInt(c.Query("limit"), 500) + activities := s.ListPluginActivities(jobType, limit) + if activities == nil { + c.JSON(http.StatusOK, []interface{}{}) + return + } + c.JSON(http.StatusOK, activities) +} + +// GetPluginSchedulerStatesAPI returns per-job-type scheduler status for monitoring. +func (s *AdminServer) GetPluginSchedulerStatesAPI(c *gin.Context) { + jobTypeFilter := strings.TrimSpace(c.Query("job_type")) + + states, err := s.ListPluginSchedulerStates() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + if jobTypeFilter != "" { + filtered := make([]interface{}, 0, len(states)) + for _, state := range states { + if state.JobType == jobTypeFilter { + filtered = append(filtered, state) + } + } + c.JSON(http.StatusOK, filtered) + return + } + + if states == nil { + c.JSON(http.StatusOK, []interface{}{}) + return + } + + c.JSON(http.StatusOK, states) +} + +// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema. +func (s *AdminServer) RequestPluginJobTypeSchemaAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + forceRefresh := c.DefaultQuery("force_refresh", "false") == "true" + + ctx, cancel := context.WithTimeout(c.Request.Context(), defaultPluginDetectionTimeout) + defer cancel() + descriptor, err := s.RequestPluginJobTypeDescriptor(ctx, jobType, forceRefresh) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + renderProtoJSON(c, http.StatusOK, descriptor) +} + +// GetPluginJobTypeDescriptorAPI returns persisted descriptor for a job type. +func (s *AdminServer) GetPluginJobTypeDescriptorAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + descriptor, err := s.LoadPluginJobTypeDescriptor(jobType) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if descriptor == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "descriptor not found"}) + return + } + + renderProtoJSON(c, http.StatusOK, descriptor) +} + +// GetPluginJobTypeConfigAPI loads persisted config for a job type. +func (s *AdminServer) GetPluginJobTypeConfigAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + config, err := s.LoadPluginJobTypeConfig(jobType) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if config == nil { + config = &plugin_pb.PersistedJobTypeConfig{ + JobType: jobType, + AdminConfigValues: map[string]*plugin_pb.ConfigValue{}, + WorkerConfigValues: map[string]*plugin_pb.ConfigValue{}, + AdminRuntime: &plugin_pb.AdminRuntimeConfig{}, + } + } + + renderProtoJSON(c, http.StatusOK, config) +} + +// UpdatePluginJobTypeConfigAPI stores persisted config for a job type. +func (s *AdminServer) UpdatePluginJobTypeConfigAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + config := &plugin_pb.PersistedJobTypeConfig{} + if err := parseProtoJSONBody(c, config); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + config.JobType = jobType + if config.UpdatedAt == nil { + config.UpdatedAt = timestamppb.Now() + } + if config.AdminRuntime == nil { + config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{} + } + if config.AdminConfigValues == nil { + config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{} + } + if config.WorkerConfigValues == nil { + config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{} + } + + username := c.GetString("username") + if username == "" { + username = "admin" + } + config.UpdatedBy = username + + if err := s.SavePluginJobTypeConfig(config); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + renderProtoJSON(c, http.StatusOK, config) +} + +// GetPluginRunHistoryAPI returns bounded run history for a job type. +func (s *AdminServer) GetPluginRunHistoryAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + history, err := s.GetPluginRunHistory(jobType) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if history == nil { + c.JSON(http.StatusOK, gin.H{ + "job_type": jobType, + "successful_runs": []interface{}{}, + "error_runs": []interface{}{}, + "last_updated_time": nil, + }) + return + } + + c.JSON(http.StatusOK, history) +} + +// TriggerPluginDetectionAPI runs one detector for this job type and returns proposals. +func (s *AdminServer) TriggerPluginDetectionAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + var req struct { + ClusterContext json.RawMessage `json:"cluster_context"` + MaxResults int32 `json:"max_results"` + TimeoutSeconds int `json:"timeout_seconds"` + } + + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()}) + return + } + + clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginDetectionTimeout, maxPluginDetectionTimeout) + ctx, cancel := context.WithTimeout(c.Request.Context(), timeout) + defer cancel() + + report, err := s.RunPluginDetectionWithReport(ctx, jobType, clusterContext, req.MaxResults) + proposals := make([]*plugin_pb.JobProposal, 0) + requestID := "" + detectorWorkerID := "" + totalProposals := int32(0) + if report != nil { + proposals = report.Proposals + requestID = report.RequestID + detectorWorkerID = report.WorkerID + if report.Complete != nil { + totalProposals = report.Complete.TotalProposals + } + } + + proposalPayloads := make([]map[string]interface{}, 0, len(proposals)) + for _, proposal := range proposals { + payload, marshalErr := protoMessageToMap(proposal) + if marshalErr != nil { + glog.Warningf("failed to marshal proposal for jobType=%s: %v", jobType, marshalErr) + continue + } + proposalPayloads = append(proposalPayloads, payload) + } + + sort.Slice(proposalPayloads, func(i, j int) bool { + iPriorityStr, _ := proposalPayloads[i]["priority"].(string) + jPriorityStr, _ := proposalPayloads[j]["priority"].(string) + + iPriority := plugin_pb.JobPriority_value[iPriorityStr] + jPriority := plugin_pb.JobPriority_value[jPriorityStr] + + if iPriority != jPriority { + return iPriority > jPriority + } + iID, _ := proposalPayloads[i]["proposal_id"].(string) + jID, _ := proposalPayloads[j]["proposal_id"].(string) + return iID < jID + }) + + activities := s.ListPluginActivities(jobType, 500) + filteredActivities := make([]interface{}, 0, len(activities)) + if requestID != "" { + for i := len(activities) - 1; i >= 0; i-- { + activity := activities[i] + if activity.RequestID != requestID { + continue + } + filteredActivities = append(filteredActivities, activity) + } + } + + response := gin.H{ + "job_type": jobType, + "request_id": requestID, + "detector_worker_id": detectorWorkerID, + "total_proposals": totalProposals, + "count": len(proposalPayloads), + "proposals": proposalPayloads, + "activities": filteredActivities, + } + + if err != nil { + response["error"] = err.Error() + c.JSON(http.StatusInternalServerError, response) + return + } + + c.JSON(http.StatusOK, response) +} + +// RunPluginJobTypeAPI runs full workflow for one job type: detect then dispatch detected jobs. +func (s *AdminServer) RunPluginJobTypeAPI(c *gin.Context) { + jobType := strings.TrimSpace(c.Param("jobType")) + if jobType == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "jobType is required"}) + return + } + + var req struct { + ClusterContext json.RawMessage `json:"cluster_context"` + MaxResults int32 `json:"max_results"` + TimeoutSeconds int `json:"timeout_seconds"` + Attempt int32 `json:"attempt"` + } + + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()}) + return + } + if req.Attempt < 1 { + req.Attempt = 1 + } + + clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginRunTimeout, maxPluginRunTimeout) + ctx, cancel := context.WithTimeout(c.Request.Context(), timeout) + defer cancel() + + proposals, err := s.RunPluginDetection(ctx, jobType, clusterContext, req.MaxResults) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + detectedCount := len(proposals) + + filteredProposals, skippedActiveCount, err := s.FilterPluginProposalsWithActiveJobs(jobType, proposals) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + type executionResult struct { + JobID string `json:"job_id"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Completion map[string]interface{} `json:"completion,omitempty"` + } + + results := make([]executionResult, 0, len(filteredProposals)) + successCount := 0 + errorCount := 0 + + for index, proposal := range filteredProposals { + job := buildJobSpecFromProposal(jobType, proposal, index) + completed, execErr := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt) + + result := executionResult{ + JobID: job.JobId, + Success: execErr == nil, + } + + if completed != nil { + if payload, marshalErr := protoMessageToMap(completed); marshalErr == nil { + result.Completion = payload + } + } + + if execErr != nil { + result.Error = execErr.Error() + errorCount++ + } else { + successCount++ + } + + results = append(results, result) + } + + c.JSON(http.StatusOK, gin.H{ + "job_type": jobType, + "detected_count": detectedCount, + "ready_to_execute_count": len(filteredProposals), + "skipped_active_count": skippedActiveCount, + "executed_count": len(results), + "success_count": successCount, + "error_count": errorCount, + "execution_results": results, + }) +} + +// ExecutePluginJobAPI executes one job on a capable worker and waits for completion. +func (s *AdminServer) ExecutePluginJobAPI(c *gin.Context) { + var req struct { + Job json.RawMessage `json:"job"` + ClusterContext json.RawMessage `json:"cluster_context"` + Attempt int32 `json:"attempt"` + TimeoutSeconds int `json:"timeout_seconds"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body: " + err.Error()}) + return + } + if len(req.Job) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "job is required"}) + return + } + + job := &plugin_pb.JobSpec{} + if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(req.Job, job); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid job payload: " + err.Error()}) + return + } + + clusterContext, err := s.parseOrBuildClusterContext(req.ClusterContext) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + if req.Attempt < 1 { + req.Attempt = 1 + } + + timeout := normalizeTimeout(req.TimeoutSeconds, defaultPluginExecutionTimeout, maxPluginExecutionTimeout) + ctx, cancel := context.WithTimeout(c.Request.Context(), timeout) + defer cancel() + + completed, err := s.ExecutePluginJob(ctx, job, clusterContext, req.Attempt) + if err != nil { + if completed != nil { + payload, marshalErr := protoMessageToMap(completed) + if marshalErr == nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error(), "completion": payload}) + return + } + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + renderProtoJSON(c, http.StatusOK, completed) +} + +func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_pb.ClusterContext, error) { + if len(raw) == 0 { + return s.buildDefaultPluginClusterContext(), nil + } + + contextMessage := &plugin_pb.ClusterContext{} + if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(raw, contextMessage); err != nil { + return nil, fmt.Errorf("invalid cluster_context payload: %w", err) + } + + fallback := s.buildDefaultPluginClusterContext() + if len(contextMessage.MasterGrpcAddresses) == 0 { + contextMessage.MasterGrpcAddresses = append(contextMessage.MasterGrpcAddresses, fallback.MasterGrpcAddresses...) + } + if len(contextMessage.FilerGrpcAddresses) == 0 { + contextMessage.FilerGrpcAddresses = append(contextMessage.FilerGrpcAddresses, fallback.FilerGrpcAddresses...) + } + if len(contextMessage.VolumeGrpcAddresses) == 0 { + contextMessage.VolumeGrpcAddresses = append(contextMessage.VolumeGrpcAddresses, fallback.VolumeGrpcAddresses...) + } + if contextMessage.Metadata == nil { + contextMessage.Metadata = map[string]string{} + } + contextMessage.Metadata["source"] = "admin" + + return contextMessage, nil +} + +func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterContext { + clusterContext := &plugin_pb.ClusterContext{ + MasterGrpcAddresses: make([]string, 0), + FilerGrpcAddresses: make([]string, 0), + VolumeGrpcAddresses: make([]string, 0), + Metadata: map[string]string{ + "source": "admin", + }, + } + + masterAddress := string(s.masterClient.GetMaster(context.Background())) + if masterAddress != "" { + clusterContext.MasterGrpcAddresses = append(clusterContext.MasterGrpcAddresses, masterAddress) + } + + filerSeen := map[string]struct{}{} + for _, filer := range s.GetAllFilers() { + filer = strings.TrimSpace(filer) + if filer == "" { + continue + } + if _, exists := filerSeen[filer]; exists { + continue + } + filerSeen[filer] = struct{}{} + clusterContext.FilerGrpcAddresses = append(clusterContext.FilerGrpcAddresses, filer) + } + + volumeSeen := map[string]struct{}{} + if volumeServers, err := s.GetClusterVolumeServers(); err == nil { + for _, server := range volumeServers.VolumeServers { + address := strings.TrimSpace(server.GetDisplayAddress()) + if address == "" { + address = strings.TrimSpace(server.Address) + } + if address == "" { + continue + } + if _, exists := volumeSeen[address]; exists { + continue + } + volumeSeen[address] = struct{}{} + clusterContext.VolumeGrpcAddresses = append(clusterContext.VolumeGrpcAddresses, address) + } + } else { + glog.V(1).Infof("failed to build default plugin volume context: %v", err) + } + + sort.Strings(clusterContext.MasterGrpcAddresses) + sort.Strings(clusterContext.FilerGrpcAddresses) + sort.Strings(clusterContext.VolumeGrpcAddresses) + + return clusterContext +} + +const parseProtoJSONBodyMaxBytes = 1 << 20 // 1 MB + +func parseProtoJSONBody(c *gin.Context, message proto.Message) error { + limitedBody := http.MaxBytesReader(c.Writer, c.Request.Body, parseProtoJSONBodyMaxBytes) + data, err := io.ReadAll(limitedBody) + if err != nil { + return fmt.Errorf("failed to read request body: %w", err) + } + if len(data) == 0 { + return fmt.Errorf("request body is empty") + } + if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(data, message); err != nil { + return fmt.Errorf("invalid protobuf json: %w", err) + } + return nil +} + +func renderProtoJSON(c *gin.Context, statusCode int, message proto.Message) { + payload, err := protojson.MarshalOptions{ + UseProtoNames: true, + EmitUnpopulated: true, + }.Marshal(message) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to encode response: " + err.Error()}) + return + } + + c.Data(statusCode, "application/json", payload) +} + +func protoMessageToMap(message proto.Message) (map[string]interface{}, error) { + payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(message) + if err != nil { + return nil, err + } + out := map[string]interface{}{} + if err := json.Unmarshal(payload, &out); err != nil { + return nil, err + } + return out, nil +} + +func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Duration) time.Duration { + if timeoutSeconds <= 0 { + return defaultTimeout + } + timeout := time.Duration(timeoutSeconds) * time.Second + if timeout > maxTimeout { + return maxTimeout + } + return timeout +} + +func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec { + now := timestamppb.Now() + suffix := make([]byte, 4) + if _, err := rand.Read(suffix); err != nil { + // Fallback to simpler ID if rand fails + suffix = []byte(fmt.Sprintf("%d", index)) + } + jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix)) + + jobSpec := &plugin_pb.JobSpec{ + JobId: jobID, + JobType: jobType, + Priority: plugin_pb.JobPriority_JOB_PRIORITY_NORMAL, + CreatedAt: now, + Labels: make(map[string]string), + Parameters: make(map[string]*plugin_pb.ConfigValue), + DedupeKey: "", + } + + if proposal != nil { + jobSpec.Summary = proposal.Summary + jobSpec.Detail = proposal.Detail + if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED { + jobSpec.Priority = proposal.Priority + } + jobSpec.DedupeKey = proposal.DedupeKey + jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters) + if proposal.Labels != nil { + for k, v := range proposal.Labels { + jobSpec.Labels[k] = v + } + } + } + + return jobSpec +} + +func parsePositiveInt(raw string, defaultValue int) int { + value, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || value <= 0 { + return defaultValue + } + return value +} + +// cloneConfigValueMap is now exported by the plugin package as CloneConfigValueMap diff --git a/weed/admin/dash/plugin_api_test.go b/weed/admin/dash/plugin_api_test.go new file mode 100644 index 000000000..c4f1b74e9 --- /dev/null +++ b/weed/admin/dash/plugin_api_test.go @@ -0,0 +1,33 @@ +package dash + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) { + t.Parallel() + + proposal := &plugin_pb.JobProposal{ + ProposalId: "vacuum-2", + DedupeKey: "vacuum:2", + JobType: "vacuum", + } + + jobA := buildJobSpecFromProposal("vacuum", proposal, 0) + jobB := buildJobSpecFromProposal("vacuum", proposal, 1) + + if jobA.JobId == proposal.ProposalId { + t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId) + } + if jobB.JobId == proposal.ProposalId { + t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId) + } + if jobA.JobId == jobB.JobId { + t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId) + } + if jobA.DedupeKey != proposal.DedupeKey { + t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey) + } +} diff --git a/weed/admin/dash/worker_grpc_server.go b/weed/admin/dash/worker_grpc_server.go index 015bd4215..98c41362a 100644 --- a/weed/admin/dash/worker_grpc_server.go +++ b/weed/admin/dash/worker_grpc_server.go @@ -5,12 +5,14 @@ import ( "fmt" "io" "net" + "strconv" "sync" "time" "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util" @@ -93,6 +95,10 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error { grpcServer := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.admin")) worker_pb.RegisterWorkerServiceServer(grpcServer, s) + if plugin := s.adminServer.GetPlugin(); plugin != nil { + plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin) + glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server") + } s.grpcServer = grpcServer s.listener = listener @@ -114,6 +120,25 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error { return nil } +// ListenPort returns the currently bound worker gRPC listen port. +func (s *WorkerGrpcServer) ListenPort() int { + if s == nil || s.listener == nil { + return 0 + } + if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok { + return tcpAddr.Port + } + _, portStr, err := net.SplitHostPort(s.listener.Addr().String()) + if err != nil { + return 0 + } + port, err := strconv.Atoi(portStr) + if err != nil { + return 0 + } + return port +} + // Stop stops the gRPC server func (s *WorkerGrpcServer) Stop() error { if !s.running { diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 8581d7162..70b0907b3 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -23,7 +23,7 @@ type AdminHandlers struct { fileBrowserHandlers *FileBrowserHandlers userHandlers *UserHandlers policyHandlers *PolicyHandlers - maintenanceHandlers *MaintenanceHandlers + pluginHandlers *PluginHandlers mqHandlers *MessageQueueHandlers serviceAccountHandlers *ServiceAccountHandlers } @@ -35,7 +35,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers { fileBrowserHandlers := NewFileBrowserHandlers(adminServer) userHandlers := NewUserHandlers(adminServer) policyHandlers := NewPolicyHandlers(adminServer) - maintenanceHandlers := NewMaintenanceHandlers(adminServer) + pluginHandlers := NewPluginHandlers(adminServer) mqHandlers := NewMessageQueueHandlers(adminServer) serviceAccountHandlers := NewServiceAccountHandlers(adminServer) return &AdminHandlers{ @@ -45,7 +45,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers { fileBrowserHandlers: fileBrowserHandlers, userHandlers: userHandlers, policyHandlers: policyHandlers, - maintenanceHandlers: maintenanceHandlers, + pluginHandlers: pluginHandlers, mqHandlers: mqHandlers, serviceAccountHandlers: serviceAccountHandlers, } @@ -119,14 +119,12 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, protected.GET("/mq/topics", h.mqHandlers.ShowTopics) protected.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails) - // Maintenance system routes - protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue) - protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers) - protected.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig) - protected.POST("/maintenance/config", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateMaintenanceConfig) - protected.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig) - protected.POST("/maintenance/config/:taskType", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateTaskConfig) - protected.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail) + protected.GET("/plugin", h.pluginHandlers.ShowPlugin) + protected.GET("/plugin/configuration", h.pluginHandlers.ShowPluginConfiguration) + protected.GET("/plugin/queue", h.pluginHandlers.ShowPluginQueue) + protected.GET("/plugin/detection", h.pluginHandlers.ShowPluginDetection) + protected.GET("/plugin/execution", h.pluginHandlers.ShowPluginExecution) + protected.GET("/plugin/monitoring", h.pluginHandlers.ShowPluginMonitoring) // API routes for AJAX calls api := r.Group("/api") @@ -226,20 +224,25 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, volumeApi.POST("/:id/:server/vacuum", dash.RequireWriteAccess(), h.clusterHandlers.VacuumVolume) } - // Maintenance API routes - maintenanceApi := api.Group("/maintenance") + // Plugin API routes + pluginApi := api.Group("/plugin") { - maintenanceApi.POST("/scan", dash.RequireWriteAccess(), h.adminServer.TriggerMaintenanceScan) - maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks) - maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask) - maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI) - maintenanceApi.POST("/tasks/:id/cancel", dash.RequireWriteAccess(), h.adminServer.CancelMaintenanceTask) - maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI) - maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker) - maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs) - maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats) - maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI) - maintenanceApi.PUT("/config", dash.RequireWriteAccess(), h.adminServer.UpdateMaintenanceConfigAPI) + pluginApi.GET("/status", h.adminServer.GetPluginStatusAPI) + pluginApi.GET("/workers", h.adminServer.GetPluginWorkersAPI) + pluginApi.GET("/job-types", h.adminServer.GetPluginJobTypesAPI) + pluginApi.GET("/jobs", h.adminServer.GetPluginJobsAPI) + pluginApi.GET("/jobs/:jobId", h.adminServer.GetPluginJobAPI) + pluginApi.GET("/jobs/:jobId/detail", h.adminServer.GetPluginJobDetailAPI) + pluginApi.GET("/activities", h.adminServer.GetPluginActivitiesAPI) + pluginApi.GET("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI) + pluginApi.GET("/job-types/:jobType/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI) + pluginApi.POST("/job-types/:jobType/schema", h.adminServer.RequestPluginJobTypeSchemaAPI) + pluginApi.GET("/job-types/:jobType/config", h.adminServer.GetPluginJobTypeConfigAPI) + pluginApi.PUT("/job-types/:jobType/config", dash.RequireWriteAccess(), h.adminServer.UpdatePluginJobTypeConfigAPI) + pluginApi.GET("/job-types/:jobType/runs", h.adminServer.GetPluginRunHistoryAPI) + pluginApi.POST("/job-types/:jobType/detect", dash.RequireWriteAccess(), h.adminServer.TriggerPluginDetectionAPI) + pluginApi.POST("/job-types/:jobType/run", dash.RequireWriteAccess(), h.adminServer.RunPluginJobTypeAPI) + pluginApi.POST("/jobs/execute", dash.RequireWriteAccess(), h.adminServer.ExecutePluginJobAPI) } // Message Queue API routes @@ -292,14 +295,12 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, r.GET("/mq/topics", h.mqHandlers.ShowTopics) r.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails) - // Maintenance system routes - r.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue) - r.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers) - r.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig) - r.POST("/maintenance/config", h.maintenanceHandlers.UpdateMaintenanceConfig) - r.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig) - r.POST("/maintenance/config/:taskType", h.maintenanceHandlers.UpdateTaskConfig) - r.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail) + r.GET("/plugin", h.pluginHandlers.ShowPlugin) + r.GET("/plugin/configuration", h.pluginHandlers.ShowPluginConfiguration) + r.GET("/plugin/queue", h.pluginHandlers.ShowPluginQueue) + r.GET("/plugin/detection", h.pluginHandlers.ShowPluginDetection) + r.GET("/plugin/execution", h.pluginHandlers.ShowPluginExecution) + r.GET("/plugin/monitoring", h.pluginHandlers.ShowPluginMonitoring) // API routes for AJAX calls api := r.Group("/api") @@ -398,20 +399,25 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, volumeApi.POST("/:id/:server/vacuum", h.clusterHandlers.VacuumVolume) } - // Maintenance API routes - maintenanceApi := api.Group("/maintenance") + // Plugin API routes + pluginApi := api.Group("/plugin") { - maintenanceApi.POST("/scan", h.adminServer.TriggerMaintenanceScan) - maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks) - maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask) - maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI) - maintenanceApi.POST("/tasks/:id/cancel", h.adminServer.CancelMaintenanceTask) - maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI) - maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker) - maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs) - maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats) - maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI) - maintenanceApi.PUT("/config", h.adminServer.UpdateMaintenanceConfigAPI) + pluginApi.GET("/status", h.adminServer.GetPluginStatusAPI) + pluginApi.GET("/workers", h.adminServer.GetPluginWorkersAPI) + pluginApi.GET("/job-types", h.adminServer.GetPluginJobTypesAPI) + pluginApi.GET("/jobs", h.adminServer.GetPluginJobsAPI) + pluginApi.GET("/jobs/:jobId", h.adminServer.GetPluginJobAPI) + pluginApi.GET("/jobs/:jobId/detail", h.adminServer.GetPluginJobDetailAPI) + pluginApi.GET("/activities", h.adminServer.GetPluginActivitiesAPI) + pluginApi.GET("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI) + pluginApi.GET("/job-types/:jobType/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI) + pluginApi.POST("/job-types/:jobType/schema", h.adminServer.RequestPluginJobTypeSchemaAPI) + pluginApi.GET("/job-types/:jobType/config", h.adminServer.GetPluginJobTypeConfigAPI) + pluginApi.PUT("/job-types/:jobType/config", h.adminServer.UpdatePluginJobTypeConfigAPI) + pluginApi.GET("/job-types/:jobType/runs", h.adminServer.GetPluginRunHistoryAPI) + pluginApi.POST("/job-types/:jobType/detect", h.adminServer.TriggerPluginDetectionAPI) + pluginApi.POST("/job-types/:jobType/run", h.adminServer.RunPluginJobTypeAPI) + pluginApi.POST("/jobs/execute", h.adminServer.ExecutePluginJobAPI) } // Message Queue API routes diff --git a/weed/admin/handlers/admin_handlers_routes_test.go b/weed/admin/handlers/admin_handlers_routes_test.go new file mode 100644 index 000000000..162435002 --- /dev/null +++ b/weed/admin/handlers/admin_handlers_routes_test.go @@ -0,0 +1,95 @@ +package handlers + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" +) + +func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + + newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true) + + if !hasRoute(router, "GET", "/api/plugin/scheduler-states") { + t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in no-auth mode") + } + if !hasRoute(router, "GET", "/api/plugin/jobs/:jobId/detail") { + t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode") + } +} + +func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + + newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true) + + if !hasRoute(router, "GET", "/api/plugin/scheduler-states") { + t.Fatalf("expected GET /api/plugin/scheduler-states to be registered in auth mode") + } + if !hasRoute(router, "GET", "/api/plugin/jobs/:jobId/detail") { + t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode") + } +} + +func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + + newRouteTestAdminHandlers().SetupRoutes(router, false, "", "", "", "", true) + + assertHasRoute(t, router, "GET", "/plugin") + assertHasRoute(t, router, "GET", "/plugin/configuration") + assertHasRoute(t, router, "GET", "/plugin/queue") + assertHasRoute(t, router, "GET", "/plugin/detection") + assertHasRoute(t, router, "GET", "/plugin/execution") + assertHasRoute(t, router, "GET", "/plugin/monitoring") +} + +func TestSetupRoutes_RegistersPluginPages_WithAuth(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + + newRouteTestAdminHandlers().SetupRoutes(router, true, "admin", "password", "", "", true) + + assertHasRoute(t, router, "GET", "/plugin") + assertHasRoute(t, router, "GET", "/plugin/configuration") + assertHasRoute(t, router, "GET", "/plugin/queue") + assertHasRoute(t, router, "GET", "/plugin/detection") + assertHasRoute(t, router, "GET", "/plugin/execution") + assertHasRoute(t, router, "GET", "/plugin/monitoring") +} + +func newRouteTestAdminHandlers() *AdminHandlers { + adminServer := &dash.AdminServer{} + return &AdminHandlers{ + adminServer: adminServer, + authHandlers: &AuthHandlers{adminServer: adminServer}, + clusterHandlers: &ClusterHandlers{adminServer: adminServer}, + fileBrowserHandlers: &FileBrowserHandlers{adminServer: adminServer}, + userHandlers: &UserHandlers{adminServer: adminServer}, + policyHandlers: &PolicyHandlers{adminServer: adminServer}, + pluginHandlers: &PluginHandlers{adminServer: adminServer}, + mqHandlers: &MessageQueueHandlers{adminServer: adminServer}, + serviceAccountHandlers: &ServiceAccountHandlers{adminServer: adminServer}, + } +} + +func hasRoute(router *gin.Engine, method string, path string) bool { + for _, route := range router.Routes() { + if route.Method == method && route.Path == path { + return true + } + } + return false +} + +func assertHasRoute(t *testing.T, router *gin.Engine, method string, path string) { + t.Helper() + if !hasRoute(router, method, path) { + t.Fatalf("expected %s %s to be registered", method, path) + } +} diff --git a/weed/admin/handlers/maintenance_handlers.go b/weed/admin/handlers/maintenance_handlers.go deleted file mode 100644 index 005f60277..000000000 --- a/weed/admin/handlers/maintenance_handlers.go +++ /dev/null @@ -1,550 +0,0 @@ -package handlers - -import ( - "context" - "fmt" - "net/http" - "reflect" - "strconv" - "strings" - "time" - - "github.com/gin-gonic/gin" - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/admin/dash" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/view/app" - "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" - "github.com/seaweedfs/seaweedfs/weed/worker/types" -) - -// MaintenanceHandlers handles maintenance-related HTTP requests -type MaintenanceHandlers struct { - adminServer *dash.AdminServer -} - -// NewMaintenanceHandlers creates a new instance of MaintenanceHandlers -func NewMaintenanceHandlers(adminServer *dash.AdminServer) *MaintenanceHandlers { - return &MaintenanceHandlers{ - adminServer: adminServer, - } -} - -// ShowTaskDetail displays the task detail page -func (h *MaintenanceHandlers) ShowTaskDetail(c *gin.Context) { - taskID := c.Param("id") - - if h.adminServer == nil { - c.String(http.StatusInternalServerError, "Admin server not initialized") - return - } - - taskDetail, err := h.adminServer.GetMaintenanceTaskDetail(taskID) - if err != nil { - glog.Errorf("DEBUG ShowTaskDetail: error getting task detail for %s: %v", taskID, err) - c.String(http.StatusNotFound, "Task not found: %s (Error: %v)", taskID, err) - return - } - - c.Header("Content-Type", "text/html") - taskDetailComponent := app.TaskDetail(taskDetail) - layoutComponent := layout.Layout(c, taskDetailComponent) - err = layoutComponent.Render(c.Request.Context(), c.Writer) - if err != nil { - glog.Errorf("DEBUG ShowTaskDetail: render error: %v", err) - c.String(http.StatusInternalServerError, "Failed to render template: %v", err) - return - } - -} - -// ShowMaintenanceQueue displays the maintenance queue page -func (h *MaintenanceHandlers) ShowMaintenanceQueue(c *gin.Context) { - // Add timeout to prevent hanging - ctx, cancel := context.WithTimeout(c.Request.Context(), 30*time.Second) - defer cancel() - - // Use a channel to handle timeout for data retrieval - type result struct { - data *maintenance.MaintenanceQueueData - err error - } - resultChan := make(chan result, 1) - - go func() { - data, err := h.getMaintenanceQueueData() - resultChan <- result{data: data, err: err} - }() - - select { - case res := <-resultChan: - if res.err != nil { - glog.V(1).Infof("ShowMaintenanceQueue: error getting data: %v", res.err) - c.JSON(http.StatusInternalServerError, gin.H{"error": res.err.Error()}) - return - } - - glog.V(2).Infof("ShowMaintenanceQueue: got data with %d tasks", len(res.data.Tasks)) - - // Render HTML template - c.Header("Content-Type", "text/html") - maintenanceComponent := app.MaintenanceQueue(res.data) - layoutComponent := layout.Layout(c, maintenanceComponent) - err := layoutComponent.Render(ctx, c.Writer) - if err != nil { - glog.V(1).Infof("ShowMaintenanceQueue: render error: %v", err) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()}) - return - } - - glog.V(3).Infof("ShowMaintenanceQueue: template rendered successfully") - - case <-ctx.Done(): - glog.Warningf("ShowMaintenanceQueue: timeout waiting for data") - c.JSON(http.StatusRequestTimeout, gin.H{ - "error": "Request timeout - maintenance data retrieval took too long. This may indicate a system issue.", - "suggestion": "Try refreshing the page or contact system administrator if the problem persists.", - }) - return - } -} - -// ShowMaintenanceWorkers displays the maintenance workers page -func (h *MaintenanceHandlers) ShowMaintenanceWorkers(c *gin.Context) { - if h.adminServer == nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Admin server not initialized"}) - return - } - workersData, err := h.adminServer.GetMaintenanceWorkersData() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // Render HTML template - c.Header("Content-Type", "text/html") - workersComponent := app.MaintenanceWorkers(workersData) - layoutComponent := layout.Layout(c, workersComponent) - err = layoutComponent.Render(c.Request.Context(), c.Writer) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()}) - return - } -} - -// ShowMaintenanceConfig displays the maintenance configuration page -func (h *MaintenanceHandlers) ShowMaintenanceConfig(c *gin.Context) { - config, err := h.getMaintenanceConfig() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - // Get the schema for dynamic form rendering - schema := maintenance.GetMaintenanceConfigSchema() - - // Render HTML template using schema-driven approach - c.Header("Content-Type", "text/html") - configComponent := app.MaintenanceConfigSchema(config, schema) - layoutComponent := layout.Layout(c, configComponent) - err = layoutComponent.Render(c.Request.Context(), c.Writer) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()}) - return - } -} - -// ShowTaskConfig displays the configuration page for a specific task type -func (h *MaintenanceHandlers) ShowTaskConfig(c *gin.Context) { - taskTypeName := c.Param("taskType") - - // Get the schema for this task type - schema := tasks.GetTaskConfigSchema(taskTypeName) - if schema == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Task type not found or no schema available"}) - return - } - - // Get the UI provider for current configuration - uiRegistry := tasks.GetGlobalUIRegistry() - typesRegistry := tasks.GetGlobalTypesRegistry() - - var provider types.TaskUIProvider - for workerTaskType := range typesRegistry.GetAllDetectors() { - if string(workerTaskType) == taskTypeName { - provider = uiRegistry.GetProvider(workerTaskType) - break - } - } - - if provider == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "UI provider not found for task type"}) - return - } - - // Get current configuration - currentConfig := provider.GetCurrentConfig() - - // Note: Do NOT apply schema defaults to current config as it overrides saved values - // Only apply defaults when creating new configs, not when displaying existing ones - - // Create task configuration data - configData := &maintenance.TaskConfigData{ - TaskType: maintenance.MaintenanceTaskType(taskTypeName), - TaskName: schema.DisplayName, - TaskIcon: schema.Icon, - Description: schema.Description, - } - - // Render HTML template using schema-based approach - c.Header("Content-Type", "text/html") - taskConfigComponent := app.TaskConfigSchema(configData, schema, currentConfig) - layoutComponent := layout.Layout(c, taskConfigComponent) - err := layoutComponent.Render(c.Request.Context(), c.Writer) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()}) - return - } -} - -// UpdateTaskConfig updates task configuration from form -func (h *MaintenanceHandlers) UpdateTaskConfig(c *gin.Context) { - taskTypeName := c.Param("taskType") - taskType := types.TaskType(taskTypeName) - - // Parse form data - err := c.Request.ParseForm() - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse form data: " + err.Error()}) - return - } - - // Debug logging - show received form data - glog.V(1).Infof("Received form data for task type %s:", taskTypeName) - for key, values := range c.Request.PostForm { - glog.V(1).Infof(" %s: %v", key, values) - } - - // Get the task configuration schema - schema := tasks.GetTaskConfigSchema(taskTypeName) - if schema == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "Schema not found for task type: " + taskTypeName}) - return - } - - // Create a new config instance based on task type and apply schema defaults - var config TaskConfig - switch taskType { - case types.TaskTypeVacuum: - config = &vacuum.Config{} - case types.TaskTypeBalance: - config = &balance.Config{} - case types.TaskTypeErasureCoding: - config = &erasure_coding.Config{} - default: - c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported task type: " + taskTypeName}) - return - } - - // Apply schema defaults first using type-safe method - if err := schema.ApplyDefaultsToConfig(config); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply defaults: " + err.Error()}) - return - } - - // First, get the current configuration to preserve existing values - currentUIRegistry := tasks.GetGlobalUIRegistry() - currentTypesRegistry := tasks.GetGlobalTypesRegistry() - - var currentProvider types.TaskUIProvider - for workerTaskType := range currentTypesRegistry.GetAllDetectors() { - if string(workerTaskType) == string(taskType) { - currentProvider = currentUIRegistry.GetProvider(workerTaskType) - break - } - } - - if currentProvider != nil { - // Copy current config values to the new config - currentConfig := currentProvider.GetCurrentConfig() - if currentConfigProtobuf, ok := currentConfig.(TaskConfig); ok { - // Apply current values using protobuf directly - no map conversion needed! - currentPolicy := currentConfigProtobuf.ToTaskPolicy() - if err := config.FromTaskPolicy(currentPolicy); err != nil { - glog.Warningf("Failed to load current config for %s: %v", taskTypeName, err) - } - } - } - - // Parse form data using schema-based approach (this will override with new values) - err = h.parseTaskConfigFromForm(c.Request.PostForm, schema, config) - if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": "Failed to parse configuration: " + err.Error()}) - return - } - - // Debug logging - show parsed config values - switch taskType { - case types.TaskTypeVacuum: - if vacuumConfig, ok := config.(*vacuum.Config); ok { - glog.V(1).Infof("Parsed vacuum config - GarbageThreshold: %f, MinVolumeAgeSeconds: %d, MinIntervalSeconds: %d", - vacuumConfig.GarbageThreshold, vacuumConfig.MinVolumeAgeSeconds, vacuumConfig.MinIntervalSeconds) - } - case types.TaskTypeErasureCoding: - if ecConfig, ok := config.(*erasure_coding.Config); ok { - glog.V(1).Infof("Parsed EC config - FullnessRatio: %f, QuietForSeconds: %d, MinSizeMB: %d, CollectionFilter: '%s'", - ecConfig.FullnessRatio, ecConfig.QuietForSeconds, ecConfig.MinSizeMB, ecConfig.CollectionFilter) - } - case types.TaskTypeBalance: - if balanceConfig, ok := config.(*balance.Config); ok { - glog.V(1).Infof("Parsed balance config - Enabled: %v, MaxConcurrent: %d, ScanIntervalSeconds: %d, ImbalanceThreshold: %f, MinServerCount: %d", - balanceConfig.Enabled, balanceConfig.MaxConcurrent, balanceConfig.ScanIntervalSeconds, balanceConfig.ImbalanceThreshold, balanceConfig.MinServerCount) - } - } - - // Validate the configuration - if validationErrors := schema.ValidateConfig(config); len(validationErrors) > 0 { - errorMessages := make([]string, len(validationErrors)) - for i, err := range validationErrors { - errorMessages[i] = err.Error() - } - c.JSON(http.StatusBadRequest, gin.H{"error": "Configuration validation failed", "details": errorMessages}) - return - } - - // Apply configuration using UIProvider - uiRegistry := tasks.GetGlobalUIRegistry() - typesRegistry := tasks.GetGlobalTypesRegistry() - - var provider types.TaskUIProvider - for workerTaskType := range typesRegistry.GetAllDetectors() { - if string(workerTaskType) == string(taskType) { - provider = uiRegistry.GetProvider(workerTaskType) - break - } - } - - if provider == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "UI provider not found for task type"}) - return - } - - // Apply configuration using provider - err = provider.ApplyTaskConfig(config) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply configuration: " + err.Error()}) - return - } - - // Save task configuration to protobuf file using ConfigPersistence - if h.adminServer != nil && h.adminServer.GetConfigPersistence() != nil { - err = h.saveTaskConfigToProtobuf(taskType, config) - if err != nil { - glog.Warningf("Failed to save task config to protobuf file: %v", err) - // Don't fail the request, just log the warning - } - } else if h.adminServer == nil { - glog.Warningf("Failed to save task config: admin server not initialized") - } - - // Trigger a configuration reload in the maintenance manager - if h.adminServer != nil { - if manager := h.adminServer.GetMaintenanceManager(); manager != nil { - err = manager.ReloadTaskConfigurations() - if err != nil { - glog.Warningf("Failed to reload task configurations: %v", err) - } else { - glog.V(1).Infof("Successfully reloaded task configurations after updating %s", taskTypeName) - } - } - } - - // Redirect back to task configuration page - c.Redirect(http.StatusSeeOther, "/maintenance/config/"+taskTypeName) -} - -// parseTaskConfigFromForm parses form data using schema definitions -func (h *MaintenanceHandlers) parseTaskConfigFromForm(formData map[string][]string, schema *tasks.TaskConfigSchema, config interface{}) error { - configValue := reflect.ValueOf(config) - if configValue.Kind() == reflect.Ptr { - configValue = configValue.Elem() - } - - if configValue.Kind() != reflect.Struct { - return fmt.Errorf("config must be a struct or pointer to struct") - } - - configType := configValue.Type() - - for i := 0; i < configValue.NumField(); i++ { - field := configValue.Field(i) - fieldType := configType.Field(i) - - // Handle embedded structs recursively - if fieldType.Anonymous && field.Kind() == reflect.Struct { - err := h.parseTaskConfigFromForm(formData, schema, field.Addr().Interface()) - if err != nil { - return fmt.Errorf("error parsing embedded struct %s: %w", fieldType.Name, err) - } - continue - } - - // Get JSON tag name - jsonTag := fieldType.Tag.Get("json") - if jsonTag == "" { - continue - } - - // Remove options like ",omitempty" - if commaIdx := strings.Index(jsonTag, ","); commaIdx > 0 { - jsonTag = jsonTag[:commaIdx] - } - - // Find corresponding schema field - schemaField := schema.GetFieldByName(jsonTag) - if schemaField == nil { - continue - } - - // Parse value based on field type - if err := h.parseFieldFromForm(formData, schemaField, field); err != nil { - return fmt.Errorf("error parsing field %s: %w", schemaField.DisplayName, err) - } - } - - return nil -} - -// parseFieldFromForm parses a single field value from form data -func (h *MaintenanceHandlers) parseFieldFromForm(formData map[string][]string, schemaField *config.Field, fieldValue reflect.Value) error { - if !fieldValue.CanSet() { - return nil - } - - switch schemaField.Type { - case config.FieldTypeBool: - // Checkbox fields - present means true, absent means false - _, exists := formData[schemaField.JSONName] - fieldValue.SetBool(exists) - - case config.FieldTypeInt: - if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 { - if intVal, err := strconv.Atoi(values[0]); err != nil { - return fmt.Errorf("invalid integer value: %s", values[0]) - } else { - fieldValue.SetInt(int64(intVal)) - } - } - - case config.FieldTypeFloat: - if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 { - if floatVal, err := strconv.ParseFloat(values[0], 64); err != nil { - return fmt.Errorf("invalid float value: %s", values[0]) - } else { - fieldValue.SetFloat(floatVal) - } - } - - case config.FieldTypeString: - if values, ok := formData[schemaField.JSONName]; ok && len(values) > 0 { - fieldValue.SetString(values[0]) - } - - case config.FieldTypeInterval: - // Parse interval fields with value + unit - valueKey := schemaField.JSONName + "_value" - unitKey := schemaField.JSONName + "_unit" - - if valueStrs, ok := formData[valueKey]; ok && len(valueStrs) > 0 { - value, err := strconv.Atoi(valueStrs[0]) - if err != nil { - return fmt.Errorf("invalid interval value: %s", valueStrs[0]) - } - - unit := "minutes" // default - if unitStrs, ok := formData[unitKey]; ok && len(unitStrs) > 0 { - unit = unitStrs[0] - } - - // Convert to seconds - seconds := config.IntervalValueUnitToSeconds(value, unit) - fieldValue.SetInt(int64(seconds)) - } - - default: - return fmt.Errorf("unsupported field type: %s", schemaField.Type) - } - - return nil -} - -// UpdateMaintenanceConfig updates maintenance configuration from form -func (h *MaintenanceHandlers) UpdateMaintenanceConfig(c *gin.Context) { - var config maintenance.MaintenanceConfig - if err := c.ShouldBind(&config); err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - err := h.updateMaintenanceConfig(&config) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - c.Redirect(http.StatusSeeOther, "/maintenance/config") -} - -// Helper methods that delegate to AdminServer - -func (h *MaintenanceHandlers) getMaintenanceQueueData() (*maintenance.MaintenanceQueueData, error) { - if h.adminServer == nil { - return nil, fmt.Errorf("admin server not initialized") - } - // Use the exported method from AdminServer used by the JSON API - return h.adminServer.GetMaintenanceQueueData() -} - -func (h *MaintenanceHandlers) getMaintenanceConfig() (*maintenance.MaintenanceConfigData, error) { - if h.adminServer == nil { - return nil, fmt.Errorf("admin server not initialized") - } - // Delegate to AdminServer's real persistence method - return h.adminServer.GetMaintenanceConfigData() -} - -func (h *MaintenanceHandlers) updateMaintenanceConfig(config *maintenance.MaintenanceConfig) error { - if h.adminServer == nil { - return fmt.Errorf("admin server not initialized") - } - // Delegate to AdminServer's real persistence method - return h.adminServer.UpdateMaintenanceConfigData(config) -} - -// saveTaskConfigToProtobuf saves task configuration to protobuf file -func (h *MaintenanceHandlers) saveTaskConfigToProtobuf(taskType types.TaskType, config TaskConfig) error { - configPersistence := h.adminServer.GetConfigPersistence() - if configPersistence == nil { - return fmt.Errorf("config persistence not available") - } - - // Use the new ToTaskPolicy method - much simpler and more maintainable! - taskPolicy := config.ToTaskPolicy() - - // Save using task-specific methods - switch taskType { - case types.TaskTypeVacuum: - return configPersistence.SaveVacuumTaskPolicy(taskPolicy) - case types.TaskTypeErasureCoding: - return configPersistence.SaveErasureCodingTaskPolicy(taskPolicy) - case types.TaskTypeBalance: - return configPersistence.SaveBalanceTaskPolicy(taskPolicy) - default: - return fmt.Errorf("unsupported task type for protobuf persistence: %s", taskType) - } -} diff --git a/weed/admin/handlers/maintenance_handlers_test.go b/weed/admin/handlers/maintenance_handlers_test.go deleted file mode 100644 index fa5a365f1..000000000 --- a/weed/admin/handlers/maintenance_handlers_test.go +++ /dev/null @@ -1,389 +0,0 @@ -package handlers - -import ( - "net/url" - "testing" - - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/base" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum" -) - -func TestParseTaskConfigFromForm_WithEmbeddedStruct(t *testing.T) { - // Create a maintenance handlers instance for testing - h := &MaintenanceHandlers{} - - // Test with balance config - t.Run("Balance Config", func(t *testing.T) { - // Simulate form data - formData := url.Values{ - "enabled": {"on"}, // checkbox field - "scan_interval_seconds_value": {"30"}, // interval field - "scan_interval_seconds_unit": {"minutes"}, // interval unit - "max_concurrent": {"2"}, // number field - "imbalance_threshold": {"0.15"}, // float field - "min_server_count": {"3"}, // number field - } - - // Get schema - schema := tasks.GetTaskConfigSchema("balance") - if schema == nil { - t.Fatal("Failed to get balance schema") - } - - // Create config instance - config := &balance.Config{} - - // Parse form data - err := h.parseTaskConfigFromForm(formData, schema, config) - if err != nil { - t.Fatalf("Failed to parse form data: %v", err) - } - - // Verify embedded struct fields were set correctly - if !config.Enabled { - t.Errorf("Expected Enabled=true, got %v", config.Enabled) - } - - if config.ScanIntervalSeconds != 1800 { // 30 minutes * 60 - t.Errorf("Expected ScanIntervalSeconds=1800, got %v", config.ScanIntervalSeconds) - } - - if config.MaxConcurrent != 2 { - t.Errorf("Expected MaxConcurrent=2, got %v", config.MaxConcurrent) - } - - // Verify balance-specific fields were set correctly - if config.ImbalanceThreshold != 0.15 { - t.Errorf("Expected ImbalanceThreshold=0.15, got %v", config.ImbalanceThreshold) - } - - if config.MinServerCount != 3 { - t.Errorf("Expected MinServerCount=3, got %v", config.MinServerCount) - } - }) - - // Test with vacuum config - t.Run("Vacuum Config", func(t *testing.T) { - // Simulate form data - formData := url.Values{ - // "enabled" field omitted to simulate unchecked checkbox - "scan_interval_seconds_value": {"4"}, // interval field - "scan_interval_seconds_unit": {"hours"}, // interval unit - "max_concurrent": {"3"}, // number field - "garbage_threshold": {"0.4"}, // float field - "min_volume_age_seconds_value": {"2"}, // interval field - "min_volume_age_seconds_unit": {"days"}, // interval unit - "min_interval_seconds_value": {"1"}, // interval field - "min_interval_seconds_unit": {"days"}, // interval unit - } - - // Get schema - schema := tasks.GetTaskConfigSchema("vacuum") - if schema == nil { - t.Fatal("Failed to get vacuum schema") - } - - // Create config instance - config := &vacuum.Config{} - - // Parse form data - err := h.parseTaskConfigFromForm(formData, schema, config) - if err != nil { - t.Fatalf("Failed to parse form data: %v", err) - } - - // Verify embedded struct fields were set correctly - if config.Enabled { - t.Errorf("Expected Enabled=false, got %v", config.Enabled) - } - - if config.ScanIntervalSeconds != 14400 { // 4 hours * 3600 - t.Errorf("Expected ScanIntervalSeconds=14400, got %v", config.ScanIntervalSeconds) - } - - if config.MaxConcurrent != 3 { - t.Errorf("Expected MaxConcurrent=3, got %v", config.MaxConcurrent) - } - - // Verify vacuum-specific fields were set correctly - if config.GarbageThreshold != 0.4 { - t.Errorf("Expected GarbageThreshold=0.4, got %v", config.GarbageThreshold) - } - - if config.MinVolumeAgeSeconds != 172800 { // 2 days * 86400 - t.Errorf("Expected MinVolumeAgeSeconds=172800, got %v", config.MinVolumeAgeSeconds) - } - - if config.MinIntervalSeconds != 86400 { // 1 day * 86400 - t.Errorf("Expected MinIntervalSeconds=86400, got %v", config.MinIntervalSeconds) - } - }) - - // Test with erasure coding config - t.Run("Erasure Coding Config", func(t *testing.T) { - // Simulate form data - formData := url.Values{ - "enabled": {"on"}, // checkbox field - "scan_interval_seconds_value": {"2"}, // interval field - "scan_interval_seconds_unit": {"hours"}, // interval unit - "max_concurrent": {"1"}, // number field - "quiet_for_seconds_value": {"10"}, // interval field - "quiet_for_seconds_unit": {"minutes"}, // interval unit - "fullness_ratio": {"0.85"}, // float field - "collection_filter": {"test_collection"}, // string field - "min_size_mb": {"50"}, // number field - } - - // Get schema - schema := tasks.GetTaskConfigSchema("erasure_coding") - if schema == nil { - t.Fatal("Failed to get erasure_coding schema") - } - - // Create config instance - config := &erasure_coding.Config{} - - // Parse form data - err := h.parseTaskConfigFromForm(formData, schema, config) - if err != nil { - t.Fatalf("Failed to parse form data: %v", err) - } - - // Verify embedded struct fields were set correctly - if !config.Enabled { - t.Errorf("Expected Enabled=true, got %v", config.Enabled) - } - - if config.ScanIntervalSeconds != 7200 { // 2 hours * 3600 - t.Errorf("Expected ScanIntervalSeconds=7200, got %v", config.ScanIntervalSeconds) - } - - if config.MaxConcurrent != 1 { - t.Errorf("Expected MaxConcurrent=1, got %v", config.MaxConcurrent) - } - - // Verify erasure coding-specific fields were set correctly - if config.QuietForSeconds != 600 { // 10 minutes * 60 - t.Errorf("Expected QuietForSeconds=600, got %v", config.QuietForSeconds) - } - - if config.FullnessRatio != 0.85 { - t.Errorf("Expected FullnessRatio=0.85, got %v", config.FullnessRatio) - } - - if config.CollectionFilter != "test_collection" { - t.Errorf("Expected CollectionFilter='test_collection', got %v", config.CollectionFilter) - } - - if config.MinSizeMB != 50 { - t.Errorf("Expected MinSizeMB=50, got %v", config.MinSizeMB) - } - }) -} - -func TestConfigurationValidation(t *testing.T) { - // Test that config structs can be validated and converted to protobuf format - taskTypes := []struct { - name string - config interface{} - }{ - { - "balance", - &balance.Config{ - BaseConfig: base.BaseConfig{ - Enabled: true, - ScanIntervalSeconds: 2400, - MaxConcurrent: 3, - }, - ImbalanceThreshold: 0.18, - MinServerCount: 4, - }, - }, - { - "vacuum", - &vacuum.Config{ - BaseConfig: base.BaseConfig{ - Enabled: false, - ScanIntervalSeconds: 7200, - MaxConcurrent: 2, - }, - GarbageThreshold: 0.35, - MinVolumeAgeSeconds: 86400, - MinIntervalSeconds: 604800, - }, - }, - { - "erasure_coding", - &erasure_coding.Config{ - BaseConfig: base.BaseConfig{ - Enabled: true, - ScanIntervalSeconds: 3600, - MaxConcurrent: 1, - }, - QuietForSeconds: 900, - FullnessRatio: 0.9, - CollectionFilter: "important", - MinSizeMB: 100, - }, - }, - } - - for _, test := range taskTypes { - t.Run(test.name, func(t *testing.T) { - // Test that configs can be converted to protobuf TaskPolicy - switch cfg := test.config.(type) { - case *balance.Config: - policy := cfg.ToTaskPolicy() - if policy == nil { - t.Fatal("ToTaskPolicy returned nil") - } - if policy.Enabled != cfg.Enabled { - t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled) - } - if policy.MaxConcurrent != int32(cfg.MaxConcurrent) { - t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent) - } - case *vacuum.Config: - policy := cfg.ToTaskPolicy() - if policy == nil { - t.Fatal("ToTaskPolicy returned nil") - } - if policy.Enabled != cfg.Enabled { - t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled) - } - if policy.MaxConcurrent != int32(cfg.MaxConcurrent) { - t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent) - } - case *erasure_coding.Config: - policy := cfg.ToTaskPolicy() - if policy == nil { - t.Fatal("ToTaskPolicy returned nil") - } - if policy.Enabled != cfg.Enabled { - t.Errorf("Expected Enabled=%v, got %v", cfg.Enabled, policy.Enabled) - } - if policy.MaxConcurrent != int32(cfg.MaxConcurrent) { - t.Errorf("Expected MaxConcurrent=%v, got %v", cfg.MaxConcurrent, policy.MaxConcurrent) - } - default: - t.Fatalf("Unknown config type: %T", test.config) - } - - // Test that configs can be validated - switch cfg := test.config.(type) { - case *balance.Config: - if err := cfg.Validate(); err != nil { - t.Errorf("Validation failed: %v", err) - } - case *vacuum.Config: - if err := cfg.Validate(); err != nil { - t.Errorf("Validation failed: %v", err) - } - case *erasure_coding.Config: - if err := cfg.Validate(); err != nil { - t.Errorf("Validation failed: %v", err) - } - } - }) - } -} - -func TestParseFieldFromForm_EdgeCases(t *testing.T) { - h := &MaintenanceHandlers{} - - // Test checkbox parsing (boolean fields) - t.Run("Checkbox Fields", func(t *testing.T) { - tests := []struct { - name string - formData url.Values - expectedValue bool - }{ - {"Checked checkbox", url.Values{"test_field": {"on"}}, true}, - {"Unchecked checkbox", url.Values{}, false}, - {"Empty value checkbox", url.Values{"test_field": {""}}, true}, // Present but empty means checked - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - schema := &tasks.TaskConfigSchema{ - Schema: config.Schema{ - Fields: []*config.Field{ - { - JSONName: "test_field", - Type: config.FieldTypeBool, - InputType: "checkbox", - }, - }, - }, - } - - type TestConfig struct { - TestField bool `json:"test_field"` - } - - config := &TestConfig{} - err := h.parseTaskConfigFromForm(test.formData, schema, config) - if err != nil { - t.Fatalf("parseTaskConfigFromForm failed: %v", err) - } - - if config.TestField != test.expectedValue { - t.Errorf("Expected %v, got %v", test.expectedValue, config.TestField) - } - }) - } - }) - - // Test interval parsing - t.Run("Interval Fields", func(t *testing.T) { - tests := []struct { - name string - value string - unit string - expectedSecs int - }{ - {"Minutes", "30", "minutes", 1800}, - {"Hours", "2", "hours", 7200}, - {"Days", "1", "days", 86400}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - formData := url.Values{ - "test_field_value": {test.value}, - "test_field_unit": {test.unit}, - } - - schema := &tasks.TaskConfigSchema{ - Schema: config.Schema{ - Fields: []*config.Field{ - { - JSONName: "test_field", - Type: config.FieldTypeInterval, - InputType: "interval", - }, - }, - }, - } - - type TestConfig struct { - TestField int `json:"test_field"` - } - - config := &TestConfig{} - err := h.parseTaskConfigFromForm(formData, schema, config) - if err != nil { - t.Fatalf("parseTaskConfigFromForm failed: %v", err) - } - - if config.TestField != test.expectedSecs { - t.Errorf("Expected %d seconds, got %d", test.expectedSecs, config.TestField) - } - }) - } - }) -} diff --git a/weed/admin/handlers/plugin_handlers.go b/weed/admin/handlers/plugin_handlers.go new file mode 100644 index 000000000..d43a0ec2c --- /dev/null +++ b/weed/admin/handlers/plugin_handlers.go @@ -0,0 +1,67 @@ +package handlers + +import ( + "bytes" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/admin/view/app" + "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" +) + +// PluginHandlers handles plugin UI pages. +type PluginHandlers struct { + adminServer *dash.AdminServer +} + +// NewPluginHandlers creates a new instance of PluginHandlers. +func NewPluginHandlers(adminServer *dash.AdminServer) *PluginHandlers { + return &PluginHandlers{ + adminServer: adminServer, + } +} + +// ShowPlugin displays plugin overview page. +func (h *PluginHandlers) ShowPlugin(c *gin.Context) { + h.renderPluginPage(c, "overview") +} + +// ShowPluginConfiguration displays plugin configuration page. +func (h *PluginHandlers) ShowPluginConfiguration(c *gin.Context) { + h.renderPluginPage(c, "configuration") +} + +// ShowPluginDetection displays plugin detection jobs page. +func (h *PluginHandlers) ShowPluginDetection(c *gin.Context) { + h.renderPluginPage(c, "detection") +} + +// ShowPluginQueue displays plugin job queue page. +func (h *PluginHandlers) ShowPluginQueue(c *gin.Context) { + h.renderPluginPage(c, "queue") +} + +// ShowPluginExecution displays plugin execution jobs page. +func (h *PluginHandlers) ShowPluginExecution(c *gin.Context) { + h.renderPluginPage(c, "execution") +} + +// ShowPluginMonitoring displays plugin monitoring page. +func (h *PluginHandlers) ShowPluginMonitoring(c *gin.Context) { + // Backward-compatible alias for the old monitoring URL. + h.renderPluginPage(c, "detection") +} + +func (h *PluginHandlers) renderPluginPage(c *gin.Context, page string) { + component := app.Plugin(page) + layoutComponent := layout.Layout(c, component) + + var buf bytes.Buffer + if err := layoutComponent.Render(c.Request.Context(), &buf); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()}) + return + } + + c.Data(http.StatusOK, "text/html; charset=utf-8", buf.Bytes()) +} diff --git a/weed/admin/plugin/DESIGN.md b/weed/admin/plugin/DESIGN.md new file mode 100644 index 000000000..04688cbae --- /dev/null +++ b/weed/admin/plugin/DESIGN.md @@ -0,0 +1,205 @@ +# Admin Worker Plugin System (Design) + +This document describes the plugin system for admin-managed workers, implemented in parallel with the current maintenance/worker mechanism. + +## Scope + +- Add a new plugin protocol and runtime model for multi-language workers. +- Keep all current admin + worker code paths untouched. +- Use gRPC for all admin-worker communication. +- Let workers describe job configuration UI declaratively via protobuf. +- Persist all job type configuration under admin server data directory. +- Support detector workers and executor workers per job type. +- Add end-to-end workflow observability (activities, active jobs, progress). + +## New Contract + +- Proto file: `weed/pb/plugin.proto` +- gRPC service: `PluginControlService.WorkerStream` +- Connection model: worker-initiated long-lived bidirectional stream. + +Why this model: + +- Works for workers in any language with gRPC support. +- Avoids admin dialing constraints in NAT/private networks. +- Allows command/response, progress streaming, and heartbeat over one channel. + +## Core Runtime Components (Admin Side) + +1. `PluginRegistry` +- Tracks connected workers and their per-job-type capabilities. +- Maintains liveness via heartbeat timeout. + +2. `SchemaCoordinator` +- For each job type, asks one capable worker for `JobTypeDescriptor`. +- Caches descriptor version and refresh timestamp. + +3. `ConfigStore` +- Persists descriptor + saved config values in `dataDir`. +- Stores both: + - Admin-owned runtime config (detection interval, dispatch concurrency, retry). + - Worker-owned config values (plugin-specific detection/execution knobs). + +4. `DetectorScheduler` +- Per job type, chooses one detector worker (`can_detect=true`). +- Sends `RunDetectionRequest` with saved configs + cluster context. +- Accepts `DetectionProposals`, dedupes by `dedupe_key`, inserts jobs. + +5. `JobDispatcher` +- Chooses executor worker (`can_execute=true`) for each pending job. +- Sends `ExecuteJobRequest`. +- Consumes `JobProgressUpdate` and `JobCompleted`. + +6. `WorkflowMonitor` +- Builds live counters and timeline from events: + - activities per job type, + - active jobs, + - per-job progress/state, + - worker health/load. + +## Worker Responsibilities + +1. Register capabilities on connect (`WorkerHello`). +2. Expose job type descriptor (`ConfigSchemaResponse`) including UI schemas: +- admin config form, +- worker config form, +- defaults. +3. Run detection on demand (`RunDetectionRequest`) and return proposals. +4. Execute assigned jobs (`ExecuteJobRequest`) and stream progress. +5. Heartbeat regularly with slot usage and running work. +6. Handle cancellation requests (`CancelRequest`) for in-flight detection/execution. + +## Declarative UI Model + +UI is fully derived from protobuf schema: + +- `ConfigForm` +- `ConfigSection` +- `ConfigField` +- `ConfigOption` +- `ValidationRule` +- `ConfigValue` (typed scalar/list/map/object value container) + +Result: + +- Admin can render forms without hardcoded task structs. +- New job types can ship UI schema from worker binary alone. +- Worker language is irrelevant as long as it can emit protobuf messages. + +## Detection and Dispatch Flow + +1. Worker connects and registers capabilities. +2. Admin requests descriptor per job type. +3. Admin persists descriptor and editable config values. +4. On detection interval (admin-owned setting): +- Admin chooses one detector worker for that job type. +- Sends `RunDetectionRequest` with: + - `AdminRuntimeConfig`, + - `admin_config_values`, + - `worker_config_values`, + - `ClusterContext` (master/filer/volume grpc locations, metadata). +5. Detector emits `DetectionProposals` and `DetectionComplete`. +6. Admin dedupes and enqueues jobs. +7. Dispatcher assigns jobs to any eligible executor worker. +8. Executor emits `JobProgressUpdate` and `JobCompleted`. +9. Monitor updates workflow UI in near-real-time. + +## Persistence Layout (Admin Data Dir) + +Current layout under `/plugin/`: + +- `job_types//descriptor.pb` +- `job_types//descriptor.json` +- `job_types//config.pb` +- `job_types//config.json` +- `job_types//runs.json` +- `jobs/tracked_jobs.json` +- `activities/activities.json` + +`config.pb` should use `PersistedJobTypeConfig` from `plugin.proto`. + +## Admin UI + +- Route: `/plugin` +- Includes: + - runtime status, + - workers/capabilities, + - declarative descriptor-driven config forms, + - run history (last 10 success + last 10 errors), + - tracked jobs and activity stream, + - manual actions for schema refresh, detection, and detect+execute workflow. + +## Scheduling Policy (Initial) + +Detector selection per job type: +- only workers with `can_detect=true`. +- prefer healthy worker with highest free detection slots. +- lease ends when heartbeat timeout or stream drop. + +Execution dispatch: +- only workers with `can_execute=true`. +- select by available execution slots and least active jobs. +- retry on failure using admin runtime retry config. + +## Safety and Reliability + +- Idempotency: dedupe proposals by (`job_type`, `dedupe_key`). +- Backpressure: enforce max jobs per detection run. +- Timeouts: detection and execution timeout from admin runtime config. +- Replay-safe persistence: write job state changes before emitting UI events. +- Heartbeat-based failover for detector/executor reassignment. + +## Backward Compatibility + +- Legacy `worker.proto` runtime remains internally available where still referenced. +- External CLI worker path is moved to plugin runtime behavior. +- Runtime is enabled by default on admin worker gRPC server. + +## Incremental Rollout Plan + +Phase 1 +- Introduce protocol and storage models only. + +Phase 2 +- Build admin registry/scheduler/dispatcher behind feature flag. + +Phase 3 +- Add dedicated plugin UI pages and metrics. + +Phase 4 +- Port one existing job type (e.g. vacuum) as external worker plugin. + +Phase 4 status (starter) +- Added `weed worker` command as an external `plugin.proto` worker process. +- Initial handler implements `vacuum` job type with: + - declarative descriptor/config form response (`ConfigSchemaResponse`), + - detection via master topology scan (`RunDetectionRequest`), + - execution via existing vacuum task logic (`ExecuteJobRequest`), + - heartbeat/load reporting for monitor UI. +- Legacy maintenance-worker-specific CLI path is removed. + +Run example: +- Start admin: `weed admin -master=localhost:9333` +- Start worker: `weed worker -admin=localhost:23646` +- Optional explicit job type: `weed worker -admin=localhost:23646 -jobType=vacuum` +- Optional stable worker ID persistence: `weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin` + +Phase 5 +- Migrate remaining job types and deprecate old mechanism. + +## Agreed Defaults + +1. Detector multiplicity +- Exactly one detector worker per job type at a time. Admin selects one worker and runs detection there. + +2. Secret handling +- No encryption at rest required for plugin config in this phase. + +3. Schema compatibility +- No migration policy required yet; this is a new system. + +4. Execution ownership +- Same worker is allowed to do both detection and execution. + +5. Retention +- Keep last 10 successful runs and last 10 error runs per job type. diff --git a/weed/admin/plugin/config_store.go b/weed/admin/plugin/config_store.go new file mode 100644 index 000000000..263050d84 --- /dev/null +++ b/weed/admin/plugin/config_store.go @@ -0,0 +1,739 @@ +package plugin + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +const ( + pluginDirName = "plugin" + jobTypesDirName = "job_types" + jobsDirName = "jobs" + jobDetailsDirName = "job_details" + activitiesDirName = "activities" + descriptorPBFileName = "descriptor.pb" + descriptorJSONFileName = "descriptor.json" + configPBFileName = "config.pb" + configJSONFileName = "config.json" + runsJSONFileName = "runs.json" + trackedJobsJSONFileName = "tracked_jobs.json" + activitiesJSONFileName = "activities.json" + defaultDirPerm = 0o755 + defaultFilePerm = 0o644 +) + +// validJobTypePattern is the canonical pattern for safe job type names. +// Only letters, digits, underscore, dash, and dot are allowed, which prevents +// path traversal because '/', '\\', and whitespace are rejected. +var validJobTypePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +// ConfigStore persists plugin configuration and bounded run history. +// If admin data dir is empty, it transparently falls back to in-memory mode. +type ConfigStore struct { + configured bool + baseDir string + + mu sync.RWMutex + + memDescriptors map[string]*plugin_pb.JobTypeDescriptor + memConfigs map[string]*plugin_pb.PersistedJobTypeConfig + memRunHistory map[string]*JobTypeRunHistory + memTrackedJobs []TrackedJob + memActivities []JobActivity + memJobDetails map[string]TrackedJob +} + +func NewConfigStore(adminDataDir string) (*ConfigStore, error) { + store := &ConfigStore{ + configured: adminDataDir != "", + memDescriptors: make(map[string]*plugin_pb.JobTypeDescriptor), + memConfigs: make(map[string]*plugin_pb.PersistedJobTypeConfig), + memRunHistory: make(map[string]*JobTypeRunHistory), + memJobDetails: make(map[string]TrackedJob), + } + + if adminDataDir == "" { + return store, nil + } + + store.baseDir = filepath.Join(adminDataDir, pluginDirName) + if err := os.MkdirAll(filepath.Join(store.baseDir, jobTypesDirName), defaultDirPerm); err != nil { + return nil, fmt.Errorf("create plugin job_types dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(store.baseDir, jobsDirName), defaultDirPerm); err != nil { + return nil, fmt.Errorf("create plugin jobs dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(store.baseDir, jobsDirName, jobDetailsDirName), defaultDirPerm); err != nil { + return nil, fmt.Errorf("create plugin job_details dir: %w", err) + } + if err := os.MkdirAll(filepath.Join(store.baseDir, activitiesDirName), defaultDirPerm); err != nil { + return nil, fmt.Errorf("create plugin activities dir: %w", err) + } + + return store, nil +} + +func (s *ConfigStore) IsConfigured() bool { + return s.configured +} + +func (s *ConfigStore) BaseDir() string { + return s.baseDir +} + +func (s *ConfigStore) SaveDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error { + if descriptor == nil { + return fmt.Errorf("descriptor is nil") + } + if _, err := sanitizeJobType(jobType); err != nil { + return err + } + + clone := proto.Clone(descriptor).(*plugin_pb.JobTypeDescriptor) + if clone.JobType == "" { + clone.JobType = jobType + } + + s.mu.Lock() + defer s.mu.Unlock() + + if !s.configured { + s.memDescriptors[jobType] = clone + return nil + } + + jobTypeDir, err := s.ensureJobTypeDir(jobType) + if err != nil { + return err + } + + pbPath := filepath.Join(jobTypeDir, descriptorPBFileName) + jsonPath := filepath.Join(jobTypeDir, descriptorJSONFileName) + + if err := writeProtoFiles(clone, pbPath, jsonPath); err != nil { + return fmt.Errorf("save descriptor for %s: %w", jobType, err) + } + + return nil +} + +func (s *ConfigStore) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) { + if _, err := sanitizeJobType(jobType); err != nil { + return nil, err + } + + s.mu.RLock() + if !s.configured { + d := s.memDescriptors[jobType] + s.mu.RUnlock() + if d == nil { + return nil, nil + } + return proto.Clone(d).(*plugin_pb.JobTypeDescriptor), nil + } + s.mu.RUnlock() + + pbPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, descriptorPBFileName) + data, err := os.ReadFile(pbPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read descriptor for %s: %w", jobType, err) + } + + var descriptor plugin_pb.JobTypeDescriptor + if err := proto.Unmarshal(data, &descriptor); err != nil { + return nil, fmt.Errorf("unmarshal descriptor for %s: %w", jobType, err) + } + return &descriptor, nil +} + +func (s *ConfigStore) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error { + if config == nil { + return fmt.Errorf("job type config is nil") + } + if config.JobType == "" { + return fmt.Errorf("job type config has empty job_type") + } + sanitizedJobType, err := sanitizeJobType(config.JobType) + if err != nil { + return err + } + // Use the sanitized job type going forward to ensure it is safe for filesystem paths. + config.JobType = sanitizedJobType + + clone := proto.Clone(config).(*plugin_pb.PersistedJobTypeConfig) + + s.mu.Lock() + defer s.mu.Unlock() + + if !s.configured { + s.memConfigs[config.JobType] = clone + return nil + } + + jobTypeDir, err := s.ensureJobTypeDir(config.JobType) + if err != nil { + return err + } + + pbPath := filepath.Join(jobTypeDir, configPBFileName) + jsonPath := filepath.Join(jobTypeDir, configJSONFileName) + + if err := writeProtoFiles(clone, pbPath, jsonPath); err != nil { + return fmt.Errorf("save job type config for %s: %w", config.JobType, err) + } + + return nil +} + +func (s *ConfigStore) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) { + if _, err := sanitizeJobType(jobType); err != nil { + return nil, err + } + + s.mu.RLock() + if !s.configured { + cfg := s.memConfigs[jobType] + s.mu.RUnlock() + if cfg == nil { + return nil, nil + } + return proto.Clone(cfg).(*plugin_pb.PersistedJobTypeConfig), nil + } + s.mu.RUnlock() + + pbPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, configPBFileName) + data, err := os.ReadFile(pbPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read job type config for %s: %w", jobType, err) + } + + var config plugin_pb.PersistedJobTypeConfig + if err := proto.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("unmarshal job type config for %s: %w", jobType, err) + } + + return &config, nil +} + +func (s *ConfigStore) AppendRunRecord(jobType string, record *JobRunRecord) error { + if record == nil { + return fmt.Errorf("run record is nil") + } + if _, err := sanitizeJobType(jobType); err != nil { + return err + } + + safeRecord := *record + if safeRecord.JobType == "" { + safeRecord.JobType = jobType + } + if safeRecord.CompletedAt == nil || safeRecord.CompletedAt.IsZero() { + safeRecord.CompletedAt = timeToPtr(time.Now().UTC()) + } + + s.mu.Lock() + defer s.mu.Unlock() + + history, err := s.loadRunHistoryLocked(jobType) + if err != nil { + return err + } + + if safeRecord.Outcome == RunOutcomeSuccess { + history.SuccessfulRuns = append(history.SuccessfulRuns, safeRecord) + } else { + safeRecord.Outcome = RunOutcomeError + history.ErrorRuns = append(history.ErrorRuns, safeRecord) + } + + history.SuccessfulRuns = trimRuns(history.SuccessfulRuns, MaxSuccessfulRunHistory) + history.ErrorRuns = trimRuns(history.ErrorRuns, MaxErrorRunHistory) + history.LastUpdatedTime = timeToPtr(time.Now().UTC()) + + return s.saveRunHistoryLocked(jobType, history) +} + +func (s *ConfigStore) LoadRunHistory(jobType string) (*JobTypeRunHistory, error) { + if _, err := sanitizeJobType(jobType); err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + history, err := s.loadRunHistoryLocked(jobType) + if err != nil { + return nil, err + } + return cloneRunHistory(history), nil +} + +func (s *ConfigStore) SaveTrackedJobs(jobs []TrackedJob) error { + s.mu.Lock() + defer s.mu.Unlock() + + clone := cloneTrackedJobs(jobs) + + if !s.configured { + s.memTrackedJobs = clone + return nil + } + + encoded, err := json.MarshalIndent(clone, "", " ") + if err != nil { + return fmt.Errorf("encode tracked jobs: %w", err) + } + + path := filepath.Join(s.baseDir, jobsDirName, trackedJobsJSONFileName) + if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil { + return fmt.Errorf("write tracked jobs: %w", err) + } + return nil +} + +func (s *ConfigStore) LoadTrackedJobs() ([]TrackedJob, error) { + s.mu.RLock() + if !s.configured { + out := cloneTrackedJobs(s.memTrackedJobs) + s.mu.RUnlock() + return out, nil + } + s.mu.RUnlock() + + path := filepath.Join(s.baseDir, jobsDirName, trackedJobsJSONFileName) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read tracked jobs: %w", err) + } + + var jobs []TrackedJob + if err := json.Unmarshal(data, &jobs); err != nil { + return nil, fmt.Errorf("parse tracked jobs: %w", err) + } + return cloneTrackedJobs(jobs), nil +} + +func (s *ConfigStore) SaveJobDetail(job TrackedJob) error { + jobID, err := sanitizeJobID(job.JobID) + if err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + clone := cloneTrackedJob(job) + clone.JobID = jobID + + if !s.configured { + s.memJobDetails[jobID] = clone + return nil + } + + encoded, err := json.MarshalIndent(clone, "", " ") + if err != nil { + return fmt.Errorf("encode job detail: %w", err) + } + + path := filepath.Join(s.baseDir, jobsDirName, jobDetailsDirName, jobDetailFileName(jobID)) + if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil { + return fmt.Errorf("write job detail: %w", err) + } + + return nil +} + +func (s *ConfigStore) LoadJobDetail(jobID string) (*TrackedJob, error) { + jobID, err := sanitizeJobID(jobID) + if err != nil { + return nil, err + } + + s.mu.RLock() + if !s.configured { + job, ok := s.memJobDetails[jobID] + s.mu.RUnlock() + if !ok { + return nil, nil + } + clone := cloneTrackedJob(job) + return &clone, nil + } + s.mu.RUnlock() + + path := filepath.Join(s.baseDir, jobsDirName, jobDetailsDirName, jobDetailFileName(jobID)) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read job detail: %w", err) + } + + var job TrackedJob + if err := json.Unmarshal(data, &job); err != nil { + return nil, fmt.Errorf("parse job detail: %w", err) + } + clone := cloneTrackedJob(job) + return &clone, nil +} + +func (s *ConfigStore) SaveActivities(activities []JobActivity) error { + s.mu.Lock() + defer s.mu.Unlock() + + clone := cloneActivities(activities) + + if !s.configured { + s.memActivities = clone + return nil + } + + encoded, err := json.MarshalIndent(clone, "", " ") + if err != nil { + return fmt.Errorf("encode activities: %w", err) + } + + path := filepath.Join(s.baseDir, activitiesDirName, activitiesJSONFileName) + if err := atomicWriteFile(path, encoded, defaultFilePerm); err != nil { + return fmt.Errorf("write activities: %w", err) + } + return nil +} + +func (s *ConfigStore) LoadActivities() ([]JobActivity, error) { + s.mu.RLock() + if !s.configured { + out := cloneActivities(s.memActivities) + s.mu.RUnlock() + return out, nil + } + s.mu.RUnlock() + + path := filepath.Join(s.baseDir, activitiesDirName, activitiesJSONFileName) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read activities: %w", err) + } + + var activities []JobActivity + if err := json.Unmarshal(data, &activities); err != nil { + return nil, fmt.Errorf("parse activities: %w", err) + } + return cloneActivities(activities), nil +} + +func (s *ConfigStore) ListJobTypes() ([]string, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + jobTypeSet := make(map[string]struct{}) + + if !s.configured { + for jobType := range s.memDescriptors { + jobTypeSet[jobType] = struct{}{} + } + for jobType := range s.memConfigs { + jobTypeSet[jobType] = struct{}{} + } + for jobType := range s.memRunHistory { + jobTypeSet[jobType] = struct{}{} + } + } else { + jobTypesPath := filepath.Join(s.baseDir, jobTypesDirName) + entries, err := os.ReadDir(jobTypesPath) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + return nil, fmt.Errorf("list job types: %w", err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + jobType := strings.TrimSpace(entry.Name()) + if _, err := sanitizeJobType(jobType); err != nil { + continue + } + jobTypeSet[jobType] = struct{}{} + } + } + + jobTypes := make([]string, 0, len(jobTypeSet)) + for jobType := range jobTypeSet { + jobTypes = append(jobTypes, jobType) + } + sort.Strings(jobTypes) + return jobTypes, nil +} + +func (s *ConfigStore) loadRunHistoryLocked(jobType string) (*JobTypeRunHistory, error) { + if !s.configured { + history, ok := s.memRunHistory[jobType] + if !ok { + history = &JobTypeRunHistory{JobType: jobType} + s.memRunHistory[jobType] = history + } + return cloneRunHistory(history), nil + } + + runsPath := filepath.Join(s.baseDir, jobTypesDirName, jobType, runsJSONFileName) + data, err := os.ReadFile(runsPath) + if err != nil { + if os.IsNotExist(err) { + return &JobTypeRunHistory{JobType: jobType}, nil + } + return nil, fmt.Errorf("read run history for %s: %w", jobType, err) + } + + var history JobTypeRunHistory + if err := json.Unmarshal(data, &history); err != nil { + return nil, fmt.Errorf("parse run history for %s: %w", jobType, err) + } + if history.JobType == "" { + history.JobType = jobType + } + return &history, nil +} + +func (s *ConfigStore) saveRunHistoryLocked(jobType string, history *JobTypeRunHistory) error { + if !s.configured { + s.memRunHistory[jobType] = cloneRunHistory(history) + return nil + } + + jobTypeDir, err := s.ensureJobTypeDir(jobType) + if err != nil { + return err + } + + encoded, err := json.MarshalIndent(history, "", " ") + if err != nil { + return fmt.Errorf("encode run history for %s: %w", jobType, err) + } + + runsPath := filepath.Join(jobTypeDir, runsJSONFileName) + if err := atomicWriteFile(runsPath, encoded, defaultFilePerm); err != nil { + return fmt.Errorf("write run history for %s: %w", jobType, err) + } + return nil +} + +func (s *ConfigStore) ensureJobTypeDir(jobType string) (string, error) { + if !s.configured { + return "", nil + } + jobTypeDir := filepath.Join(s.baseDir, jobTypesDirName, jobType) + if err := os.MkdirAll(jobTypeDir, defaultDirPerm); err != nil { + return "", fmt.Errorf("create job type dir for %s: %w", jobType, err) + } + return jobTypeDir, nil +} + +func sanitizeJobType(jobType string) (string, error) { + jobType = strings.TrimSpace(jobType) + if jobType == "" { + return "", fmt.Errorf("job type is empty") + } + // Enforce a strict, path-safe pattern for job types: only letters, digits, underscore, dash and dot. + // This prevents path traversal because '/', '\\' and whitespace are rejected. + if !validJobTypePattern.MatchString(jobType) { + return "", fmt.Errorf("invalid job type %q: must match %s", jobType, validJobTypePattern.String()) + } + return jobType, nil +} + +// validJobIDPattern allows letters, digits, dash, underscore, and dot. +// url.PathEscape in jobDetailFileName provides a second layer of defense. +var validJobIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +func sanitizeJobID(jobID string) (string, error) { + jobID = strings.TrimSpace(jobID) + if jobID == "" { + return "", fmt.Errorf("job id is empty") + } + if !validJobIDPattern.MatchString(jobID) { + return "", fmt.Errorf("invalid job id %q: must match %s", jobID, validJobIDPattern.String()) + } + return jobID, nil +} + +func jobDetailFileName(jobID string) string { + return url.PathEscape(jobID) + ".json" +} + +func trimRuns(runs []JobRunRecord, maxKeep int) []JobRunRecord { + if len(runs) == 0 { + return runs + } + sort.Slice(runs, func(i, j int) bool { + ti := time.Time{} + if runs[i].CompletedAt != nil { + ti = *runs[i].CompletedAt + } + tj := time.Time{} + if runs[j].CompletedAt != nil { + tj = *runs[j].CompletedAt + } + return ti.After(tj) + }) + if len(runs) > maxKeep { + runs = runs[:maxKeep] + } + return runs +} + +func cloneRunHistory(in *JobTypeRunHistory) *JobTypeRunHistory { + if in == nil { + return nil + } + out := *in + if in.SuccessfulRuns != nil { + out.SuccessfulRuns = append([]JobRunRecord(nil), in.SuccessfulRuns...) + } + if in.ErrorRuns != nil { + out.ErrorRuns = append([]JobRunRecord(nil), in.ErrorRuns...) + } + return &out +} + +func cloneTrackedJobs(in []TrackedJob) []TrackedJob { + if len(in) == 0 { + return nil + } + + out := make([]TrackedJob, len(in)) + for i := range in { + out[i] = cloneTrackedJob(in[i]) + } + return out +} + +func cloneTrackedJob(in TrackedJob) TrackedJob { + out := in + if in.Parameters != nil { + out.Parameters = make(map[string]interface{}, len(in.Parameters)) + for key, value := range in.Parameters { + out.Parameters[key] = deepCopyGenericValue(value) + } + } + if in.Labels != nil { + out.Labels = make(map[string]string, len(in.Labels)) + for key, value := range in.Labels { + out.Labels[key] = value + } + } + if in.ResultOutputValues != nil { + out.ResultOutputValues = make(map[string]interface{}, len(in.ResultOutputValues)) + for key, value := range in.ResultOutputValues { + out.ResultOutputValues[key] = deepCopyGenericValue(value) + } + } + return out +} + +func deepCopyGenericValue(val interface{}) interface{} { + switch v := val.(type) { + case map[string]interface{}: + res := make(map[string]interface{}, len(v)) + for k, val := range v { + res[k] = deepCopyGenericValue(val) + } + return res + case []interface{}: + res := make([]interface{}, len(v)) + for i, val := range v { + res[i] = deepCopyGenericValue(val) + } + return res + default: + return v + } +} + +func cloneActivities(in []JobActivity) []JobActivity { + if len(in) == 0 { + return nil + } + + out := make([]JobActivity, len(in)) + for i := range in { + out[i] = in[i] + if in[i].Details != nil { + out[i].Details = make(map[string]interface{}, len(in[i].Details)) + for key, value := range in[i].Details { + out[i].Details[key] = deepCopyGenericValue(value) + } + } + } + return out +} + +// writeProtoFiles writes message to both a binary protobuf file (pbPath) and a +// human-readable JSON file (jsonPath) using atomicWriteFile for each. +// The .pb file is the authoritative source of truth: all reads use proto.Unmarshal +// on the .pb file. The .json file is for human inspection only, so a partial +// failure where .pb succeeds but .json fails leaves the store in a consistent state. +func writeProtoFiles(message proto.Message, pbPath string, jsonPath string) error { + pbData, err := proto.Marshal(message) + if err != nil { + return fmt.Errorf("marshal protobuf: %w", err) + } + if err := atomicWriteFile(pbPath, pbData, defaultFilePerm); err != nil { + return fmt.Errorf("write protobuf file: %w", err) + } + + jsonData, err := protojson.MarshalOptions{ + Multiline: true, + Indent: " ", + EmitUnpopulated: true, + }.Marshal(message) + if err != nil { + return fmt.Errorf("marshal json: %w", err) + } + if err := atomicWriteFile(jsonPath, jsonData, defaultFilePerm); err != nil { + return fmt.Errorf("write json file: %w", err) + } + + return nil +} +func atomicWriteFile(filename string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(filename) + if err := os.MkdirAll(dir, defaultDirPerm); err != nil { + return fmt.Errorf("create directory %s: %w", dir, err) + } + tmpFile := filename + ".tmp" + if err := os.WriteFile(tmpFile, data, perm); err != nil { + return err + } + if err := os.Rename(tmpFile, filename); err != nil { + _ = os.Remove(tmpFile) + return err + } + return nil +} diff --git a/weed/admin/plugin/config_store_test.go b/weed/admin/plugin/config_store_test.go new file mode 100644 index 000000000..689ec4e0a --- /dev/null +++ b/weed/admin/plugin/config_store_test.go @@ -0,0 +1,257 @@ +package plugin + +import ( + "reflect" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestConfigStoreDescriptorRoundTrip(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + store, err := NewConfigStore(tempDir) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + descriptor := &plugin_pb.JobTypeDescriptor{ + JobType: "vacuum", + DisplayName: "Vacuum", + Description: "Vacuum volumes", + DescriptorVersion: 1, + } + + if err := store.SaveDescriptor("vacuum", descriptor); err != nil { + t.Fatalf("SaveDescriptor: %v", err) + } + + got, err := store.LoadDescriptor("vacuum") + if err != nil { + t.Fatalf("LoadDescriptor: %v", err) + } + if got == nil { + t.Fatalf("LoadDescriptor: nil descriptor") + } + if got.DisplayName != descriptor.DisplayName { + t.Fatalf("unexpected display name: got %q want %q", got.DisplayName, descriptor.DisplayName) + } + +} + +func TestConfigStoreRunHistoryRetention(t *testing.T) { + t.Parallel() + + store, err := NewConfigStore(t.TempDir()) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + base := time.Now().UTC().Add(-24 * time.Hour) + for i := 0; i < 15; i++ { + err := store.AppendRunRecord("balance", &JobRunRecord{ + RunID: "s" + time.Duration(i).String(), + JobID: "job-success", + JobType: "balance", + WorkerID: "worker-a", + Outcome: RunOutcomeSuccess, + CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)), + }) + if err != nil { + t.Fatalf("AppendRunRecord success[%d]: %v", i, err) + } + } + + for i := 0; i < 12; i++ { + err := store.AppendRunRecord("balance", &JobRunRecord{ + RunID: "e" + time.Duration(i).String(), + JobID: "job-error", + JobType: "balance", + WorkerID: "worker-b", + Outcome: RunOutcomeError, + CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)), + }) + if err != nil { + t.Fatalf("AppendRunRecord error[%d]: %v", i, err) + } + } + + history, err := store.LoadRunHistory("balance") + if err != nil { + t.Fatalf("LoadRunHistory: %v", err) + } + if len(history.SuccessfulRuns) != MaxSuccessfulRunHistory { + t.Fatalf("successful retention mismatch: got %d want %d", len(history.SuccessfulRuns), MaxSuccessfulRunHistory) + } + if len(history.ErrorRuns) != MaxErrorRunHistory { + t.Fatalf("error retention mismatch: got %d want %d", len(history.ErrorRuns), MaxErrorRunHistory) + } + + for i := 1; i < len(history.SuccessfulRuns); i++ { + t1 := time.Time{} + if history.SuccessfulRuns[i-1].CompletedAt != nil { + t1 = *history.SuccessfulRuns[i-1].CompletedAt + } + t2 := time.Time{} + if history.SuccessfulRuns[i].CompletedAt != nil { + t2 = *history.SuccessfulRuns[i].CompletedAt + } + if t1.Before(t2) { + t.Fatalf("successful run order not descending at %d", i) + } + } + for i := 1; i < len(history.ErrorRuns); i++ { + t1 := time.Time{} + if history.ErrorRuns[i-1].CompletedAt != nil { + t1 = *history.ErrorRuns[i-1].CompletedAt + } + t2 := time.Time{} + if history.ErrorRuns[i].CompletedAt != nil { + t2 = *history.ErrorRuns[i].CompletedAt + } + if t1.Before(t2) { + t.Fatalf("error run order not descending at %d", i) + } + } +} + +func TestConfigStoreListJobTypes(t *testing.T) { + t.Parallel() + + store, err := NewConfigStore("") + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + if err := store.SaveDescriptor("vacuum", &plugin_pb.JobTypeDescriptor{JobType: "vacuum"}); err != nil { + t.Fatalf("SaveDescriptor: %v", err) + } + if err := store.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{ + JobType: "balance", + AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true}, + }); err != nil { + t.Fatalf("SaveJobTypeConfig: %v", err) + } + if err := store.AppendRunRecord("ec", &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(time.Now().UTC())}); err != nil { + t.Fatalf("AppendRunRecord: %v", err) + } + + got, err := store.ListJobTypes() + if err != nil { + t.Fatalf("ListJobTypes: %v", err) + } + want := []string{"balance", "ec", "vacuum"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected job types: got=%v want=%v", got, want) + } +} + +func TestConfigStoreMonitorStateRoundTrip(t *testing.T) { + t.Parallel() + + store, err := NewConfigStore(t.TempDir()) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + tracked := []TrackedJob{ + { + JobID: "job-1", + JobType: "vacuum", + State: "running", + Progress: 55, + WorkerID: "worker-a", + CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)), + UpdatedAt: timeToPtr(time.Now().UTC().Add(-1 * time.Minute)), + }, + } + activities := []JobActivity{ + { + JobID: "job-1", + JobType: "vacuum", + Source: "worker_progress", + Message: "processing", + Stage: "running", + OccurredAt: timeToPtr(time.Now().UTC()), + Details: map[string]interface{}{ + "step": "scan", + }, + }, + } + + if err := store.SaveTrackedJobs(tracked); err != nil { + t.Fatalf("SaveTrackedJobs: %v", err) + } + if err := store.SaveActivities(activities); err != nil { + t.Fatalf("SaveActivities: %v", err) + } + + gotTracked, err := store.LoadTrackedJobs() + if err != nil { + t.Fatalf("LoadTrackedJobs: %v", err) + } + if len(gotTracked) != 1 || gotTracked[0].JobID != tracked[0].JobID { + t.Fatalf("unexpected tracked jobs: %+v", gotTracked) + } + + gotActivities, err := store.LoadActivities() + if err != nil { + t.Fatalf("LoadActivities: %v", err) + } + if len(gotActivities) != 1 || gotActivities[0].Message != activities[0].Message { + t.Fatalf("unexpected activities: %+v", gotActivities) + } + if gotActivities[0].Details["step"] != "scan" { + t.Fatalf("unexpected activity details: %+v", gotActivities[0].Details) + } +} + +func TestConfigStoreJobDetailRoundTrip(t *testing.T) { + t.Parallel() + + store, err := NewConfigStore(t.TempDir()) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + input := TrackedJob{ + JobID: "job-detail-1", + JobType: "vacuum", + Summary: "detail summary", + Detail: "detail payload", + CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)), + UpdatedAt: timeToPtr(time.Now().UTC()), + Parameters: map[string]interface{}{ + "volume_id": map[string]interface{}{"int64_value": "3"}, + }, + Labels: map[string]string{ + "source": "detector", + }, + ResultOutputValues: map[string]interface{}{ + "moved": map[string]interface{}{"bool_value": true}, + }, + } + + if err := store.SaveJobDetail(input); err != nil { + t.Fatalf("SaveJobDetail: %v", err) + } + + got, err := store.LoadJobDetail(input.JobID) + if err != nil { + t.Fatalf("LoadJobDetail: %v", err) + } + if got == nil { + t.Fatalf("LoadJobDetail returned nil") + } + if got.Detail != input.Detail { + t.Fatalf("unexpected detail: got=%q want=%q", got.Detail, input.Detail) + } + if got.Labels["source"] != "detector" { + t.Fatalf("unexpected labels: %+v", got.Labels) + } + if got.ResultOutputValues == nil { + t.Fatalf("expected result output values") + } +} diff --git a/weed/admin/plugin/job_execution_plan.go b/weed/admin/plugin/job_execution_plan.go new file mode 100644 index 000000000..c1503e9cb --- /dev/null +++ b/weed/admin/plugin/job_execution_plan.go @@ -0,0 +1,231 @@ +package plugin + +import ( + "encoding/base64" + "sort" + "strconv" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "google.golang.org/protobuf/proto" +) + +func enrichTrackedJobParameters(jobType string, parameters map[string]interface{}) map[string]interface{} { + if len(parameters) == 0 { + return parameters + } + if _, exists := parameters["execution_plan"]; exists { + return parameters + } + + taskParams, ok := decodeTaskParamsFromPlainParameters(parameters) + if !ok || taskParams == nil { + return parameters + } + + plan := buildExecutionPlan(strings.TrimSpace(jobType), taskParams) + if plan == nil { + return parameters + } + + enriched := make(map[string]interface{}, len(parameters)+1) + for key, value := range parameters { + enriched[key] = value + } + enriched["execution_plan"] = plan + return enriched +} + +func decodeTaskParamsFromPlainParameters(parameters map[string]interface{}) (*worker_pb.TaskParams, bool) { + rawField, ok := parameters["task_params_pb"] + if !ok || rawField == nil { + return nil, false + } + + fieldMap, ok := rawField.(map[string]interface{}) + if !ok { + return nil, false + } + + bytesValue, _ := fieldMap["bytes_value"].(string) + bytesValue = strings.TrimSpace(bytesValue) + if bytesValue == "" { + return nil, false + } + + payload, err := base64.StdEncoding.DecodeString(bytesValue) + if err != nil { + return nil, false + } + + params := &worker_pb.TaskParams{} + if err := proto.Unmarshal(payload, params); err != nil { + return nil, false + } + + return params, true +} + +func buildExecutionPlan(jobType string, params *worker_pb.TaskParams) map[string]interface{} { + if params == nil { + return nil + } + + normalizedJobType := strings.TrimSpace(jobType) + if normalizedJobType == "" && params.GetErasureCodingParams() != nil { + normalizedJobType = "erasure_coding" + } + + switch normalizedJobType { + case "erasure_coding": + return buildErasureCodingExecutionPlan(params) + default: + return nil + } +} + +func buildErasureCodingExecutionPlan(params *worker_pb.TaskParams) map[string]interface{} { + if params == nil { + return nil + } + + ecParams := params.GetErasureCodingParams() + if ecParams == nil { + return nil + } + + dataShards := int(ecParams.DataShards) + if dataShards <= 0 { + dataShards = int(erasure_coding.DataShardsCount) + } + parityShards := int(ecParams.ParityShards) + if parityShards <= 0 { + parityShards = int(erasure_coding.ParityShardsCount) + } + totalShards := dataShards + parityShards + + sources := make([]map[string]interface{}, 0, len(params.Sources)) + for _, source := range params.Sources { + if source == nil { + continue + } + sources = append(sources, buildExecutionEndpoint( + source.Node, + source.DataCenter, + source.Rack, + source.VolumeId, + source.ShardIds, + dataShards, + )) + } + + targets := make([]map[string]interface{}, 0, len(params.Targets)) + shardAssignments := make([]map[string]interface{}, 0, totalShards) + for targetIndex, target := range params.Targets { + if target == nil { + continue + } + + targets = append(targets, buildExecutionEndpoint( + target.Node, + target.DataCenter, + target.Rack, + target.VolumeId, + target.ShardIds, + dataShards, + )) + + for _, shardID := range normalizeShardIDs(target.ShardIds) { + kind, label := classifyShardID(shardID, dataShards) + shardAssignments = append(shardAssignments, map[string]interface{}{ + "shard_id": shardID, + "kind": kind, + "label": label, + "target_index": targetIndex + 1, + "target_node": strings.TrimSpace(target.Node), + "target_data_center": strings.TrimSpace(target.DataCenter), + "target_rack": strings.TrimSpace(target.Rack), + "target_volume_id": int(target.VolumeId), + }) + } + } + sort.Slice(shardAssignments, func(i, j int) bool { + left, _ := shardAssignments[i]["shard_id"].(int) + right, _ := shardAssignments[j]["shard_id"].(int) + return left < right + }) + + plan := map[string]interface{}{ + "job_type": "erasure_coding", + "task_id": strings.TrimSpace(params.TaskId), + "volume_id": int(params.VolumeId), + "collection": strings.TrimSpace(params.Collection), + "data_shards": dataShards, + "parity_shards": parityShards, + "total_shards": totalShards, + "sources": sources, + "targets": targets, + "source_count": len(sources), + "target_count": len(targets), + } + + if len(shardAssignments) > 0 { + plan["shard_assignments"] = shardAssignments + } + + return plan +} + +func buildExecutionEndpoint( + node string, + dataCenter string, + rack string, + volumeID uint32, + shardIDs []uint32, + dataShardCount int, +) map[string]interface{} { + allShards := normalizeShardIDs(shardIDs) + dataShards := make([]int, 0, len(allShards)) + parityShards := make([]int, 0, len(allShards)) + for _, shardID := range allShards { + if shardID < dataShardCount { + dataShards = append(dataShards, shardID) + } else { + parityShards = append(parityShards, shardID) + } + } + + return map[string]interface{}{ + "node": strings.TrimSpace(node), + "data_center": strings.TrimSpace(dataCenter), + "rack": strings.TrimSpace(rack), + "volume_id": int(volumeID), + "shard_ids": allShards, + "data_shard_ids": dataShards, + "parity_shard_ids": parityShards, + } +} + +func normalizeShardIDs(shardIDs []uint32) []int { + if len(shardIDs) == 0 { + return nil + } + + out := make([]int, 0, len(shardIDs)) + for _, shardID := range shardIDs { + out = append(out, int(shardID)) + } + sort.Ints(out) + return out +} + +func classifyShardID(shardID int, dataShardCount int) (kind string, label string) { + if dataShardCount <= 0 { + dataShardCount = int(erasure_coding.DataShardsCount) + } + if shardID < dataShardCount { + return "data", "D" + strconv.Itoa(shardID) + } + return "parity", "P" + strconv.Itoa(shardID) +} diff --git a/weed/admin/plugin/plugin.go b/weed/admin/plugin/plugin.go new file mode 100644 index 000000000..6fa0eab35 --- /dev/null +++ b/weed/admin/plugin/plugin.go @@ -0,0 +1,1243 @@ +package plugin + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultOutgoingBuffer = 128 + defaultSendTimeout = 5 * time.Second + defaultHeartbeatInterval = 30 + defaultReconnectDelay = 5 + defaultPendingSchemaBuffer = 1 +) + +type Options struct { + DataDir string + OutgoingBufferSize int + SendTimeout time.Duration + SchedulerTick time.Duration + ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error) +} + +type Plugin struct { + plugin_pb.UnimplementedPluginControlServiceServer + + store *ConfigStore + registry *Registry + + outgoingBuffer int + sendTimeout time.Duration + + schedulerTick time.Duration + clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error) + + schedulerMu sync.Mutex + nextDetectionAt map[string]time.Time + detectionInFlight map[string]bool + + detectorLeaseMu sync.Mutex + detectorLeases map[string]string + + schedulerExecMu sync.Mutex + schedulerExecReservations map[string]int + + dedupeMu sync.Mutex + recentDedupeByType map[string]map[string]time.Time + + sessionsMu sync.RWMutex + sessions map[string]*streamSession + + pendingSchemaMu sync.Mutex + pendingSchema map[string]chan *plugin_pb.ConfigSchemaResponse + + pendingDetectionMu sync.Mutex + pendingDetection map[string]*pendingDetectionState + + pendingExecutionMu sync.Mutex + pendingExecution map[string]chan *plugin_pb.JobCompleted + + jobsMu sync.RWMutex + jobs map[string]*TrackedJob + + jobDetailsMu sync.Mutex + + activitiesMu sync.RWMutex + activities []JobActivity + + dirtyJobs bool + dirtyActivities bool + persistTicker *time.Ticker + + ctx context.Context + ctxCancel context.CancelFunc + + shutdownCh chan struct{} + shutdownOnce sync.Once + wg sync.WaitGroup +} + +type streamSession struct { + workerID string + outgoing chan *plugin_pb.AdminToWorkerMessage + closeOnce sync.Once +} + +type pendingDetectionState struct { + proposals []*plugin_pb.JobProposal + complete chan *plugin_pb.DetectionComplete + jobType string + workerID string +} + +// DetectionReport captures one detection run including request metadata. +type DetectionReport struct { + RequestID string + JobType string + WorkerID string + Proposals []*plugin_pb.JobProposal + Complete *plugin_pb.DetectionComplete +} + +func New(options Options) (*Plugin, error) { + store, err := NewConfigStore(options.DataDir) + if err != nil { + return nil, err + } + + bufferSize := options.OutgoingBufferSize + if bufferSize <= 0 { + bufferSize = defaultOutgoingBuffer + } + sendTimeout := options.SendTimeout + if sendTimeout <= 0 { + sendTimeout = defaultSendTimeout + } + schedulerTick := options.SchedulerTick + if schedulerTick <= 0 { + schedulerTick = defaultSchedulerTick + } + + plugin := &Plugin{ + store: store, + registry: NewRegistry(), + outgoingBuffer: bufferSize, + sendTimeout: sendTimeout, + schedulerTick: schedulerTick, + clusterContextProvider: options.ClusterContextProvider, + sessions: make(map[string]*streamSession), + pendingSchema: make(map[string]chan *plugin_pb.ConfigSchemaResponse), + pendingDetection: make(map[string]*pendingDetectionState), + pendingExecution: make(map[string]chan *plugin_pb.JobCompleted), + nextDetectionAt: make(map[string]time.Time), + detectionInFlight: make(map[string]bool), + detectorLeases: make(map[string]string), + schedulerExecReservations: make(map[string]int), + recentDedupeByType: make(map[string]map[string]time.Time), + jobs: make(map[string]*TrackedJob), + activities: make([]JobActivity, 0, 256), + persistTicker: time.NewTicker(2 * time.Second), + shutdownCh: make(chan struct{}), + } + plugin.ctx, plugin.ctxCancel = context.WithCancel(context.Background()) + + if err := plugin.loadPersistedMonitorState(); err != nil { + glog.Warningf("Plugin failed to load persisted monitoring state: %v", err) + } + + if plugin.clusterContextProvider != nil { + plugin.wg.Add(1) + go plugin.schedulerLoop() + } + plugin.wg.Add(1) + go plugin.persistenceLoop() + + return plugin, nil +} + +func (r *Plugin) Shutdown() { + if r.ctxCancel != nil { + r.ctxCancel() + } + if r.persistTicker != nil { + r.persistTicker.Stop() + } + + r.shutdownOnce.Do(func() { close(r.shutdownCh) }) + + r.sessionsMu.Lock() + for workerID, session := range r.sessions { + session.close() + delete(r.sessions, workerID) + } + r.sessionsMu.Unlock() + + r.pendingSchemaMu.Lock() + for requestID, ch := range r.pendingSchema { + close(ch) + delete(r.pendingSchema, requestID) + } + r.pendingSchemaMu.Unlock() + + r.pendingDetectionMu.Lock() + for requestID, state := range r.pendingDetection { + close(state.complete) + delete(r.pendingDetection, requestID) + } + r.pendingDetectionMu.Unlock() + + r.pendingExecutionMu.Lock() + for requestID, ch := range r.pendingExecution { + close(ch) + delete(r.pendingExecution, requestID) + } + r.pendingExecutionMu.Unlock() + + r.wg.Wait() +} + +func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStreamServer) error { + first, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("receive worker hello: %w", err) + } + + hello := first.GetHello() + if hello == nil { + return fmt.Errorf("first message must be hello") + } + if strings.TrimSpace(hello.WorkerId) == "" { + return fmt.Errorf("worker_id is required") + } + + workerID := hello.WorkerId + r.registry.UpsertFromHello(hello) + + session := &streamSession{ + workerID: workerID, + outgoing: make(chan *plugin_pb.AdminToWorkerMessage, r.outgoingBuffer), + } + r.putSession(session) + defer r.cleanupSession(workerID) + + glog.V(0).Infof("Plugin worker connected: %s (%s)", workerID, hello.Address) + + sendErrCh := make(chan error, 1) + r.wg.Add(1) + go func() { + defer r.wg.Done() + sendErrCh <- r.sendLoop(stream.Context(), stream, session) + }() + + if err := r.sendAdminHello(workerID); err != nil { + glog.Warningf("failed to send plugin admin hello to %s: %v", workerID, err) + } + go r.prefetchDescriptorsFromHello(hello) + + recvErrCh := make(chan error, 1) + r.wg.Add(1) + go func() { + defer r.wg.Done() + for { + message, recvErr := stream.Recv() + if recvErr != nil { + recvErrCh <- recvErr + return + } + r.handleWorkerMessage(workerID, message) + } + }() + + for { + select { + case <-r.shutdownCh: + return nil + case err := <-sendErrCh: + if err != nil && !errors.Is(err, context.Canceled) { + return err + } + return nil + case recvErr := <-recvErrCh: + if errors.Is(recvErr, io.EOF) || errors.Is(recvErr, context.Canceled) { + return nil + } + return fmt.Errorf("receive plugin message from %s: %w", workerID, recvErr) + } + } +} + +func (r *Plugin) RequestConfigSchema(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) { + if !forceRefresh { + descriptor, err := r.store.LoadDescriptor(jobType) + if err != nil { + return nil, err + } + if descriptor != nil { + return descriptor, nil + } + } + + provider, err := r.registry.PickSchemaProvider(jobType) + if err != nil { + return nil, err + } + + requestID, err := newRequestID("schema") + if err != nil { + return nil, err + } + + responseCh := make(chan *plugin_pb.ConfigSchemaResponse, defaultPendingSchemaBuffer) + r.pendingSchemaMu.Lock() + r.pendingSchema[requestID] = responseCh + r.pendingSchemaMu.Unlock() + defer func() { + r.pendingSchemaMu.Lock() + delete(r.pendingSchema, requestID) + r.pendingSchemaMu.Unlock() + }() + + requestMessage := &plugin_pb.AdminToWorkerMessage{ + RequestId: requestID, + SentAt: timestamppb.Now(), + Body: &plugin_pb.AdminToWorkerMessage_RequestConfigSchema{ + RequestConfigSchema: &plugin_pb.RequestConfigSchema{ + JobType: jobType, + ForceRefresh: forceRefresh, + }, + }, + } + + if err := r.sendToWorker(provider.WorkerID, requestMessage); err != nil { + return nil, err + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case response, ok := <-responseCh: + if !ok { + return nil, fmt.Errorf("schema request %s interrupted", requestID) + } + if response == nil { + return nil, fmt.Errorf("schema request %s returned empty response", requestID) + } + if !response.Success { + return nil, fmt.Errorf("schema request failed for %s: %s", jobType, response.ErrorMessage) + } + if response.GetJobTypeDescriptor() == nil { + return nil, fmt.Errorf("schema request for %s returned no descriptor", jobType) + } + return response.GetJobTypeDescriptor(), nil + } +} + +func (r *Plugin) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) { + return r.store.LoadJobTypeConfig(jobType) +} + +func (r *Plugin) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error { + return r.store.SaveJobTypeConfig(config) +} + +func (r *Plugin) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) { + return r.store.LoadDescriptor(jobType) +} + +func (r *Plugin) LoadRunHistory(jobType string) (*JobTypeRunHistory, error) { + return r.store.LoadRunHistory(jobType) +} + +func (r *Plugin) IsConfigured() bool { + return r.store.IsConfigured() +} + +func (r *Plugin) BaseDir() string { + return r.store.BaseDir() +} + +// RunDetectionWithReport requests one detector worker and returns proposals with request metadata. +func (r *Plugin) RunDetectionWithReport( + ctx context.Context, + jobType string, + clusterContext *plugin_pb.ClusterContext, + maxResults int32, +) (*DetectionReport, error) { + detector, err := r.pickDetector(jobType) + if err != nil { + return nil, err + } + + requestID, err := newRequestID("detect") + if err != nil { + return nil, err + } + + adminRuntime, adminConfigValues, workerConfigValues, err := r.loadJobTypeConfigPayload(jobType) + if err != nil { + return nil, err + } + lastSuccessfulRun := r.loadLastSuccessfulRun(jobType) + + state := &pendingDetectionState{ + complete: make(chan *plugin_pb.DetectionComplete, 1), + jobType: jobType, + workerID: detector.WorkerID, + } + r.pendingDetectionMu.Lock() + r.pendingDetection[requestID] = state + r.pendingDetectionMu.Unlock() + defer func() { + r.pendingDetectionMu.Lock() + delete(r.pendingDetection, requestID) + r.pendingDetectionMu.Unlock() + }() + + r.appendActivity(JobActivity{ + JobType: jobType, + RequestID: requestID, + WorkerID: detector.WorkerID, + Source: "detector", + Stage: "requested", + Message: "detection requested", + OccurredAt: timeToPtr(time.Now().UTC()), + Details: map[string]interface{}{ + "max_results": maxResults, + }, + }) + + message := &plugin_pb.AdminToWorkerMessage{ + RequestId: requestID, + SentAt: timestamppb.Now(), + Body: &plugin_pb.AdminToWorkerMessage_RunDetectionRequest{ + RunDetectionRequest: &plugin_pb.RunDetectionRequest{ + RequestId: requestID, + JobType: jobType, + DetectionSequence: time.Now().UnixNano(), + AdminRuntime: adminRuntime, + AdminConfigValues: adminConfigValues, + WorkerConfigValues: workerConfigValues, + ClusterContext: clusterContext, + LastSuccessfulRun: lastSuccessfulRun, + MaxResults: maxResults, + }, + }, + } + + if err := r.sendToWorker(detector.WorkerID, message); err != nil { + r.clearDetectorLease(jobType, detector.WorkerID) + r.appendActivity(JobActivity{ + JobType: jobType, + RequestID: requestID, + WorkerID: detector.WorkerID, + Source: "detector", + Stage: "failed_to_send", + Message: err.Error(), + OccurredAt: timeToPtr(time.Now().UTC()), + }) + return nil, err + } + + select { + case <-ctx.Done(): + r.sendCancel(detector.WorkerID, requestID, plugin_pb.WorkKind_WORK_KIND_DETECTION, ctx.Err()) + r.appendActivity(JobActivity{ + JobType: jobType, + RequestID: requestID, + WorkerID: detector.WorkerID, + Source: "detector", + Stage: "canceled", + Message: "detection canceled", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + return &DetectionReport{ + RequestID: requestID, + JobType: jobType, + WorkerID: detector.WorkerID, + }, ctx.Err() + case complete, ok := <-state.complete: + if !ok { + return &DetectionReport{ + RequestID: requestID, + JobType: jobType, + WorkerID: detector.WorkerID, + }, fmt.Errorf("detection request %s interrupted", requestID) + } + proposals := cloneJobProposals(state.proposals) + report := &DetectionReport{ + RequestID: requestID, + JobType: jobType, + WorkerID: detector.WorkerID, + Proposals: proposals, + Complete: complete, + } + if complete == nil { + return report, fmt.Errorf("detection request %s returned no completion state", requestID) + } + if !complete.Success { + return report, fmt.Errorf("detection failed for %s: %s", jobType, complete.ErrorMessage) + } + return report, nil + } +} + +// RunDetection requests one detector worker to produce job proposals for a job type. +func (r *Plugin) RunDetection( + ctx context.Context, + jobType string, + clusterContext *plugin_pb.ClusterContext, + maxResults int32, +) ([]*plugin_pb.JobProposal, error) { + report, err := r.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults) + if report == nil { + return nil, err + } + return report.Proposals, err +} + +// ExecuteJob sends one job to a capable executor worker and waits for completion. +func (r *Plugin) ExecuteJob( + ctx context.Context, + job *plugin_pb.JobSpec, + clusterContext *plugin_pb.ClusterContext, + attempt int32, +) (*plugin_pb.JobCompleted, error) { + if job == nil { + return nil, fmt.Errorf("job is nil") + } + if strings.TrimSpace(job.JobType) == "" { + return nil, fmt.Errorf("job_type is required") + } + + executor, err := r.registry.PickExecutor(job.JobType) + if err != nil { + return nil, err + } + + return r.executeJobWithExecutor(ctx, executor, job, clusterContext, attempt) +} + +func (r *Plugin) executeJobWithExecutor( + ctx context.Context, + executor *WorkerSession, + job *plugin_pb.JobSpec, + clusterContext *plugin_pb.ClusterContext, + attempt int32, +) (*plugin_pb.JobCompleted, error) { + if executor == nil { + return nil, fmt.Errorf("executor is nil") + } + if job == nil { + return nil, fmt.Errorf("job is nil") + } + if strings.TrimSpace(job.JobType) == "" { + return nil, fmt.Errorf("job_type is required") + } + + if strings.TrimSpace(job.JobId) == "" { + var err error + job.JobId, err = newRequestID("job") + if err != nil { + return nil, err + } + } + + requestID, err := newRequestID("exec") + if err != nil { + return nil, err + } + + adminRuntime, adminConfigValues, workerConfigValues, err := r.loadJobTypeConfigPayload(job.JobType) + if err != nil { + return nil, err + } + + completedCh := make(chan *plugin_pb.JobCompleted, 1) + r.pendingExecutionMu.Lock() + r.pendingExecution[requestID] = completedCh + r.pendingExecutionMu.Unlock() + defer func() { + r.pendingExecutionMu.Lock() + delete(r.pendingExecution, requestID) + r.pendingExecutionMu.Unlock() + }() + + r.trackExecutionStart(requestID, executor.WorkerID, job, attempt) + + message := &plugin_pb.AdminToWorkerMessage{ + RequestId: requestID, + SentAt: timestamppb.Now(), + Body: &plugin_pb.AdminToWorkerMessage_ExecuteJobRequest{ + ExecuteJobRequest: &plugin_pb.ExecuteJobRequest{ + RequestId: requestID, + Job: job, + AdminRuntime: adminRuntime, + AdminConfigValues: adminConfigValues, + WorkerConfigValues: workerConfigValues, + ClusterContext: clusterContext, + Attempt: attempt, + }, + }, + } + + if err := r.sendToWorker(executor.WorkerID, message); err != nil { + return nil, err + } + + select { + case <-ctx.Done(): + r.sendCancel(executor.WorkerID, requestID, plugin_pb.WorkKind_WORK_KIND_EXECUTION, ctx.Err()) + return nil, ctx.Err() + case completed, ok := <-completedCh: + if !ok { + return nil, fmt.Errorf("execution request %s interrupted", requestID) + } + if completed == nil { + return nil, fmt.Errorf("execution request %s returned empty completion", requestID) + } + if !completed.Success { + return completed, fmt.Errorf("job %s failed: %s", job.JobId, completed.ErrorMessage) + } + return completed, nil + } +} + +func (r *Plugin) ListWorkers() []*WorkerSession { + return r.registry.List() +} + +func (r *Plugin) ListKnownJobTypes() ([]string, error) { + registryJobTypes := r.registry.JobTypes() + storedJobTypes, err := r.store.ListJobTypes() + if err != nil { + return nil, err + } + + jobTypeSet := make(map[string]struct{}, len(registryJobTypes)+len(storedJobTypes)) + for _, jobType := range registryJobTypes { + jobTypeSet[jobType] = struct{}{} + } + for _, jobType := range storedJobTypes { + jobTypeSet[jobType] = struct{}{} + } + + out := make([]string, 0, len(jobTypeSet)) + for jobType := range jobTypeSet { + out = append(out, jobType) + } + sort.Strings(out) + return out, nil +} + +// FilterProposalsWithActiveJobs drops proposals that are already assigned/running. +func (r *Plugin) FilterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) { + return r.filterProposalsWithActiveJobs(jobType, proposals) +} + +func (r *Plugin) PickDetectorWorker(jobType string) (*WorkerSession, error) { + return r.pickDetector(jobType) +} + +func (r *Plugin) PickExecutorWorker(jobType string) (*WorkerSession, error) { + return r.registry.PickExecutor(jobType) +} + +func (r *Plugin) pickDetector(jobType string) (*WorkerSession, error) { + leasedWorkerID := r.getDetectorLease(jobType) + if leasedWorkerID != "" { + if worker, ok := r.registry.Get(leasedWorkerID); ok { + if capability := worker.Capabilities[jobType]; capability != nil && capability.CanDetect { + return worker, nil + } + } + r.clearDetectorLease(jobType, leasedWorkerID) + } + + detector, err := r.registry.PickDetector(jobType) + if err != nil { + return nil, err + } + + r.setDetectorLease(jobType, detector.WorkerID) + return detector, nil +} + +func (r *Plugin) getDetectorLease(jobType string) string { + r.detectorLeaseMu.Lock() + defer r.detectorLeaseMu.Unlock() + return r.detectorLeases[jobType] +} + +func (r *Plugin) setDetectorLease(jobType string, workerID string) { + r.detectorLeaseMu.Lock() + defer r.detectorLeaseMu.Unlock() + if jobType == "" || workerID == "" { + return + } + r.detectorLeases[jobType] = workerID +} + +func (r *Plugin) clearDetectorLease(jobType string, workerID string) { + r.detectorLeaseMu.Lock() + defer r.detectorLeaseMu.Unlock() + + current := r.detectorLeases[jobType] + if current == "" { + return + } + if workerID != "" && current != workerID { + return + } + delete(r.detectorLeases, jobType) +} + +func (r *Plugin) sendCancel(workerID, targetID string, kind plugin_pb.WorkKind, cause error) { + if strings.TrimSpace(workerID) == "" || strings.TrimSpace(targetID) == "" { + return + } + + requestID, err := newRequestID("cancel") + if err != nil { + requestID = "" + } + reason := "request canceled" + if cause != nil { + reason = cause.Error() + } + + message := &plugin_pb.AdminToWorkerMessage{ + RequestId: requestID, + SentAt: timestamppb.Now(), + Body: &plugin_pb.AdminToWorkerMessage_CancelRequest{ + CancelRequest: &plugin_pb.CancelRequest{ + TargetId: targetID, + TargetKind: kind, + Reason: reason, + }, + }, + } + if err := r.sendToWorker(workerID, message); err != nil { + glog.V(1).Infof("Plugin failed to send cancel request to worker=%s target=%s: %v", workerID, targetID, err) + } +} + +func (r *Plugin) sendAdminHello(workerID string) error { + msg := &plugin_pb.AdminToWorkerMessage{ + RequestId: "", + SentAt: timestamppb.Now(), + Body: &plugin_pb.AdminToWorkerMessage_Hello{ + Hello: &plugin_pb.AdminHello{ + Accepted: true, + Message: "plugin connected", + HeartbeatIntervalSeconds: defaultHeartbeatInterval, + ReconnectDelaySeconds: defaultReconnectDelay, + }, + }, + } + return r.sendToWorker(workerID, msg) +} + +func (r *Plugin) sendLoop( + ctx context.Context, + stream plugin_pb.PluginControlService_WorkerStreamServer, + session *streamSession, +) error { + for { + select { + case <-ctx.Done(): + return nil + case <-r.shutdownCh: + return nil + case msg, ok := <-session.outgoing: + if !ok { + return nil + } + if err := stream.Send(msg); err != nil { + return err + } + } + } +} + +func (r *Plugin) sendToWorker(workerID string, message *plugin_pb.AdminToWorkerMessage) error { + r.sessionsMu.RLock() + session, ok := r.sessions[workerID] + r.sessionsMu.RUnlock() + if !ok { + return fmt.Errorf("worker %s is not connected", workerID) + } + + select { + case <-r.shutdownCh: + return fmt.Errorf("plugin is shutting down") + case session.outgoing <- message: + return nil + case <-time.After(r.sendTimeout): + return fmt.Errorf("timed out sending message to worker %s", workerID) + } +} + +func (r *Plugin) handleWorkerMessage(workerID string, message *plugin_pb.WorkerToAdminMessage) { + if message == nil { + return + } + + switch body := message.Body.(type) { + case *plugin_pb.WorkerToAdminMessage_Hello: + r.registry.UpsertFromHello(body.Hello) + case *plugin_pb.WorkerToAdminMessage_Heartbeat: + r.registry.UpdateHeartbeat(workerID, body.Heartbeat) + case *plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse: + r.handleConfigSchemaResponse(body.ConfigSchemaResponse) + case *plugin_pb.WorkerToAdminMessage_DetectionProposals: + r.handleDetectionProposals(workerID, body.DetectionProposals) + case *plugin_pb.WorkerToAdminMessage_DetectionComplete: + r.handleDetectionComplete(workerID, body.DetectionComplete) + case *plugin_pb.WorkerToAdminMessage_JobProgressUpdate: + r.handleJobProgressUpdate(workerID, body.JobProgressUpdate) + case *plugin_pb.WorkerToAdminMessage_JobCompleted: + r.handleJobCompleted(body.JobCompleted) + case *plugin_pb.WorkerToAdminMessage_Acknowledge: + if !body.Acknowledge.Accepted { + glog.Warningf("Plugin worker %s rejected request %s: %s", workerID, body.Acknowledge.RequestId, body.Acknowledge.Message) + } + default: + // Keep the transport open even if admin does not yet consume all message variants. + } +} + +func (r *Plugin) handleConfigSchemaResponse(response *plugin_pb.ConfigSchemaResponse) { + if response == nil { + return + } + + if response.Success && response.GetJobTypeDescriptor() != nil { + jobType := response.JobType + if jobType == "" { + jobType = response.GetJobTypeDescriptor().JobType + } + if jobType != "" { + if err := r.store.SaveDescriptor(jobType, response.GetJobTypeDescriptor()); err != nil { + glog.Warningf("Plugin failed to persist descriptor for %s: %v", jobType, err) + } + if err := r.ensureJobTypeConfigFromDescriptor(jobType, response.GetJobTypeDescriptor()); err != nil { + glog.Warningf("Plugin failed to bootstrap config for %s: %v", jobType, err) + } + } + } + + r.safeSendSchemaResponse(response.RequestId, response) +} + +func (r *Plugin) safeSendSchemaResponse(requestID string, response *plugin_pb.ConfigSchemaResponse) { + r.pendingSchemaMu.Lock() + ch := r.pendingSchema[requestID] + r.pendingSchemaMu.Unlock() + safeSendCh(ch, response, r.shutdownCh) +} + +func safeSendCh[T any](ch chan T, val T, shutdownCh <-chan struct{}) { + if ch == nil { + return + } + defer func() { + recover() + }() + select { + case ch <- val: + case <-shutdownCh: + } +} + +func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error { + if descriptor == nil || strings.TrimSpace(jobType) == "" { + return nil + } + + existing, err := r.store.LoadJobTypeConfig(jobType) + if err != nil { + return err + } + if existing != nil { + return nil + } + + workerDefaults := CloneConfigValueMap(descriptor.WorkerDefaultValues) + if len(workerDefaults) == 0 && descriptor.WorkerConfigForm != nil { + workerDefaults = CloneConfigValueMap(descriptor.WorkerConfigForm.DefaultValues) + } + + adminDefaults := map[string]*plugin_pb.ConfigValue{} + if descriptor.AdminConfigForm != nil { + adminDefaults = CloneConfigValueMap(descriptor.AdminConfigForm.DefaultValues) + } + + adminRuntime := &plugin_pb.AdminRuntimeConfig{} + if descriptor.AdminRuntimeDefaults != nil { + defaults := descriptor.AdminRuntimeDefaults + adminRuntime = &plugin_pb.AdminRuntimeConfig{ + Enabled: defaults.Enabled, + DetectionIntervalSeconds: defaults.DetectionIntervalSeconds, + DetectionTimeoutSeconds: defaults.DetectionTimeoutSeconds, + MaxJobsPerDetection: defaults.MaxJobsPerDetection, + GlobalExecutionConcurrency: defaults.GlobalExecutionConcurrency, + PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency, + RetryLimit: defaults.RetryLimit, + RetryBackoffSeconds: defaults.RetryBackoffSeconds, + } + } + + cfg := &plugin_pb.PersistedJobTypeConfig{ + JobType: jobType, + DescriptorVersion: descriptor.DescriptorVersion, + AdminConfigValues: adminDefaults, + WorkerConfigValues: workerDefaults, + AdminRuntime: adminRuntime, + UpdatedAt: timestamppb.Now(), + UpdatedBy: "plugin", + } + + return r.store.SaveJobTypeConfig(cfg) +} + +func (r *Plugin) handleDetectionProposals(workerID string, message *plugin_pb.DetectionProposals) { + if message == nil || message.RequestId == "" { + return + } + + r.pendingDetectionMu.Lock() + state := r.pendingDetection[message.RequestId] + if state != nil { + state.proposals = append(state.proposals, cloneJobProposals(message.Proposals)...) + } + r.pendingDetectionMu.Unlock() + if state == nil { + return + } + + resolvedWorkerID := strings.TrimSpace(workerID) + if resolvedWorkerID == "" { + resolvedWorkerID = state.workerID + } + resolvedJobType := strings.TrimSpace(message.JobType) + if resolvedJobType == "" { + resolvedJobType = state.jobType + } + if resolvedJobType == "" { + resolvedJobType = "unknown" + } + + r.appendActivity(JobActivity{ + JobType: resolvedJobType, + RequestID: message.RequestId, + WorkerID: resolvedWorkerID, + Source: "detector", + Stage: "proposals_batch", + Message: fmt.Sprintf("received %d proposal(s)", len(message.Proposals)), + OccurredAt: timeToPtr(time.Now().UTC()), + Details: map[string]interface{}{ + "batch_size": len(message.Proposals), + "has_more": message.HasMore, + }, + }) + + for _, proposal := range message.Proposals { + if proposal == nil { + continue + } + details := map[string]interface{}{ + "proposal_id": proposal.ProposalId, + "dedupe_key": proposal.DedupeKey, + "priority": proposal.Priority.String(), + "summary": proposal.Summary, + "detail": proposal.Detail, + "labels": proposal.Labels, + } + if params := configValueMapToPlain(proposal.Parameters); len(params) > 0 { + details["parameters"] = params + } + + messageText := strings.TrimSpace(proposal.Summary) + if messageText == "" { + messageText = fmt.Sprintf("proposal %s", strings.TrimSpace(proposal.ProposalId)) + } + if messageText == "" { + messageText = "proposal generated" + } + + r.appendActivity(JobActivity{ + JobType: resolvedJobType, + RequestID: message.RequestId, + WorkerID: resolvedWorkerID, + Source: "detector", + Stage: "proposal", + Message: messageText, + OccurredAt: timeToPtr(time.Now().UTC()), + Details: details, + }) + } +} + +func (r *Plugin) handleDetectionComplete(workerID string, message *plugin_pb.DetectionComplete) { + if message == nil { + return + } + if !message.Success { + glog.Warningf("Plugin detection failed job_type=%s: %s", message.JobType, message.ErrorMessage) + } + if message.RequestId == "" { + return + } + + r.pendingDetectionMu.Lock() + state := r.pendingDetection[message.RequestId] + r.pendingDetectionMu.Unlock() + if state == nil { + return + } + + resolvedWorkerID := strings.TrimSpace(workerID) + if resolvedWorkerID == "" { + resolvedWorkerID = state.workerID + } + resolvedJobType := strings.TrimSpace(message.JobType) + if resolvedJobType == "" { + resolvedJobType = state.jobType + } + if resolvedJobType == "" { + resolvedJobType = "unknown" + } + + stage := "completed" + messageText := "detection completed" + if !message.Success { + stage = "failed" + messageText = strings.TrimSpace(message.ErrorMessage) + if messageText == "" { + messageText = "detection failed" + } + } + r.appendActivity(JobActivity{ + JobType: resolvedJobType, + RequestID: message.RequestId, + WorkerID: resolvedWorkerID, + Source: "detector", + Stage: stage, + Message: messageText, + OccurredAt: timeToPtr(time.Now().UTC()), + Details: map[string]interface{}{ + "success": message.Success, + "total_proposals": message.TotalProposals, + }, + }) + + r.safeSendDetectionComplete(message.RequestId, message) +} + +func (r *Plugin) safeSendDetectionComplete(requestID string, message *plugin_pb.DetectionComplete) { + r.pendingDetectionMu.Lock() + state, found := r.pendingDetection[requestID] + var ch chan *plugin_pb.DetectionComplete + if found && state != nil { + ch = state.complete + } + r.pendingDetectionMu.Unlock() + + if ch != nil { + safeSendCh(ch, message, r.shutdownCh) + } +} + +func (r *Plugin) handleJobCompleted(completed *plugin_pb.JobCompleted) { + if completed == nil || completed.JobType == "" { + return + } + + if completed.RequestId != "" { + r.safeSendJobCompleted(completed.RequestId, completed) + } + + tracked := r.trackExecutionCompletion(completed) + workerID := "" + if tracked != nil && tracked.WorkerID != "" { + workerID = tracked.WorkerID + } + + r.trackWorkerActivities(completed.JobType, completed.JobId, completed.RequestId, workerID, completed.Activities) + + record := &JobRunRecord{ + RunID: completed.RequestId, + JobID: completed.JobId, + JobType: completed.JobType, + WorkerID: "", + Outcome: RunOutcomeError, + CompletedAt: timeToPtr(time.Now().UTC()), + } + if completed.CompletedAt != nil { + record.CompletedAt = timeToPtr(completed.CompletedAt.AsTime().UTC()) + } + if completed.Success { + record.Outcome = RunOutcomeSuccess + record.Message = "completed" + if completed.Result != nil && completed.Result.Summary != "" { + record.Message = completed.Result.Summary + } + } else { + record.Outcome = RunOutcomeError + record.Message = completed.ErrorMessage + } + + if tracked != nil { + if workerID != "" { + record.WorkerID = workerID + } + if tracked.CreatedAt != nil && record.CompletedAt != nil && record.CompletedAt.After(*tracked.CreatedAt) { + record.DurationMs = int64(record.CompletedAt.Sub(*tracked.CreatedAt) / time.Millisecond) + } + } + + if err := r.store.AppendRunRecord(completed.JobType, record); err != nil { + glog.Warningf("Plugin failed to append run record for %s: %v", completed.JobType, err) + } +} + +func (r *Plugin) safeSendJobCompleted(requestID string, completed *plugin_pb.JobCompleted) { + r.pendingExecutionMu.Lock() + ch := r.pendingExecution[requestID] + r.pendingExecutionMu.Unlock() + safeSendCh(ch, completed, r.shutdownCh) +} + +func (r *Plugin) putSession(session *streamSession) { + r.sessionsMu.Lock() + defer r.sessionsMu.Unlock() + + if old, exists := r.sessions[session.workerID]; exists { + old.close() + } + r.sessions[session.workerID] = session +} + +func (r *Plugin) cleanupSession(workerID string) { + r.registry.Remove(workerID) + + r.sessionsMu.Lock() + session, exists := r.sessions[workerID] + if exists { + delete(r.sessions, workerID) + session.close() + } + r.sessionsMu.Unlock() + + glog.V(0).Infof("Plugin worker disconnected: %s", workerID) +} + +func newRequestID(prefix string) (string, error) { + buf := make([]byte, 8) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("generate request id: %w", err) + } + if prefix == "" { + prefix = "req" + } + return fmt.Sprintf("%s-%d-%s", prefix, time.Now().UnixNano(), hex.EncodeToString(buf)), nil +} + +func (r *Plugin) loadJobTypeConfigPayload(jobType string) ( + *plugin_pb.AdminRuntimeConfig, + map[string]*plugin_pb.ConfigValue, + map[string]*plugin_pb.ConfigValue, + error, +) { + config, err := r.store.LoadJobTypeConfig(jobType) + if err != nil { + return nil, nil, nil, err + } + + if config == nil { + return &plugin_pb.AdminRuntimeConfig{}, map[string]*plugin_pb.ConfigValue{}, map[string]*plugin_pb.ConfigValue{}, nil + } + + adminRuntime := config.AdminRuntime + if adminRuntime == nil { + adminRuntime = &plugin_pb.AdminRuntimeConfig{} + } + return adminRuntime, CloneConfigValueMap(config.AdminConfigValues), CloneConfigValueMap(config.WorkerConfigValues), nil +} + +func cloneJobProposals(in []*plugin_pb.JobProposal) []*plugin_pb.JobProposal { + if len(in) == 0 { + return nil + } + out := make([]*plugin_pb.JobProposal, 0, len(in)) + for _, proposal := range in { + if proposal == nil { + continue + } + out = append(out, proto.Clone(proposal).(*plugin_pb.JobProposal)) + } + return out +} + +func (r *Plugin) loadLastSuccessfulRun(jobType string) *timestamppb.Timestamp { + history, err := r.store.LoadRunHistory(jobType) + if err != nil { + glog.Warningf("Plugin failed to load run history for %s: %v", jobType, err) + return nil + } + if history == nil || len(history.SuccessfulRuns) == 0 { + return nil + } + + var latest time.Time + for i := range history.SuccessfulRuns { + completedAt := history.SuccessfulRuns[i].CompletedAt + if completedAt == nil || completedAt.IsZero() { + continue + } + if latest.IsZero() || completedAt.After(latest) { + latest = *completedAt + } + } + if latest.IsZero() { + return nil + } + return timestamppb.New(latest.UTC()) +} + +func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugin_pb.ConfigValue { + if len(in) == 0 { + return map[string]*plugin_pb.ConfigValue{} + } + out := make(map[string]*plugin_pb.ConfigValue, len(in)) + for key, value := range in { + if value == nil { + continue + } + out[key] = proto.Clone(value).(*plugin_pb.ConfigValue) + } + return out +} + +func (s *streamSession) close() { + s.closeOnce.Do(func() { + close(s.outgoing) + }) +} diff --git a/weed/admin/plugin/plugin_cancel_test.go b/weed/admin/plugin/plugin_cancel_test.go new file mode 100644 index 000000000..bb597e3f7 --- /dev/null +++ b/weed/admin/plugin/plugin_cancel_test.go @@ -0,0 +1,112 @@ +package plugin + +import ( + "context" + "errors" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestRunDetectionSendsCancelOnContextDone(t *testing.T) { + t.Parallel() + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New plugin error: %v", err) + } + defer pluginSvc.Shutdown() + + const workerID = "worker-detect" + const jobType = "vacuum" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: workerID, + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1}, + }, + }) + session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)} + pluginSvc.putSession(session) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, runErr := pluginSvc.RunDetection(ctx, jobType, &plugin_pb.ClusterContext{}, 10) + errCh <- runErr + }() + + first := <-session.outgoing + if first.GetRunDetectionRequest() == nil { + t.Fatalf("expected first message to be run_detection_request") + } + + cancel() + + second := <-session.outgoing + cancelReq := second.GetCancelRequest() + if cancelReq == nil { + t.Fatalf("expected second message to be cancel_request") + } + if cancelReq.TargetId != first.RequestId { + t.Fatalf("unexpected cancel target id: got=%s want=%s", cancelReq.TargetId, first.RequestId) + } + if cancelReq.TargetKind != plugin_pb.WorkKind_WORK_KIND_DETECTION { + t.Fatalf("unexpected cancel target kind: %v", cancelReq.TargetKind) + } + + runErr := <-errCh + if !errors.Is(runErr, context.Canceled) { + t.Fatalf("expected context canceled error, got %v", runErr) + } +} + +func TestExecuteJobSendsCancelOnContextDone(t *testing.T) { + t.Parallel() + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New plugin error: %v", err) + } + defer pluginSvc.Shutdown() + + const workerID = "worker-exec" + const jobType = "vacuum" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: workerID, + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1}, + }, + }) + session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)} + pluginSvc.putSession(session) + + job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType} + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + _, runErr := pluginSvc.ExecuteJob(ctx, job, &plugin_pb.ClusterContext{}, 1) + errCh <- runErr + }() + + first := <-session.outgoing + if first.GetExecuteJobRequest() == nil { + t.Fatalf("expected first message to be execute_job_request") + } + + cancel() + + second := <-session.outgoing + cancelReq := second.GetCancelRequest() + if cancelReq == nil { + t.Fatalf("expected second message to be cancel_request") + } + if cancelReq.TargetId != first.RequestId { + t.Fatalf("unexpected cancel target id: got=%s want=%s", cancelReq.TargetId, first.RequestId) + } + if cancelReq.TargetKind != plugin_pb.WorkKind_WORK_KIND_EXECUTION { + t.Fatalf("unexpected cancel target kind: %v", cancelReq.TargetKind) + } + + runErr := <-errCh + if !errors.Is(runErr, context.Canceled) { + t.Fatalf("expected context canceled error, got %v", runErr) + } +} diff --git a/weed/admin/plugin/plugin_config_bootstrap_test.go b/weed/admin/plugin/plugin_config_bootstrap_test.go new file mode 100644 index 000000000..83eacf284 --- /dev/null +++ b/weed/admin/plugin/plugin_config_bootstrap_test.go @@ -0,0 +1,125 @@ +package plugin + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestEnsureJobTypeConfigFromDescriptorBootstrapsDefaults(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + descriptor := &plugin_pb.JobTypeDescriptor{ + JobType: "vacuum", + DescriptorVersion: 3, + AdminConfigForm: &plugin_pb.ConfigForm{ + DefaultValues: map[string]*plugin_pb.ConfigValue{ + "scan_scope": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "all"}}, + }, + }, + WorkerConfigForm: &plugin_pb.ConfigForm{ + DefaultValues: map[string]*plugin_pb.ConfigValue{ + "threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3}}, + }, + }, + AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ + Enabled: true, + DetectionIntervalSeconds: 60, + DetectionTimeoutSeconds: 20, + MaxJobsPerDetection: 30, + GlobalExecutionConcurrency: 4, + PerWorkerExecutionConcurrency: 2, + RetryLimit: 3, + RetryBackoffSeconds: 5, + }, + } + + if err := pluginSvc.ensureJobTypeConfigFromDescriptor("vacuum", descriptor); err != nil { + t.Fatalf("ensureJobTypeConfigFromDescriptor: %v", err) + } + + cfg, err := pluginSvc.LoadJobTypeConfig("vacuum") + if err != nil { + t.Fatalf("LoadJobTypeConfig: %v", err) + } + if cfg == nil { + t.Fatalf("expected non-nil config") + } + if cfg.DescriptorVersion != 3 { + t.Fatalf("unexpected descriptor version: got=%d", cfg.DescriptorVersion) + } + if cfg.AdminRuntime == nil || !cfg.AdminRuntime.Enabled { + t.Fatalf("expected enabled admin settings") + } + if cfg.AdminRuntime.GlobalExecutionConcurrency != 4 { + t.Fatalf("unexpected global execution concurrency: %d", cfg.AdminRuntime.GlobalExecutionConcurrency) + } + if _, ok := cfg.AdminConfigValues["scan_scope"]; !ok { + t.Fatalf("missing admin default value") + } + if _, ok := cfg.WorkerConfigValues["threshold"]; !ok { + t.Fatalf("missing worker default value") + } +} + +func TestEnsureJobTypeConfigFromDescriptorDoesNotOverwriteExisting(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + if err := pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{ + JobType: "balance", + AdminRuntime: &plugin_pb.AdminRuntimeConfig{ + Enabled: true, + GlobalExecutionConcurrency: 9, + }, + AdminConfigValues: map[string]*plugin_pb.ConfigValue{ + "custom": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "keep"}}, + }, + }); err != nil { + t.Fatalf("SaveJobTypeConfig: %v", err) + } + + descriptor := &plugin_pb.JobTypeDescriptor{ + JobType: "balance", + DescriptorVersion: 7, + AdminConfigForm: &plugin_pb.ConfigForm{ + DefaultValues: map[string]*plugin_pb.ConfigValue{ + "custom": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "overwrite"}}, + }, + }, + AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ + Enabled: true, + GlobalExecutionConcurrency: 1, + }, + } + + if err := pluginSvc.ensureJobTypeConfigFromDescriptor("balance", descriptor); err != nil { + t.Fatalf("ensureJobTypeConfigFromDescriptor: %v", err) + } + + cfg, err := pluginSvc.LoadJobTypeConfig("balance") + if err != nil { + t.Fatalf("LoadJobTypeConfig: %v", err) + } + if cfg == nil { + t.Fatalf("expected config") + } + if cfg.AdminRuntime == nil || cfg.AdminRuntime.GlobalExecutionConcurrency != 9 { + t.Fatalf("existing admin settings should be preserved, got=%v", cfg.AdminRuntime) + } + custom := cfg.AdminConfigValues["custom"] + if custom == nil || custom.GetStringValue() != "keep" { + t.Fatalf("existing admin config should be preserved") + } +} diff --git a/weed/admin/plugin/plugin_detection_test.go b/weed/admin/plugin/plugin_detection_test.go new file mode 100644 index 000000000..755ade4cd --- /dev/null +++ b/weed/admin/plugin/plugin_detection_test.go @@ -0,0 +1,197 @@ +package plugin + +import ( + "context" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) { + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New plugin error: %v", err) + } + defer pluginSvc.Shutdown() + + jobType := "vacuum" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1}, + }, + }) + session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)} + pluginSvc.putSession(session) + + oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + latestSuccess := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) + if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(oldSuccess)}); err != nil { + t.Fatalf("AppendRunRecord old success: %v", err) + } + if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeError, CompletedAt: timeToPtr(latestSuccess.Add(2 * time.Hour))}); err != nil { + t.Fatalf("AppendRunRecord error run: %v", err) + } + if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(latestSuccess)}); err != nil { + t.Fatalf("AppendRunRecord latest success: %v", err) + } + + resultCh := make(chan error, 1) + go func() { + _, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10) + resultCh <- runErr + }() + + message := <-session.outgoing + detectRequest := message.GetRunDetectionRequest() + if detectRequest == nil { + t.Fatalf("expected run detection request message") + } + if detectRequest.LastSuccessfulRun == nil { + t.Fatalf("expected last_successful_run to be set") + } + if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(latestSuccess) { + t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, latestSuccess) + } + + pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{ + RequestId: message.RequestId, + JobType: jobType, + Success: true, + }) + + if runErr := <-resultCh; runErr != nil { + t.Fatalf("RunDetection error: %v", runErr) + } +} + +func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) { + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New plugin error: %v", err) + } + defer pluginSvc.Shutdown() + + jobType := "vacuum" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1}, + }, + }) + session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)} + pluginSvc.putSession(session) + + if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{ + Outcome: RunOutcomeError, + CompletedAt: timeToPtr(time.Date(2026, 2, 10, 0, 0, 0, 0, time.UTC)), + }); err != nil { + t.Fatalf("AppendRunRecord error run: %v", err) + } + + resultCh := make(chan error, 1) + go func() { + _, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10) + resultCh <- runErr + }() + + message := <-session.outgoing + detectRequest := message.GetRunDetectionRequest() + if detectRequest == nil { + t.Fatalf("expected run detection request message") + } + if detectRequest.LastSuccessfulRun != nil { + t.Fatalf("expected last_successful_run to be nil when no success history") + } + + pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{ + RequestId: message.RequestId, + JobType: jobType, + Success: true, + }) + + if runErr := <-resultCh; runErr != nil { + t.Fatalf("RunDetection error: %v", runErr) + } +} + +func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) { + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New plugin error: %v", err) + } + defer pluginSvc.Shutdown() + + jobType := "vacuum" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1}, + }, + }) + session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)} + pluginSvc.putSession(session) + + reportCh := make(chan *DetectionReport, 1) + errCh := make(chan error, 1) + go func() { + report, runErr := pluginSvc.RunDetectionWithReport(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10) + reportCh <- report + errCh <- runErr + }() + + message := <-session.outgoing + requestID := message.GetRequestId() + if requestID == "" { + t.Fatalf("expected request id in detection request") + } + + pluginSvc.handleDetectionProposals("worker-a", &plugin_pb.DetectionProposals{ + RequestId: requestID, + JobType: jobType, + Proposals: []*plugin_pb.JobProposal{ + { + ProposalId: "proposal-1", + JobType: jobType, + Summary: "vacuum proposal", + Detail: "based on garbage ratio", + }, + }, + }) + pluginSvc.handleDetectionComplete("worker-a", &plugin_pb.DetectionComplete{ + RequestId: requestID, + JobType: jobType, + Success: true, + TotalProposals: 1, + }) + + report := <-reportCh + if report == nil { + t.Fatalf("expected detection report") + } + if report.RequestID == "" { + t.Fatalf("expected detection report request id") + } + if report.WorkerID != "worker-a" { + t.Fatalf("expected worker-a, got %q", report.WorkerID) + } + if len(report.Proposals) != 1 { + t.Fatalf("expected one proposal in report, got %d", len(report.Proposals)) + } + if runErr := <-errCh; runErr != nil { + t.Fatalf("RunDetectionWithReport error: %v", runErr) + } + + activities := pluginSvc.ListActivities(jobType, 0) + stages := map[string]bool{} + for _, activity := range activities { + if activity.RequestID != report.RequestID { + continue + } + stages[activity.Stage] = true + } + if !stages["requested"] || !stages["proposal"] || !stages["completed"] { + t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages) + } +} diff --git a/weed/admin/plugin/plugin_monitor.go b/weed/admin/plugin/plugin_monitor.go new file mode 100644 index 000000000..8ced147ae --- /dev/null +++ b/weed/admin/plugin/plugin_monitor.go @@ -0,0 +1,896 @@ +package plugin + +import ( + "encoding/json" + "sort" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + maxTrackedJobsTotal = 1000 + maxActivityRecords = 4000 + maxRelatedJobs = 100 +) + +var ( + StateSucceeded = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String()) + StateFailed = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String()) + StateCanceled = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String()) +) + +// activityLess reports whether activity a occurred after activity b (newest-first order). +// A nil OccurredAt is treated as the zero time. +func activityLess(a, b JobActivity) bool { + ta := time.Time{} + if a.OccurredAt != nil { + ta = *a.OccurredAt + } + tb := time.Time{} + if b.OccurredAt != nil { + tb = *b.OccurredAt + } + return ta.After(tb) +} + +func (r *Plugin) loadPersistedMonitorState() error { + trackedJobs, err := r.store.LoadTrackedJobs() + if err != nil { + return err + } + activities, err := r.store.LoadActivities() + if err != nil { + return err + } + + if len(trackedJobs) > 0 { + r.jobsMu.Lock() + for i := range trackedJobs { + job := trackedJobs[i] + if strings.TrimSpace(job.JobID) == "" { + continue + } + // Backward compatibility: migrate older inline detail payloads + // out of tracked_jobs.json into dedicated per-job detail files. + if hasTrackedJobRichDetails(job) { + if err := r.store.SaveJobDetail(job); err != nil { + glog.Warningf("Plugin failed to migrate detail snapshot for job %s: %v", job.JobID, err) + } + } + stripTrackedJobDetailFields(&job) + jobCopy := job + r.jobs[job.JobID] = &jobCopy + } + r.pruneTrackedJobsLocked() + r.jobsMu.Unlock() + } + + if len(activities) > maxActivityRecords { + activities = activities[len(activities)-maxActivityRecords:] + } + if len(activities) > 0 { + r.activitiesMu.Lock() + r.activities = append([]JobActivity(nil), activities...) + r.activitiesMu.Unlock() + } + + return nil +} + +func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob { + r.jobsMu.RLock() + defer r.jobsMu.RUnlock() + + normalizedJobType := strings.TrimSpace(jobType) + normalizedState := strings.TrimSpace(strings.ToLower(state)) + + items := make([]TrackedJob, 0, len(r.jobs)) + for _, job := range r.jobs { + if job == nil { + continue + } + if normalizedJobType != "" && job.JobType != normalizedJobType { + continue + } + if normalizedState != "" && strings.ToLower(job.State) != normalizedState { + continue + } + items = append(items, cloneTrackedJob(*job)) + } + + sort.Slice(items, func(i, j int) bool { + ti := time.Time{} + if items[i].UpdatedAt != nil { + ti = *items[i].UpdatedAt + } + tj := time.Time{} + if items[j].UpdatedAt != nil { + tj = *items[j].UpdatedAt + } + if !ti.Equal(tj) { + return ti.After(tj) + } + return items[i].JobID < items[j].JobID + }) + + if limit > 0 && len(items) > limit { + items = items[:limit] + } + return items +} + +func (r *Plugin) GetTrackedJob(jobID string) (*TrackedJob, bool) { + r.jobsMu.RLock() + defer r.jobsMu.RUnlock() + + job, ok := r.jobs[jobID] + if !ok || job == nil { + return nil, false + } + clone := cloneTrackedJob(*job) + return &clone, true +} + +func (r *Plugin) ListActivities(jobType string, limit int) []JobActivity { + r.activitiesMu.RLock() + defer r.activitiesMu.RUnlock() + + normalized := strings.TrimSpace(jobType) + activities := make([]JobActivity, 0, len(r.activities)) + for _, activity := range r.activities { + if normalized != "" && activity.JobType != normalized { + continue + } + activities = append(activities, activity) + } + + sort.Slice(activities, func(i, j int) bool { + return activityLess(activities[i], activities[j]) + }) + if limit > 0 && len(activities) > limit { + activities = activities[:limit] + } + return activities +} + +func (r *Plugin) ListJobActivities(jobID string, limit int) []JobActivity { + normalizedJobID := strings.TrimSpace(jobID) + if normalizedJobID == "" { + return nil + } + + r.activitiesMu.RLock() + activities := make([]JobActivity, 0, len(r.activities)) + for _, activity := range r.activities { + if strings.TrimSpace(activity.JobID) != normalizedJobID { + continue + } + activities = append(activities, activity) + } + r.activitiesMu.RUnlock() + + sort.Slice(activities, func(i, j int) bool { + return !activityLess(activities[i], activities[j]) // oldest-first for job timeline + }) + if limit > 0 && len(activities) > limit { + activities = activities[len(activities)-limit:] + } + return activities +} + +func (r *Plugin) BuildJobDetail(jobID string, activityLimit int, relatedLimit int) (*JobDetail, bool, error) { + normalizedJobID := strings.TrimSpace(jobID) + if normalizedJobID == "" { + return nil, false, nil + } + + // Clamp relatedLimit to a safe range to avoid excessive memory allocation from untrusted input. + if relatedLimit <= 0 { + relatedLimit = 0 + } else if relatedLimit > maxRelatedJobs { + relatedLimit = maxRelatedJobs + } + + r.jobsMu.RLock() + trackedSnapshot, ok := r.jobs[normalizedJobID] + if ok && trackedSnapshot != nil { + candidate := cloneTrackedJob(*trackedSnapshot) + stripTrackedJobDetailFields(&candidate) + trackedSnapshot = &candidate + } else { + trackedSnapshot = nil + } + r.jobsMu.RUnlock() + + detailJob, err := r.store.LoadJobDetail(normalizedJobID) + if err != nil { + return nil, false, err + } + + if trackedSnapshot == nil && detailJob == nil { + return nil, false, nil + } + if detailJob == nil && trackedSnapshot != nil { + clone := cloneTrackedJob(*trackedSnapshot) + detailJob = &clone + } + if detailJob == nil { + return nil, false, nil + } + if trackedSnapshot != nil { + mergeTrackedStatusIntoDetail(detailJob, trackedSnapshot) + } + detailJob.Parameters = enrichTrackedJobParameters(detailJob.JobType, detailJob.Parameters) + + r.activitiesMu.RLock() + activities := append([]JobActivity(nil), r.activities...) + r.activitiesMu.RUnlock() + + detail := &JobDetail{ + Job: detailJob, + Activities: filterJobActivitiesFromSlice(activities, normalizedJobID, activityLimit), + LastUpdated: timeToPtr(time.Now().UTC()), + } + + if history, err := r.store.LoadRunHistory(detailJob.JobType); err != nil { + return nil, true, err + } else if history != nil { + for i := range history.SuccessfulRuns { + record := history.SuccessfulRuns[i] + if strings.TrimSpace(record.JobID) == normalizedJobID { + recordCopy := record + detail.RunRecord = &recordCopy + break + } + } + if detail.RunRecord == nil { + for i := range history.ErrorRuns { + record := history.ErrorRuns[i] + if strings.TrimSpace(record.JobID) == normalizedJobID { + recordCopy := record + detail.RunRecord = &recordCopy + break + } + } + } + } + + if relatedLimit > 0 { + related := make([]TrackedJob, 0, relatedLimit) + r.jobsMu.RLock() + for _, candidate := range r.jobs { + if strings.TrimSpace(candidate.JobType) != strings.TrimSpace(detailJob.JobType) { + continue + } + if strings.TrimSpace(candidate.JobID) == normalizedJobID { + continue + } + cloned := cloneTrackedJob(*candidate) + stripTrackedJobDetailFields(&cloned) + related = append(related, cloned) + if len(related) >= relatedLimit { + break + } + } + r.jobsMu.RUnlock() + detail.RelatedJobs = related + } + + return detail, true, nil +} + +func filterJobActivitiesFromSlice(all []JobActivity, jobID string, limit int) []JobActivity { + if strings.TrimSpace(jobID) == "" || len(all) == 0 { + return nil + } + + activities := make([]JobActivity, 0, len(all)) + for _, activity := range all { + if strings.TrimSpace(activity.JobID) != jobID { + continue + } + activities = append(activities, activity) + } + + sort.Slice(activities, func(i, j int) bool { + return !activityLess(activities[i], activities[j]) // oldest-first for job timeline + }) + if limit > 0 && len(activities) > limit { + activities = activities[len(activities)-limit:] + } + return activities +} + +func stripTrackedJobDetailFields(job *TrackedJob) { + if job == nil { + return + } + job.Detail = "" + job.Parameters = nil + job.Labels = nil + job.ResultOutputValues = nil +} + +func hasTrackedJobRichDetails(job TrackedJob) bool { + return strings.TrimSpace(job.Detail) != "" || + len(job.Parameters) > 0 || + len(job.Labels) > 0 || + len(job.ResultOutputValues) > 0 +} + +func mergeTrackedStatusIntoDetail(detail *TrackedJob, tracked *TrackedJob) { + if detail == nil || tracked == nil { + return + } + + if detail.JobType == "" { + detail.JobType = tracked.JobType + } + if detail.RequestID == "" { + detail.RequestID = tracked.RequestID + } + if detail.WorkerID == "" { + detail.WorkerID = tracked.WorkerID + } + if detail.DedupeKey == "" { + detail.DedupeKey = tracked.DedupeKey + } + if detail.Summary == "" { + detail.Summary = tracked.Summary + } + if detail.State == "" { + detail.State = tracked.State + } + if detail.Progress == 0 { + detail.Progress = tracked.Progress + } + if detail.Stage == "" { + detail.Stage = tracked.Stage + } + if detail.Message == "" { + detail.Message = tracked.Message + } + if detail.Attempt == 0 { + detail.Attempt = tracked.Attempt + } + if detail.CreatedAt == nil || detail.CreatedAt.IsZero() { + detail.CreatedAt = tracked.CreatedAt + } + if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() { + detail.UpdatedAt = tracked.UpdatedAt + } + if detail.CompletedAt == nil || detail.CompletedAt.IsZero() { + detail.CompletedAt = tracked.CompletedAt + } + if detail.ErrorMessage == "" { + detail.ErrorMessage = tracked.ErrorMessage + } + if detail.ResultSummary == "" { + detail.ResultSummary = tracked.ResultSummary + } +} + +func (r *Plugin) handleJobProgressUpdate(workerID string, update *plugin_pb.JobProgressUpdate) { + if update == nil { + return + } + + now := time.Now().UTC() + resolvedWorkerID := strings.TrimSpace(workerID) + + if strings.TrimSpace(update.JobId) != "" { + r.jobsMu.Lock() + job := r.jobs[update.JobId] + if job == nil { + job = &TrackedJob{ + JobID: update.JobId, + JobType: update.JobType, + RequestID: update.RequestId, + WorkerID: resolvedWorkerID, + CreatedAt: timeToPtr(now), + } + r.jobs[update.JobId] = job + } + + if update.JobType != "" { + job.JobType = update.JobType + } + if update.RequestId != "" { + job.RequestID = update.RequestId + } + if job.WorkerID != "" { + resolvedWorkerID = job.WorkerID + } else if resolvedWorkerID != "" { + job.WorkerID = resolvedWorkerID + } + job.State = strings.ToLower(update.State.String()) + job.Progress = update.ProgressPercent + job.Stage = update.Stage + job.Message = update.Message + job.UpdatedAt = timeToPtr(now) + r.pruneTrackedJobsLocked() + r.dirtyJobs = true + r.jobsMu.Unlock() + } + + r.trackWorkerActivities(update.JobType, update.JobId, update.RequestId, resolvedWorkerID, update.Activities) + if update.Message != "" || update.Stage != "" { + source := "worker_progress" + if strings.TrimSpace(update.JobId) == "" { + source = "worker_detection" + } + r.appendActivity(JobActivity{ + JobID: update.JobId, + JobType: update.JobType, + RequestID: update.RequestId, + WorkerID: resolvedWorkerID, + Source: source, + Message: update.Message, + Stage: update.Stage, + OccurredAt: timeToPtr(now), + }) + } +} + +func (r *Plugin) trackExecutionStart(requestID, workerID string, job *plugin_pb.JobSpec, attempt int32) { + if job == nil || strings.TrimSpace(job.JobId) == "" { + return + } + + now := time.Now().UTC() + + r.jobsMu.Lock() + tracked := r.jobs[job.JobId] + if tracked == nil { + tracked = &TrackedJob{ + JobID: job.JobId, + CreatedAt: timeToPtr(now), + } + r.jobs[job.JobId] = tracked + } + + tracked.JobType = job.JobType + tracked.RequestID = requestID + tracked.WorkerID = workerID + tracked.DedupeKey = job.DedupeKey + tracked.Summary = job.Summary + tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_ASSIGNED.String()) + tracked.Progress = 0 + tracked.Stage = "assigned" + tracked.Message = "job assigned to worker" + tracked.Attempt = attempt + if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() { + tracked.CreatedAt = timeToPtr(now) + } + tracked.UpdatedAt = timeToPtr(now) + trackedSnapshot := cloneTrackedJob(*tracked) + r.pruneTrackedJobsLocked() + r.dirtyJobs = true + r.jobsMu.Unlock() + r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) { + detail.JobID = job.JobId + detail.JobType = job.JobType + detail.RequestID = requestID + detail.WorkerID = workerID + detail.DedupeKey = job.DedupeKey + detail.Summary = job.Summary + detail.Detail = job.Detail + detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters)) + if len(job.Labels) > 0 { + labels := make(map[string]string, len(job.Labels)) + for key, value := range job.Labels { + labels[key] = value + } + detail.Labels = labels + } else { + detail.Labels = nil + } + detail.State = trackedSnapshot.State + detail.Progress = trackedSnapshot.Progress + detail.Stage = trackedSnapshot.Stage + detail.Message = trackedSnapshot.Message + detail.Attempt = attempt + if detail.CreatedAt == nil || detail.CreatedAt.IsZero() { + detail.CreatedAt = trackedSnapshot.CreatedAt + } + detail.UpdatedAt = trackedSnapshot.UpdatedAt + }) + + r.appendActivity(JobActivity{ + JobID: job.JobId, + JobType: job.JobType, + RequestID: requestID, + WorkerID: workerID, + Source: "admin_dispatch", + Message: "job assigned", + Stage: "assigned", + OccurredAt: timeToPtr(now), + }) +} + +func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) { + if job == nil || strings.TrimSpace(job.JobId) == "" { + return + } + + now := time.Now().UTC() + + r.jobsMu.Lock() + tracked := r.jobs[job.JobId] + if tracked == nil { + tracked = &TrackedJob{ + JobID: job.JobId, + CreatedAt: timeToPtr(now), + } + r.jobs[job.JobId] = tracked + } + + tracked.JobType = job.JobType + tracked.DedupeKey = job.DedupeKey + tracked.Summary = job.Summary + tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_PENDING.String()) + tracked.Progress = 0 + tracked.Stage = "queued" + tracked.Message = "waiting for available executor" + if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() { + tracked.CreatedAt = timeToPtr(now) + } + tracked.UpdatedAt = timeToPtr(now) + trackedSnapshot := cloneTrackedJob(*tracked) + r.pruneTrackedJobsLocked() + r.dirtyJobs = true + r.jobsMu.Unlock() + r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) { + detail.JobID = job.JobId + detail.JobType = job.JobType + detail.DedupeKey = job.DedupeKey + detail.Summary = job.Summary + detail.Detail = job.Detail + detail.Parameters = enrichTrackedJobParameters(job.JobType, configValueMapToPlain(job.Parameters)) + if len(job.Labels) > 0 { + labels := make(map[string]string, len(job.Labels)) + for key, value := range job.Labels { + labels[key] = value + } + detail.Labels = labels + } else { + detail.Labels = nil + } + detail.State = trackedSnapshot.State + detail.Progress = trackedSnapshot.Progress + detail.Stage = trackedSnapshot.Stage + detail.Message = trackedSnapshot.Message + if detail.CreatedAt == nil || detail.CreatedAt.IsZero() { + detail.CreatedAt = trackedSnapshot.CreatedAt + } + detail.UpdatedAt = trackedSnapshot.UpdatedAt + }) + + r.appendActivity(JobActivity{ + JobID: job.JobId, + JobType: job.JobType, + Source: "admin_scheduler", + Message: "job queued for execution", + Stage: "queued", + OccurredAt: timeToPtr(now), + }) +} + +func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob { + if completed == nil || strings.TrimSpace(completed.JobId) == "" { + return nil + } + + now := time.Now().UTC() + if completed.CompletedAt != nil { + now = completed.CompletedAt.AsTime().UTC() + } + + r.jobsMu.Lock() + tracked := r.jobs[completed.JobId] + if tracked == nil { + tracked = &TrackedJob{ + JobID: completed.JobId, + CreatedAt: timeToPtr(now), + } + r.jobs[completed.JobId] = tracked + } + + if completed.JobType != "" { + tracked.JobType = completed.JobType + } + if completed.RequestId != "" { + tracked.RequestID = completed.RequestId + } + if completed.Success { + tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_SUCCEEDED.String()) + tracked.Progress = 100 + tracked.Stage = "completed" + if completed.Result != nil { + tracked.ResultSummary = completed.Result.Summary + } + tracked.Message = tracked.ResultSummary + if tracked.Message == "" { + tracked.Message = "completed" + } + tracked.ErrorMessage = "" + } else { + tracked.State = strings.ToLower(plugin_pb.JobState_JOB_STATE_FAILED.String()) + tracked.Stage = "failed" + tracked.ErrorMessage = completed.ErrorMessage + tracked.Message = completed.ErrorMessage + } + + tracked.UpdatedAt = timeToPtr(now) + tracked.CompletedAt = timeToPtr(now) + r.pruneTrackedJobsLocked() + clone := cloneTrackedJob(*tracked) + r.dirtyJobs = true + r.jobsMu.Unlock() + r.persistJobDetailSnapshot(completed.JobId, func(detail *TrackedJob) { + detail.JobID = completed.JobId + if completed.JobType != "" { + detail.JobType = completed.JobType + } + if completed.RequestId != "" { + detail.RequestID = completed.RequestId + } + detail.State = clone.State + detail.Progress = clone.Progress + detail.Stage = clone.Stage + detail.Message = clone.Message + detail.ErrorMessage = clone.ErrorMessage + detail.ResultSummary = clone.ResultSummary + if completed.Success && completed.Result != nil { + detail.ResultOutputValues = configValueMapToPlain(completed.Result.OutputValues) + } else { + detail.ResultOutputValues = nil + } + if detail.CreatedAt == nil || detail.CreatedAt.IsZero() { + detail.CreatedAt = clone.CreatedAt + } + if detail.UpdatedAt == nil || detail.UpdatedAt.IsZero() { + detail.UpdatedAt = clone.UpdatedAt + } + if detail.CompletedAt == nil || detail.CompletedAt.IsZero() { + detail.CompletedAt = clone.CompletedAt + } + }) + + r.appendActivity(JobActivity{ + JobID: completed.JobId, + JobType: completed.JobType, + RequestID: completed.RequestId, + WorkerID: clone.WorkerID, + Source: "worker_completion", + Message: clone.Message, + Stage: clone.Stage, + OccurredAt: timeToPtr(now), + }) + + return &clone +} + +func (r *Plugin) trackWorkerActivities(jobType, jobID, requestID, workerID string, events []*plugin_pb.ActivityEvent) { + if len(events) == 0 { + return + } + for _, event := range events { + if event == nil { + continue + } + timestamp := time.Now().UTC() + if event.CreatedAt != nil { + timestamp = event.CreatedAt.AsTime().UTC() + } + r.appendActivity(JobActivity{ + JobID: jobID, + JobType: jobType, + RequestID: requestID, + WorkerID: workerID, + Source: strings.ToLower(event.Source.String()), + Message: event.Message, + Stage: event.Stage, + Details: configValueMapToPlain(event.Details), + OccurredAt: timeToPtr(timestamp), + }) + } +} + +func (r *Plugin) appendActivity(activity JobActivity) { + if activity.OccurredAt == nil || activity.OccurredAt.IsZero() { + activity.OccurredAt = timeToPtr(time.Now().UTC()) + } + + r.activitiesMu.Lock() + r.activities = append(r.activities, activity) + if len(r.activities) > maxActivityRecords { + r.activities = r.activities[len(r.activities)-maxActivityRecords:] + } + r.dirtyActivities = true + r.activitiesMu.Unlock() +} + +func (r *Plugin) pruneTrackedJobsLocked() { + if len(r.jobs) <= maxTrackedJobsTotal { + return + } + + type sortableJob struct { + jobID string + updatedAt time.Time + } + terminalJobs := make([]sortableJob, 0) + for jobID, job := range r.jobs { + if job.State == StateSucceeded || + job.State == StateFailed || + job.State == StateCanceled { + updAt := time.Time{} + if job.UpdatedAt != nil { + updAt = *job.UpdatedAt + } + terminalJobs = append(terminalJobs, sortableJob{jobID, updAt}) + } + } + + if len(terminalJobs) == 0 { + return + } + + sort.Slice(terminalJobs, func(i, j int) bool { + return terminalJobs[i].updatedAt.Before(terminalJobs[j].updatedAt) + }) + + toDelete := len(r.jobs) - maxTrackedJobsTotal + if toDelete <= 0 { + return + } + if toDelete > len(terminalJobs) { + toDelete = len(terminalJobs) + } + + for i := 0; i < toDelete; i++ { + delete(r.jobs, terminalJobs[i].jobID) + } +} + +func configValueMapToPlain(values map[string]*plugin_pb.ConfigValue) map[string]interface{} { + if len(values) == 0 { + return nil + } + + payload, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(&plugin_pb.ValueMap{Fields: values}) + if err != nil { + return nil + } + + decoded := map[string]interface{}{} + if err := json.Unmarshal(payload, &decoded); err != nil { + return nil + } + + fields, ok := decoded["fields"].(map[string]interface{}) + if !ok { + return nil + } + return fields +} + +func (r *Plugin) persistTrackedJobsSnapshot() { + r.jobsMu.Lock() + r.dirtyJobs = false + jobs := make([]TrackedJob, 0, len(r.jobs)) + for _, job := range r.jobs { + if job == nil || strings.TrimSpace(job.JobID) == "" { + continue + } + clone := cloneTrackedJob(*job) + stripTrackedJobDetailFields(&clone) + jobs = append(jobs, clone) + } + r.jobsMu.Unlock() + + if len(jobs) == 0 { + return + } + + sort.Slice(jobs, func(i, j int) bool { + ti := time.Time{} + if jobs[i].UpdatedAt != nil { + ti = *jobs[i].UpdatedAt + } + tj := time.Time{} + if jobs[j].UpdatedAt != nil { + tj = *jobs[j].UpdatedAt + } + if !ti.Equal(tj) { + return ti.After(tj) + } + return jobs[i].JobID < jobs[j].JobID + }) + if len(jobs) > maxTrackedJobsTotal { + jobs = jobs[:maxTrackedJobsTotal] + } + + if err := r.store.SaveTrackedJobs(jobs); err != nil { + glog.Warningf("Plugin failed to persist tracked jobs: %v", err) + } +} + +func (r *Plugin) persistJobDetailSnapshot(jobID string, apply func(detail *TrackedJob)) { + normalizedJobID, _ := sanitizeJobID(jobID) + if normalizedJobID == "" { + return + } + + r.jobDetailsMu.Lock() + defer r.jobDetailsMu.Unlock() + + detail, err := r.store.LoadJobDetail(normalizedJobID) + if err != nil { + glog.Warningf("Plugin failed to load job detail snapshot for %s: %v", normalizedJobID, err) + return + } + if detail == nil { + detail = &TrackedJob{ + JobID: normalizedJobID, + } + } + + if apply != nil { + apply(detail) + } + + if err := r.store.SaveJobDetail(*detail); err != nil { + glog.Warningf("Plugin failed to persist job detail snapshot for %s: %v", normalizedJobID, err) + } +} + +func (r *Plugin) persistActivitiesSnapshot() { + r.activitiesMu.Lock() + r.dirtyActivities = false + activities := append([]JobActivity(nil), r.activities...) + r.activitiesMu.Unlock() + + if len(activities) == 0 { + return + } + + if len(activities) > maxActivityRecords { + activities = activities[len(activities)-maxActivityRecords:] + } + + if err := r.store.SaveActivities(activities); err != nil { + glog.Warningf("Plugin failed to persist activities: %v", err) + } +} + +func (r *Plugin) persistenceLoop() { + defer r.wg.Done() + for { + select { + case <-r.shutdownCh: + r.persistTrackedJobsSnapshot() + r.persistActivitiesSnapshot() + return + case <-r.persistTicker.C: + r.jobsMu.RLock() + needsJobsFlush := r.dirtyJobs + r.jobsMu.RUnlock() + if needsJobsFlush { + r.persistTrackedJobsSnapshot() + } + + r.activitiesMu.RLock() + needsActivitiesFlush := r.dirtyActivities + r.activitiesMu.RUnlock() + if needsActivitiesFlush { + r.persistActivitiesSnapshot() + } + } + } +} diff --git a/weed/admin/plugin/plugin_monitor_test.go b/weed/admin/plugin/plugin_monitor_test.go new file mode 100644 index 000000000..09281257f --- /dev/null +++ b/weed/admin/plugin/plugin_monitor_test.go @@ -0,0 +1,600 @@ +package plugin + +import ( + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/worker_pb" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestPluginLoadsPersistedMonitorStateOnStart(t *testing.T) { + t.Parallel() + + dataDir := t.TempDir() + store, err := NewConfigStore(dataDir) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + seedJobs := []TrackedJob{ + { + JobID: "job-seeded", + JobType: "vacuum", + State: "running", + CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)), + UpdatedAt: timeToPtr(time.Now().UTC().Add(-1 * time.Minute)), + }, + } + seedActivities := []JobActivity{ + { + JobID: "job-seeded", + JobType: "vacuum", + Source: "worker_progress", + Message: "seeded", + OccurredAt: timeToPtr(time.Now().UTC().Add(-30 * time.Second)), + }, + } + + if err := store.SaveTrackedJobs(seedJobs); err != nil { + t.Fatalf("SaveTrackedJobs: %v", err) + } + if err := store.SaveActivities(seedActivities); err != nil { + t.Fatalf("SaveActivities: %v", err) + } + + pluginSvc, err := New(Options{DataDir: dataDir}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + gotJobs := pluginSvc.ListTrackedJobs("", "", 0) + if len(gotJobs) != 1 || gotJobs[0].JobID != "job-seeded" { + t.Fatalf("unexpected loaded jobs: %+v", gotJobs) + } + + gotActivities := pluginSvc.ListActivities("", 0) + if len(gotActivities) != 1 || gotActivities[0].Message != "seeded" { + t.Fatalf("unexpected loaded activities: %+v", gotActivities) + } +} + +func TestPluginPersistsMonitorStateAfterJobUpdates(t *testing.T) { + t.Parallel() + + dataDir := t.TempDir() + pluginSvc, err := New(Options{DataDir: dataDir}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + job := &plugin_pb.JobSpec{ + JobId: "job-persist", + JobType: "vacuum", + Summary: "persist test", + } + pluginSvc.trackExecutionStart("req-persist", "worker-a", job, 1) + + pluginSvc.trackExecutionCompletion(&plugin_pb.JobCompleted{ + RequestId: "req-persist", + JobId: "job-persist", + JobType: "vacuum", + Success: true, + Result: &plugin_pb.JobResult{Summary: "done"}, + CompletedAt: timestamppb.New(time.Now().UTC()), + }) + pluginSvc.Shutdown() + + store, err := NewConfigStore(dataDir) + if err != nil { + t.Fatalf("NewConfigStore: %v", err) + } + + trackedJobs, err := store.LoadTrackedJobs() + if err != nil { + t.Fatalf("LoadTrackedJobs: %v", err) + } + if len(trackedJobs) == 0 { + t.Fatalf("expected persisted tracked jobs") + } + + found := false + for _, tracked := range trackedJobs { + if tracked.JobID == "job-persist" { + found = true + if tracked.State == "" { + t.Fatalf("persisted job state should not be empty") + } + } + } + if !found { + t.Fatalf("persisted tracked jobs missing job-persist") + } + + activities, err := store.LoadActivities() + if err != nil { + t.Fatalf("LoadActivities: %v", err) + } + if len(activities) == 0 { + t.Fatalf("expected persisted activities") + } +} + +func TestTrackExecutionQueuedMarksPendingState(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{ + JobId: "job-pending-1", + JobType: "vacuum", + DedupeKey: "vacuum:1", + Summary: "pending queue item", + }) + + jobs := pluginSvc.ListTrackedJobs("vacuum", "", 10) + if len(jobs) != 1 { + t.Fatalf("expected one tracked pending job, got=%d", len(jobs)) + } + job := jobs[0] + if job.JobID != "job-pending-1" { + t.Fatalf("unexpected pending job id: %s", job.JobID) + } + if job.State != "job_state_pending" { + t.Fatalf("unexpected pending job state: %s", job.State) + } + if job.Stage != "queued" { + t.Fatalf("unexpected pending job stage: %s", job.Stage) + } + + activities := pluginSvc.ListActivities("vacuum", 50) + found := false + for _, activity := range activities { + if activity.JobID == "job-pending-1" && activity.Stage == "queued" && activity.Source == "admin_scheduler" { + found = true + break + } + } + if !found { + t.Fatalf("expected queued activity for pending job") + } +} + +func TestHandleJobProgressUpdateCarriesWorkerIDInActivities(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + job := &plugin_pb.JobSpec{ + JobId: "job-progress-worker", + JobType: "vacuum", + } + pluginSvc.trackExecutionStart("req-progress-worker", "worker-a", job, 1) + + pluginSvc.handleJobProgressUpdate("worker-a", &plugin_pb.JobProgressUpdate{ + RequestId: "req-progress-worker", + JobId: "job-progress-worker", + JobType: "vacuum", + State: plugin_pb.JobState_JOB_STATE_RUNNING, + ProgressPercent: 42.0, + Stage: "scan", + Message: "in progress", + Activities: []*plugin_pb.ActivityEvent{ + { + Source: plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR, + Message: "volume scanned", + Stage: "scan", + }, + }, + }) + + activities := pluginSvc.ListActivities("vacuum", 0) + if len(activities) == 0 { + t.Fatalf("expected activity entries") + } + + foundProgress := false + foundEvent := false + for _, activity := range activities { + if activity.Source == "worker_progress" && activity.Message == "in progress" { + foundProgress = true + if activity.WorkerID != "worker-a" { + t.Fatalf("worker_progress activity worker mismatch: got=%q want=%q", activity.WorkerID, "worker-a") + } + } + if activity.Message == "volume scanned" { + foundEvent = true + if activity.WorkerID != "worker-a" { + t.Fatalf("worker event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-a") + } + } + } + + if !foundProgress { + t.Fatalf("expected worker_progress activity") + } + if !foundEvent { + t.Fatalf("expected worker activity event") + } +} + +func TestHandleJobProgressUpdateWithoutJobIDTracksDetectionActivities(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.handleJobProgressUpdate("worker-detector", &plugin_pb.JobProgressUpdate{ + RequestId: "detect-req-1", + JobType: "vacuum", + State: plugin_pb.JobState_JOB_STATE_RUNNING, + Stage: "decision_summary", + Message: "VACUUM: No tasks created for 3 volumes", + Activities: []*plugin_pb.ActivityEvent{ + { + Source: plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR, + Stage: "decision_summary", + Message: "VACUUM: No tasks created for 3 volumes", + }, + }, + }) + + activities := pluginSvc.ListActivities("vacuum", 0) + if len(activities) == 0 { + t.Fatalf("expected activity entries") + } + + foundDetectionProgress := false + foundDetectorEvent := false + for _, activity := range activities { + if activity.RequestID != "detect-req-1" { + continue + } + if activity.Source == "worker_detection" { + foundDetectionProgress = true + if activity.WorkerID != "worker-detector" { + t.Fatalf("worker_detection worker mismatch: got=%q want=%q", activity.WorkerID, "worker-detector") + } + } + if activity.Source == "activity_source_detector" { + foundDetectorEvent = true + if activity.WorkerID != "worker-detector" { + t.Fatalf("detector event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-detector") + } + } + } + + if !foundDetectionProgress { + t.Fatalf("expected worker_detection activity") + } + if !foundDetectorEvent { + t.Fatalf("expected detector activity event") + } +} + +func TestHandleJobCompletedCarriesWorkerIDInActivitiesAndRunHistory(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + job := &plugin_pb.JobSpec{ + JobId: "job-complete-worker", + JobType: "vacuum", + } + pluginSvc.trackExecutionStart("req-complete-worker", "worker-b", job, 1) + + pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{ + RequestId: "req-complete-worker", + JobId: "job-complete-worker", + JobType: "vacuum", + Success: true, + Activities: []*plugin_pb.ActivityEvent{ + { + Source: plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR, + Message: "finalizer done", + Stage: "finalize", + }, + }, + CompletedAt: timestamppb.Now(), + }) + pluginSvc.Shutdown() + + activities := pluginSvc.ListActivities("vacuum", 0) + foundWorkerEvent := false + for _, activity := range activities { + if activity.Message == "finalizer done" { + foundWorkerEvent = true + if activity.WorkerID != "worker-b" { + t.Fatalf("worker completion event worker mismatch: got=%q want=%q", activity.WorkerID, "worker-b") + } + } + } + if !foundWorkerEvent { + t.Fatalf("expected completion worker event activity") + } + + history, err := pluginSvc.LoadRunHistory("vacuum") + if err != nil { + t.Fatalf("LoadRunHistory: %v", err) + } + if history == nil || len(history.SuccessfulRuns) == 0 { + t.Fatalf("expected successful run history entry") + } + if history.SuccessfulRuns[0].WorkerID != "worker-b" { + t.Fatalf("run history worker mismatch: got=%q want=%q", history.SuccessfulRuns[0].WorkerID, "worker-b") + } +} + +func TestTrackExecutionStartStoresJobPayloadDetails(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.trackExecutionStart("req-payload", "worker-c", &plugin_pb.JobSpec{ + JobId: "job-payload", + JobType: "vacuum", + Summary: "payload summary", + Detail: "payload detail", + Parameters: map[string]*plugin_pb.ConfigValue{ + "volume_id": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 9}, + }, + }, + Labels: map[string]string{ + "source": "detector", + }, + }, 2) + pluginSvc.Shutdown() + + job, found := pluginSvc.GetTrackedJob("job-payload") + if !found || job == nil { + t.Fatalf("expected tracked job") + } + if job.Detail != "" { + t.Fatalf("expected in-memory tracked job detail to be stripped, got=%q", job.Detail) + } + if job.Attempt != 2 { + t.Fatalf("unexpected attempt: %d", job.Attempt) + } + if len(job.Labels) != 0 { + t.Fatalf("expected in-memory labels to be stripped, got=%+v", job.Labels) + } + if len(job.Parameters) != 0 { + t.Fatalf("expected in-memory parameters to be stripped, got=%+v", job.Parameters) + } + + detail, found, err := pluginSvc.BuildJobDetail("job-payload", 100, 0) + if err != nil { + t.Fatalf("BuildJobDetail: %v", err) + } + if !found || detail == nil || detail.Job == nil { + t.Fatalf("expected disk-backed job detail") + } + if detail.Job.Detail != "payload detail" { + t.Fatalf("unexpected disk-backed detail: %q", detail.Job.Detail) + } + if got := detail.Job.Labels["source"]; got != "detector" { + t.Fatalf("unexpected disk-backed label source: %q", got) + } + if got, ok := detail.Job.Parameters["volume_id"].(map[string]interface{}); !ok || got["int64_value"] != "9" { + t.Fatalf("unexpected disk-backed parameters payload: %#v", detail.Job.Parameters["volume_id"]) + } +} + +func TestTrackExecutionStartStoresErasureCodingExecutionPlan(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + taskParams := &worker_pb.TaskParams{ + TaskId: "task-ec-1", + VolumeId: 29, + Collection: "photos", + Sources: []*worker_pb.TaskSource{ + { + Node: "source-a:8080", + DataCenter: "dc1", + Rack: "rack1", + VolumeId: 29, + }, + }, + Targets: []*worker_pb.TaskTarget{ + { + Node: "target-a:8080", + DataCenter: "dc1", + Rack: "rack2", + VolumeId: 29, + ShardIds: []uint32{0, 10}, + }, + { + Node: "target-b:8080", + DataCenter: "dc2", + Rack: "rack3", + VolumeId: 29, + ShardIds: []uint32{1, 11}, + }, + }, + TaskParams: &worker_pb.TaskParams_ErasureCodingParams{ + ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{ + DataShards: 10, + ParityShards: 4, + }, + }, + } + payload, err := proto.Marshal(taskParams) + if err != nil { + t.Fatalf("Marshal task params: %v", err) + } + + pluginSvc.trackExecutionStart("req-ec-plan", "worker-ec", &plugin_pb.JobSpec{ + JobId: "job-ec-plan", + JobType: "erasure_coding", + Parameters: map[string]*plugin_pb.ConfigValue{ + "task_params_pb": { + Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}, + }, + }, + }, 1) + pluginSvc.Shutdown() + + detail, found, err := pluginSvc.BuildJobDetail("job-ec-plan", 100, 0) + if err != nil { + t.Fatalf("BuildJobDetail: %v", err) + } + if !found || detail == nil || detail.Job == nil { + t.Fatalf("expected disk-backed detail") + } + + rawPlan, ok := detail.Job.Parameters["execution_plan"] + if !ok { + t.Fatalf("expected execution_plan in parameters, got=%+v", detail.Job.Parameters) + } + plan, ok := rawPlan.(map[string]interface{}) + if !ok { + t.Fatalf("unexpected execution_plan type: %T", rawPlan) + } + if plan["job_type"] != "erasure_coding" { + t.Fatalf("unexpected execution plan job type: %+v", plan["job_type"]) + } + if plan["volume_id"] != float64(29) { + t.Fatalf("unexpected execution plan volume id: %+v", plan["volume_id"]) + } + targets, ok := plan["targets"].([]interface{}) + if !ok || len(targets) != 2 { + t.Fatalf("unexpected targets in execution plan: %+v", plan["targets"]) + } + assignments, ok := plan["shard_assignments"].([]interface{}) + if !ok || len(assignments) != 4 { + t.Fatalf("unexpected shard assignments in execution plan: %+v", plan["shard_assignments"]) + } + firstAssignment, ok := assignments[0].(map[string]interface{}) + if !ok { + t.Fatalf("unexpected first assignment payload: %+v", assignments[0]) + } + if firstAssignment["shard_id"] != float64(0) || firstAssignment["kind"] != "data" { + t.Fatalf("unexpected first assignment: %+v", firstAssignment) + } +} + +func TestBuildJobDetailIncludesActivitiesAndRunRecord(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.trackExecutionStart("req-detail", "worker-z", &plugin_pb.JobSpec{ + JobId: "job-detail", + JobType: "vacuum", + Summary: "detail summary", + }, 1) + pluginSvc.handleJobProgressUpdate("worker-z", &plugin_pb.JobProgressUpdate{ + RequestId: "req-detail", + JobId: "job-detail", + JobType: "vacuum", + State: plugin_pb.JobState_JOB_STATE_RUNNING, + Stage: "scan", + Message: "scanning volume", + }) + pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{ + RequestId: "req-detail", + JobId: "job-detail", + JobType: "vacuum", + Success: true, + Result: &plugin_pb.JobResult{ + Summary: "done", + OutputValues: map[string]*plugin_pb.ConfigValue{ + "affected": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}, + }, + }, + }, + CompletedAt: timestamppb.Now(), + }) + pluginSvc.Shutdown() + + detail, found, err := pluginSvc.BuildJobDetail("job-detail", 100, 5) + if err != nil { + t.Fatalf("BuildJobDetail error: %v", err) + } + if !found || detail == nil { + t.Fatalf("expected job detail") + } + if detail.Job == nil || detail.Job.JobID != "job-detail" { + t.Fatalf("unexpected job detail payload: %+v", detail.Job) + } + if detail.RunRecord == nil || detail.RunRecord.JobID != "job-detail" { + t.Fatalf("expected run record for job-detail, got=%+v", detail.RunRecord) + } + if len(detail.Activities) == 0 { + t.Fatalf("expected activity timeline entries") + } + if detail.Job.ResultOutputValues == nil { + t.Fatalf("expected result output values") + } +} + +func TestBuildJobDetailLoadsFromDiskWhenMemoryCleared(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.trackExecutionStart("req-disk", "worker-d", &plugin_pb.JobSpec{ + JobId: "job-disk", + JobType: "vacuum", + Summary: "disk summary", + Detail: "disk detail payload", + }, 1) + pluginSvc.Shutdown() + + pluginSvc.jobsMu.Lock() + pluginSvc.jobs = map[string]*TrackedJob{} + pluginSvc.jobsMu.Unlock() + pluginSvc.activitiesMu.Lock() + pluginSvc.activities = nil + pluginSvc.activitiesMu.Unlock() + + detail, found, err := pluginSvc.BuildJobDetail("job-disk", 100, 0) + if err != nil { + t.Fatalf("BuildJobDetail: %v", err) + } + if !found || detail == nil || detail.Job == nil { + t.Fatalf("expected detail from disk") + } + if detail.Job.Detail != "disk detail payload" { + t.Fatalf("unexpected disk detail payload: %q", detail.Job.Detail) + } +} diff --git a/weed/admin/plugin/plugin_scheduler.go b/weed/admin/plugin/plugin_scheduler.go new file mode 100644 index 000000000..5951fc5fe --- /dev/null +++ b/weed/admin/plugin/plugin_scheduler.go @@ -0,0 +1,945 @@ +package plugin + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var errExecutorAtCapacity = errors.New("executor is at capacity") + +const ( + defaultSchedulerTick = 5 * time.Second + defaultScheduledDetectionInterval = 300 * time.Second + defaultScheduledDetectionTimeout = 45 * time.Second + defaultScheduledExecutionTimeout = 90 * time.Second + defaultScheduledMaxResults int32 = 1000 + defaultScheduledExecutionConcurrency = 1 + defaultScheduledPerWorkerConcurrency = 1 + maxScheduledExecutionConcurrency = 128 + defaultScheduledRetryBackoff = 5 * time.Second + defaultClusterContextTimeout = 10 * time.Second + defaultWaitingBacklogFloor = 8 + defaultWaitingBacklogMultiplier = 4 +) + +type schedulerPolicy struct { + DetectionInterval time.Duration + DetectionTimeout time.Duration + ExecutionTimeout time.Duration + RetryBackoff time.Duration + MaxResults int32 + ExecutionConcurrency int + PerWorkerConcurrency int + RetryLimit int + ExecutorReserveBackoff time.Duration +} + +func (r *Plugin) schedulerLoop() { + defer r.wg.Done() + ticker := time.NewTicker(r.schedulerTick) + defer ticker.Stop() + + // Try once immediately on startup. + r.runSchedulerTick() + + for { + select { + case <-r.shutdownCh: + return + case <-ticker.C: + r.runSchedulerTick() + } + } +} + +func (r *Plugin) runSchedulerTick() { + jobTypes := r.registry.DetectableJobTypes() + if len(jobTypes) == 0 { + return + } + + active := make(map[string]struct{}, len(jobTypes)) + for _, jobType := range jobTypes { + active[jobType] = struct{}{} + + policy, enabled, err := r.loadSchedulerPolicy(jobType) + if err != nil { + glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err) + continue + } + if !enabled { + r.clearSchedulerJobType(jobType) + continue + } + + if !r.markDetectionDue(jobType, policy.DetectionInterval) { + continue + } + + r.wg.Add(1) + go func(jt string, p schedulerPolicy) { + defer r.wg.Done() + r.runScheduledDetection(jt, p) + }(jobType, policy) + } + + r.pruneSchedulerState(active) + r.pruneDetectorLeases(active) +} + +func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) { + cfg, err := r.store.LoadJobTypeConfig(jobType) + if err != nil { + return schedulerPolicy{}, false, err + } + descriptor, err := r.store.LoadDescriptor(jobType) + if err != nil { + return schedulerPolicy{}, false, err + } + + adminRuntime := deriveSchedulerAdminRuntime(cfg, descriptor) + if adminRuntime == nil { + return schedulerPolicy{}, false, nil + } + if !adminRuntime.Enabled { + return schedulerPolicy{}, false, nil + } + + policy := schedulerPolicy{ + DetectionInterval: durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval), + DetectionTimeout: durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout), + ExecutionTimeout: defaultScheduledExecutionTimeout, + RetryBackoff: durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff), + MaxResults: adminRuntime.MaxJobsPerDetection, + ExecutionConcurrency: int(adminRuntime.GlobalExecutionConcurrency), + PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency), + RetryLimit: int(adminRuntime.RetryLimit), + ExecutorReserveBackoff: 200 * time.Millisecond, + } + + if policy.DetectionInterval < r.schedulerTick { + policy.DetectionInterval = r.schedulerTick + } + if policy.MaxResults <= 0 { + policy.MaxResults = defaultScheduledMaxResults + } + if policy.ExecutionConcurrency <= 0 { + policy.ExecutionConcurrency = defaultScheduledExecutionConcurrency + } + if policy.ExecutionConcurrency > maxScheduledExecutionConcurrency { + policy.ExecutionConcurrency = maxScheduledExecutionConcurrency + } + if policy.PerWorkerConcurrency <= 0 { + policy.PerWorkerConcurrency = defaultScheduledPerWorkerConcurrency + } + if policy.PerWorkerConcurrency > policy.ExecutionConcurrency { + policy.PerWorkerConcurrency = policy.ExecutionConcurrency + } + if policy.RetryLimit < 0 { + policy.RetryLimit = 0 + } + + // Plugin protocol currently has only detection timeout in admin settings. + execTimeout := time.Duration(adminRuntime.DetectionTimeoutSeconds*2) * time.Second + if execTimeout < defaultScheduledExecutionTimeout { + execTimeout = defaultScheduledExecutionTimeout + } + policy.ExecutionTimeout = execTimeout + + return policy, true, nil +} + +func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) { + jobTypes, err := r.ListKnownJobTypes() + if err != nil { + return nil, err + } + + r.schedulerMu.Lock() + nextDetectionAt := make(map[string]time.Time, len(r.nextDetectionAt)) + for jobType, nextRun := range r.nextDetectionAt { + nextDetectionAt[jobType] = nextRun + } + detectionInFlight := make(map[string]bool, len(r.detectionInFlight)) + for jobType, inFlight := range r.detectionInFlight { + detectionInFlight[jobType] = inFlight + } + r.schedulerMu.Unlock() + + states := make([]SchedulerJobTypeState, 0, len(jobTypes)) + for _, jobType := range jobTypes { + state := SchedulerJobTypeState{ + JobType: jobType, + DetectionInFlight: detectionInFlight[jobType], + } + + if nextRun, ok := nextDetectionAt[jobType]; ok && !nextRun.IsZero() { + nextRunUTC := nextRun.UTC() + state.NextDetectionAt = &nextRunUTC + } + + policy, enabled, loadErr := r.loadSchedulerPolicy(jobType) + if loadErr != nil { + state.PolicyError = loadErr.Error() + } else { + state.Enabled = enabled + if enabled { + state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval) + state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout) + state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout) + state.MaxJobsPerDetection = policy.MaxResults + state.GlobalExecutionConcurrency = policy.ExecutionConcurrency + state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency + state.RetryLimit = policy.RetryLimit + state.RetryBackoffSeconds = secondsFromDuration(policy.RetryBackoff) + } + } + + leasedWorkerID := r.getDetectorLease(jobType) + if leasedWorkerID != "" { + state.DetectorWorkerID = leasedWorkerID + if worker, ok := r.registry.Get(leasedWorkerID); ok { + if capability := worker.Capabilities[jobType]; capability != nil && capability.CanDetect { + state.DetectorAvailable = true + } + } + } + if state.DetectorWorkerID == "" { + detector, detectorErr := r.registry.PickDetector(jobType) + if detectorErr == nil && detector != nil { + state.DetectorAvailable = true + state.DetectorWorkerID = detector.WorkerID + } + } + + executors, executorErr := r.registry.ListExecutors(jobType) + if executorErr == nil { + state.ExecutorWorkerCount = len(executors) + } + + states = append(states, state) + } + + return states, nil +} + +func deriveSchedulerAdminRuntime( + cfg *plugin_pb.PersistedJobTypeConfig, + descriptor *plugin_pb.JobTypeDescriptor, +) *plugin_pb.AdminRuntimeConfig { + if cfg != nil && cfg.AdminRuntime != nil { + adminConfig := *cfg.AdminRuntime + return &adminConfig + } + + if descriptor == nil || descriptor.AdminRuntimeDefaults == nil { + return nil + } + + defaults := descriptor.AdminRuntimeDefaults + return &plugin_pb.AdminRuntimeConfig{ + Enabled: defaults.Enabled, + DetectionIntervalSeconds: defaults.DetectionIntervalSeconds, + DetectionTimeoutSeconds: defaults.DetectionTimeoutSeconds, + MaxJobsPerDetection: defaults.MaxJobsPerDetection, + GlobalExecutionConcurrency: defaults.GlobalExecutionConcurrency, + PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency, + RetryLimit: defaults.RetryLimit, + RetryBackoffSeconds: defaults.RetryBackoffSeconds, + } +} + +func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool { + now := time.Now().UTC() + + r.schedulerMu.Lock() + defer r.schedulerMu.Unlock() + + if r.detectionInFlight[jobType] { + return false + } + + nextRun, exists := r.nextDetectionAt[jobType] + if exists && now.Before(nextRun) { + return false + } + + r.nextDetectionAt[jobType] = now.Add(interval) + r.detectionInFlight[jobType] = true + return true +} + +func (r *Plugin) finishDetection(jobType string) { + r.schedulerMu.Lock() + delete(r.detectionInFlight, jobType) + r.schedulerMu.Unlock() +} + +func (r *Plugin) pruneSchedulerState(activeJobTypes map[string]struct{}) { + r.schedulerMu.Lock() + defer r.schedulerMu.Unlock() + + for jobType := range r.nextDetectionAt { + if _, ok := activeJobTypes[jobType]; !ok { + delete(r.nextDetectionAt, jobType) + delete(r.detectionInFlight, jobType) + } + } +} + +func (r *Plugin) clearSchedulerJobType(jobType string) { + r.schedulerMu.Lock() + delete(r.nextDetectionAt, jobType) + delete(r.detectionInFlight, jobType) + r.schedulerMu.Unlock() + r.clearDetectorLease(jobType, "") +} + +func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) { + r.detectorLeaseMu.Lock() + defer r.detectorLeaseMu.Unlock() + + for jobType := range r.detectorLeases { + if _, ok := activeJobTypes[jobType]; !ok { + delete(r.detectorLeases, jobType) + } + } +} + +func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) { + defer r.finishDetection(jobType) + + start := time.Now().UTC() + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: "scheduled detection started", + Stage: "detecting", + OccurredAt: timeToPtr(start), + }) + + if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip { + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold), + Stage: "skipped_waiting_backlog", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + return + } + + clusterContext, err := r.loadSchedulerClusterContext() + if err != nil { + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection aborted: %v", err), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout) + proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults) + cancel() + if err != nil { + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection failed: %v", err), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + return + } + + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)), + Stage: "detected", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + + filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals) + if skippedActive > 0 { + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive), + Stage: "deduped_active_jobs", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + } + + if len(filteredByActive) == 0 { + return + } + + filtered := r.filterScheduledProposals(filteredByActive) + if len(filtered) != len(filteredByActive) { + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)), + Stage: "deduped", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + } + + if len(filtered) == 0 { + return + } + + r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy) +} + +func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) { + if r.clusterContextProvider == nil { + return nil, fmt.Errorf("cluster context provider is not configured") + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterContextTimeout) + defer cancel() + + clusterContext, err := r.clusterContextProvider(ctx) + if err != nil { + return nil, err + } + if clusterContext == nil { + return nil, fmt.Errorf("cluster context provider returned nil") + } + return clusterContext, nil +} + +func (r *Plugin) dispatchScheduledProposals( + jobType string, + proposals []*plugin_pb.JobProposal, + clusterContext *plugin_pb.ClusterContext, + policy schedulerPolicy, +) { + jobQueue := make(chan *plugin_pb.JobSpec, len(proposals)) + for index, proposal := range proposals { + job := buildScheduledJobSpec(jobType, proposal, index) + r.trackExecutionQueued(job) + select { + case <-r.shutdownCh: + close(jobQueue) + return + default: + jobQueue <- job + } + } + close(jobQueue) + + var wg sync.WaitGroup + var statsMu sync.Mutex + successCount := 0 + errorCount := 0 + + workerCount := policy.ExecutionConcurrency + if workerCount < 1 { + workerCount = 1 + } + + for i := 0; i < workerCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + for job := range jobQueue { + select { + case <-r.shutdownCh: + return + default: + } + + for { + select { + case <-r.shutdownCh: + return + default: + } + + executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy) + if reserveErr != nil { + select { + case <-r.shutdownCh: + return + default: + } + statsMu.Lock() + errorCount++ + statsMu.Unlock() + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + break + } + + err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy) + release() + if errors.Is(err, errExecutorAtCapacity) { + r.trackExecutionQueued(job) + if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) { + return + } + continue + } + if err != nil { + statsMu.Lock() + errorCount++ + statsMu.Unlock() + r.appendActivity(JobActivity{ + JobID: job.JobId, + JobType: job.JobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled execution failed: %v", err), + Stage: "failed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + break + } + + statsMu.Lock() + successCount++ + statsMu.Unlock() + break + } + } + }() + } + + wg.Wait() + + r.appendActivity(JobActivity{ + JobType: jobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("scheduled execution finished: success=%d error=%d", successCount, errorCount), + Stage: "executed", + OccurredAt: timeToPtr(time.Now().UTC()), + }) +} + +func (r *Plugin) reserveScheduledExecutor( + jobType string, + policy schedulerPolicy, +) (*WorkerSession, func(), error) { + deadline := time.Now().Add(policy.ExecutionTimeout) + if policy.ExecutionTimeout <= 0 { + deadline = time.Now().Add(10 * time.Minute) // Default cap + } + + for { + select { + case <-r.shutdownCh: + return nil, nil, fmt.Errorf("plugin is shutting down") + default: + } + + if time.Now().After(deadline) { + return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType) + } + + executors, err := r.registry.ListExecutors(jobType) + if err != nil { + if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) { + return nil, nil, fmt.Errorf("plugin is shutting down") + } + continue + } + + for _, executor := range executors { + release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy) + if !ok { + continue + } + return executor, release, nil + } + + if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) { + return nil, nil, fmt.Errorf("plugin is shutting down") + } + } +} + +func (r *Plugin) tryReserveExecutorCapacity( + executor *WorkerSession, + jobType string, + policy schedulerPolicy, +) (func(), bool) { + if executor == nil || strings.TrimSpace(executor.WorkerID) == "" { + return nil, false + } + + limit := schedulerWorkerExecutionLimit(executor, jobType, policy) + if limit <= 0 { + return nil, false + } + heartbeatUsed := 0 + if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 { + heartbeatUsed = int(executor.Heartbeat.ExecutionSlotsUsed) + } + + workerID := strings.TrimSpace(executor.WorkerID) + + r.schedulerExecMu.Lock() + reserved := r.schedulerExecReservations[workerID] + if heartbeatUsed+reserved >= limit { + r.schedulerExecMu.Unlock() + return nil, false + } + r.schedulerExecReservations[workerID] = reserved + 1 + r.schedulerExecMu.Unlock() + + release := func() { + r.releaseExecutorCapacity(workerID) + } + return release, true +} + +func (r *Plugin) releaseExecutorCapacity(workerID string) { + workerID = strings.TrimSpace(workerID) + if workerID == "" { + return + } + + r.schedulerExecMu.Lock() + defer r.schedulerExecMu.Unlock() + + current := r.schedulerExecReservations[workerID] + if current <= 1 { + delete(r.schedulerExecReservations, workerID) + return + } + r.schedulerExecReservations[workerID] = current - 1 +} + +func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, policy schedulerPolicy) int { + limit := policy.PerWorkerConcurrency + if limit <= 0 { + limit = defaultScheduledPerWorkerConcurrency + } + + if capability := executor.Capabilities[jobType]; capability != nil && capability.MaxExecutionConcurrency > 0 { + capLimit := int(capability.MaxExecutionConcurrency) + if capLimit < limit { + limit = capLimit + } + } + + if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsTotal > 0 { + heartbeatLimit := int(executor.Heartbeat.ExecutionSlotsTotal) + if heartbeatLimit < limit { + limit = heartbeatLimit + } + } + + if limit < 0 { + return 0 + } + return limit +} + +func (r *Plugin) executeScheduledJobWithExecutor( + executor *WorkerSession, + job *plugin_pb.JobSpec, + clusterContext *plugin_pb.ClusterContext, + policy schedulerPolicy, +) error { + maxAttempts := policy.RetryLimit + 1 + if maxAttempts < 1 { + maxAttempts = 1 + } + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + select { + case <-r.shutdownCh: + return fmt.Errorf("plugin is shutting down") + default: + } + + execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout) + _, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt)) + cancel() + if err == nil { + return nil + } + if isExecutorAtCapacityError(err) { + return errExecutorAtCapacity + } + lastErr = err + + if attempt < maxAttempts { + r.appendActivity(JobActivity{ + JobID: job.JobId, + JobType: job.JobType, + Source: "admin_scheduler", + Message: fmt.Sprintf("retrying job attempt %d/%d after error: %v", attempt, maxAttempts, err), + Stage: "retry", + OccurredAt: timeToPtr(time.Now().UTC()), + }) + if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) { + return fmt.Errorf("plugin is shutting down") + } + } + } + + if lastErr == nil { + lastErr = fmt.Errorf("execution failed without an explicit error") + } + return lastErr +} + +func (r *Plugin) shouldSkipDetectionForWaitingJobs(jobType string, policy schedulerPolicy) (bool, int, int) { + waitingCount := r.countWaitingTrackedJobs(jobType) + threshold := waitingBacklogThreshold(policy) + if threshold <= 0 { + return false, waitingCount, threshold + } + return waitingCount >= threshold, waitingCount, threshold +} + +func (r *Plugin) countWaitingTrackedJobs(jobType string) int { + normalizedJobType := strings.TrimSpace(jobType) + if normalizedJobType == "" { + return 0 + } + + waiting := 0 + r.jobsMu.RLock() + for _, job := range r.jobs { + if job == nil { + continue + } + if strings.TrimSpace(job.JobType) != normalizedJobType { + continue + } + if !isWaitingTrackedJobState(job.State) { + continue + } + waiting++ + } + r.jobsMu.RUnlock() + + return waiting +} + +func waitingBacklogThreshold(policy schedulerPolicy) int { + concurrency := policy.ExecutionConcurrency + if concurrency <= 0 { + concurrency = defaultScheduledExecutionConcurrency + } + threshold := concurrency * defaultWaitingBacklogMultiplier + if threshold < defaultWaitingBacklogFloor { + threshold = defaultWaitingBacklogFloor + } + if policy.MaxResults > 0 && threshold > int(policy.MaxResults) { + threshold = int(policy.MaxResults) + } + return threshold +} + +func isExecutorAtCapacityError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, errExecutorAtCapacity) { + return true + } + return strings.Contains(strings.ToLower(err.Error()), "executor is at capacity") +} + +func buildScheduledJobSpec(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec { + now := timestamppb.Now() + + jobID := fmt.Sprintf("%s-scheduled-%d-%d", jobType, now.AsTime().UnixNano(), index) + + job := &plugin_pb.JobSpec{ + JobId: jobID, + JobType: jobType, + Priority: plugin_pb.JobPriority_JOB_PRIORITY_NORMAL, + Parameters: map[string]*plugin_pb.ConfigValue{}, + Labels: map[string]string{}, + CreatedAt: now, + ScheduledAt: now, + } + + if proposal == nil { + return job + } + + if proposal.JobType != "" { + job.JobType = proposal.JobType + } + job.Summary = proposal.Summary + job.Detail = proposal.Detail + if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED { + job.Priority = proposal.Priority + } + job.DedupeKey = proposal.DedupeKey + job.Parameters = CloneConfigValueMap(proposal.Parameters) + if proposal.Labels != nil { + job.Labels = make(map[string]string, len(proposal.Labels)) + for k, v := range proposal.Labels { + job.Labels[k] = v + } + } + if proposal.NotBefore != nil { + job.ScheduledAt = proposal.NotBefore + } + + return job +} + +func durationFromSeconds(seconds int32, defaultValue time.Duration) time.Duration { + if seconds <= 0 { + return defaultValue + } + return time.Duration(seconds) * time.Second +} + +func secondsFromDuration(duration time.Duration) int32 { + if duration <= 0 { + return 0 + } + return int32(duration / time.Second) +} + +func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bool { + if duration <= 0 { + return true + } + + timer := time.NewTimer(duration) + defer timer.Stop() + + select { + case <-shutdown: + return false + case <-timer.C: + return true + } +} + +func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) { + if len(proposals) == 0 { + return proposals, 0 + } + + activeKeys := make(map[string]struct{}) + r.jobsMu.RLock() + for _, job := range r.jobs { + if job == nil { + continue + } + if strings.TrimSpace(job.JobType) != strings.TrimSpace(jobType) { + continue + } + if !isActiveTrackedJobState(job.State) { + continue + } + + key := strings.TrimSpace(job.DedupeKey) + if key == "" { + key = strings.TrimSpace(job.JobID) + } + if key == "" { + continue + } + activeKeys[key] = struct{}{} + } + r.jobsMu.RUnlock() + + if len(activeKeys) == 0 { + return proposals, 0 + } + + filtered := make([]*plugin_pb.JobProposal, 0, len(proposals)) + skipped := 0 + for _, proposal := range proposals { + if proposal == nil { + continue + } + key := proposalExecutionKey(proposal) + if key != "" { + if _, exists := activeKeys[key]; exists { + skipped++ + continue + } + } + filtered = append(filtered, proposal) + } + + return filtered, skipped +} + +func proposalExecutionKey(proposal *plugin_pb.JobProposal) string { + if proposal == nil { + return "" + } + key := strings.TrimSpace(proposal.DedupeKey) + if key != "" { + return key + } + return strings.TrimSpace(proposal.ProposalId) +} + +func isActiveTrackedJobState(state string) bool { + normalized := strings.ToLower(strings.TrimSpace(state)) + switch normalized { + case "pending", "assigned", "running", "in_progress", "job_state_pending", "job_state_assigned", "job_state_running": + return true + default: + return false + } +} + +func isWaitingTrackedJobState(state string) bool { + normalized := strings.ToLower(strings.TrimSpace(state)) + return normalized == "pending" || normalized == "job_state_pending" +} + +func (r *Plugin) filterScheduledProposals(proposals []*plugin_pb.JobProposal) []*plugin_pb.JobProposal { + filtered := make([]*plugin_pb.JobProposal, 0, len(proposals)) + seenInRun := make(map[string]struct{}, len(proposals)) + + for _, proposal := range proposals { + if proposal == nil { + continue + } + + key := proposal.DedupeKey + if key == "" { + key = proposal.ProposalId + } + if key == "" { + filtered = append(filtered, proposal) + continue + } + + if _, exists := seenInRun[key]; exists { + continue + } + + seenInRun[key] = struct{}{} + filtered = append(filtered, proposal) + } + + return filtered +} diff --git a/weed/admin/plugin/plugin_scheduler_test.go b/weed/admin/plugin/plugin_scheduler_test.go new file mode 100644 index 000000000..3fa9067c2 --- /dev/null +++ b/weed/admin/plugin/plugin_scheduler_test.go @@ -0,0 +1,583 @@ +package plugin + +import ( + "fmt" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{ + JobType: "vacuum", + AdminRuntime: &plugin_pb.AdminRuntimeConfig{ + Enabled: true, + DetectionIntervalSeconds: 30, + DetectionTimeoutSeconds: 20, + MaxJobsPerDetection: 123, + GlobalExecutionConcurrency: 5, + PerWorkerExecutionConcurrency: 2, + RetryLimit: 4, + RetryBackoffSeconds: 7, + }, + }) + if err != nil { + t.Fatalf("SaveJobTypeConfig: %v", err) + } + + policy, enabled, err := pluginSvc.loadSchedulerPolicy("vacuum") + if err != nil { + t.Fatalf("loadSchedulerPolicy: %v", err) + } + if !enabled { + t.Fatalf("expected enabled policy") + } + if policy.MaxResults != 123 { + t.Fatalf("unexpected max results: got=%d", policy.MaxResults) + } + if policy.ExecutionConcurrency != 5 { + t.Fatalf("unexpected global concurrency: got=%d", policy.ExecutionConcurrency) + } + if policy.PerWorkerConcurrency != 2 { + t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency) + } + if policy.RetryLimit != 4 { + t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit) + } +} + +func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + err = pluginSvc.store.SaveDescriptor("ec", &plugin_pb.JobTypeDescriptor{ + JobType: "ec", + AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ + Enabled: true, + DetectionIntervalSeconds: 60, + DetectionTimeoutSeconds: 25, + MaxJobsPerDetection: 30, + GlobalExecutionConcurrency: 4, + PerWorkerExecutionConcurrency: 2, + RetryLimit: 3, + RetryBackoffSeconds: 6, + }, + }) + if err != nil { + t.Fatalf("SaveDescriptor: %v", err) + } + + policy, enabled, err := pluginSvc.loadSchedulerPolicy("ec") + if err != nil { + t.Fatalf("loadSchedulerPolicy: %v", err) + } + if !enabled { + t.Fatalf("expected enabled policy from descriptor defaults") + } + if policy.MaxResults != 30 { + t.Fatalf("unexpected max results: got=%d", policy.MaxResults) + } + if policy.ExecutionConcurrency != 4 { + t.Fatalf("unexpected global concurrency: got=%d", policy.ExecutionConcurrency) + } + if policy.PerWorkerConcurrency != 2 { + t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency) + } +} + +func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 4}, + }, + }) + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2}, + }, + }) + + policy := schedulerPolicy{ + PerWorkerConcurrency: 1, + ExecutorReserveBackoff: time.Millisecond, + } + + executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy) + if err != nil { + t.Fatalf("reserve executor 1: %v", err) + } + defer release1() + + executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy) + if err != nil { + t.Fatalf("reserve executor 2: %v", err) + } + defer release2() + + if executor1.WorkerID == executor2.WorkerID { + t.Fatalf("expected different executors due per-worker limit, got same worker %s", executor1.WorkerID) + } +} + +func TestFilterScheduledProposalsDedupe(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + proposals := []*plugin_pb.JobProposal{ + {ProposalId: "p1", DedupeKey: "d1"}, + {ProposalId: "p2", DedupeKey: "d1"}, // same dedupe key + {ProposalId: "p3", DedupeKey: "d3"}, + {ProposalId: "p3"}, // fallback dedupe by proposal id + {ProposalId: "p4"}, + {ProposalId: "p4"}, // same proposal id, no dedupe key + } + + filtered := pluginSvc.filterScheduledProposals(proposals) + if len(filtered) != 4 { + t.Fatalf("unexpected filtered size: got=%d want=4", len(filtered)) + } + + filtered2 := pluginSvc.filterScheduledProposals(proposals) + if len(filtered2) != 4 { + t.Fatalf("expected second run dedupe to be per-run only, got=%d", len(filtered2)) + } +} + +func TestBuildScheduledJobSpecDoesNotReuseProposalID(t *testing.T) { + t.Parallel() + + proposal := &plugin_pb.JobProposal{ + ProposalId: "vacuum-2", + DedupeKey: "vacuum:2", + JobType: "vacuum", + } + + jobA := buildScheduledJobSpec("vacuum", proposal, 0) + jobB := buildScheduledJobSpec("vacuum", proposal, 1) + + if jobA.JobId == proposal.ProposalId { + t.Fatalf("scheduled job id must not reuse proposal id: %s", jobA.JobId) + } + if jobB.JobId == proposal.ProposalId { + t.Fatalf("scheduled job id must not reuse proposal id: %s", jobB.JobId) + } + if jobA.JobId == jobB.JobId { + t.Fatalf("scheduled job ids must be unique across jobs: %s", jobA.JobId) + } +} + +func TestFilterProposalsWithActiveJobs(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{ + JobId: "job-1", + JobType: "vacuum", + DedupeKey: "vacuum:k1", + }, 1) + pluginSvc.trackExecutionStart("req-2", "worker-b", &plugin_pb.JobSpec{ + JobId: "job-2", + JobType: "vacuum", + }, 1) + pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{ + JobId: "job-3", + JobType: "vacuum", + DedupeKey: "vacuum:k4", + }) + + filtered, skipped := pluginSvc.filterProposalsWithActiveJobs("vacuum", []*plugin_pb.JobProposal{ + {ProposalId: "proposal-1", JobType: "vacuum", DedupeKey: "vacuum:k1"}, + {ProposalId: "job-2", JobType: "vacuum"}, + {ProposalId: "proposal-2b", JobType: "vacuum", DedupeKey: "vacuum:k4"}, + {ProposalId: "proposal-3", JobType: "vacuum", DedupeKey: "vacuum:k3"}, + {ProposalId: "proposal-4", JobType: "balance", DedupeKey: "balance:k1"}, + }) + if skipped != 3 { + t.Fatalf("unexpected skipped count: got=%d want=3", skipped) + } + if len(filtered) != 2 { + t.Fatalf("unexpected filtered size: got=%d want=2", len(filtered)) + } + if filtered[0].ProposalId != "proposal-3" || filtered[1].ProposalId != "proposal-4" { + t.Fatalf("unexpected filtered proposals: got=%s,%s", filtered[0].ProposalId, filtered[1].ProposalId) + } +} + +func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + policy := schedulerPolicy{ + ExecutionTimeout: 30 * time.Millisecond, + ExecutorReserveBackoff: 5 * time.Millisecond, + PerWorkerConcurrency: 1, + } + + start := time.Now() + pluginSvc.Shutdown() + _, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy) + if err == nil { + t.Fatalf("expected reservation shutdown error") + } + if time.Since(start) > 50*time.Millisecond { + t.Fatalf("reservation returned too late after shutdown: duration=%v", time.Since(start)) + } +} + +func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1}, + }, + }) + + policy := schedulerPolicy{ + ExecutionTimeout: time.Second, + PerWorkerConcurrency: 8, + ExecutorReserveBackoff: 5 * time.Millisecond, + } + + _, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy) + if err != nil { + t.Fatalf("reserve executor 1: %v", err) + } + defer release1() + + type reserveResult struct { + err error + } + secondReserveCh := make(chan reserveResult, 1) + go func() { + _, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy) + if release2 != nil { + release2() + } + secondReserveCh <- reserveResult{err: reserveErr} + }() + + select { + case result := <-secondReserveCh: + t.Fatalf("expected second reservation to wait for capacity, got=%v", result.err) + case <-time.After(25 * time.Millisecond): + // Expected: still waiting. + } + + release1() + + select { + case result := <-secondReserveCh: + if result.err != nil { + t.Fatalf("second reservation error: %v", result.err) + } + case <-time.After(200 * time.Millisecond): + t.Fatalf("second reservation did not acquire after capacity release") + } +} + +func TestShouldSkipDetectionForWaitingJobs(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + policy := schedulerPolicy{ + ExecutionConcurrency: 2, + MaxResults: 100, + } + threshold := waitingBacklogThreshold(policy) + if threshold <= 0 { + t.Fatalf("expected positive waiting threshold") + } + + for i := 0; i < threshold; i++ { + pluginSvc.trackExecutionQueued(&plugin_pb.JobSpec{ + JobId: fmt.Sprintf("job-waiting-%d", i), + JobType: "vacuum", + DedupeKey: fmt.Sprintf("vacuum:%d", i), + }) + } + + skip, waitingCount, waitingThreshold := pluginSvc.shouldSkipDetectionForWaitingJobs("vacuum", policy) + if !skip { + t.Fatalf("expected detection to skip when waiting backlog reaches threshold") + } + if waitingCount != threshold { + t.Fatalf("unexpected waiting count: got=%d want=%d", waitingCount, threshold) + } + if waitingThreshold != threshold { + t.Fatalf("unexpected waiting threshold: got=%d want=%d", waitingThreshold, threshold) + } +} + +func TestWaitingBacklogThresholdHonorsMaxResultsCap(t *testing.T) { + t.Parallel() + + policy := schedulerPolicy{ + ExecutionConcurrency: 8, + MaxResults: 6, + } + threshold := waitingBacklogThreshold(policy) + if threshold != 6 { + t.Fatalf("expected threshold to be capped by max results, got=%d", threshold) + } +} + +func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + const jobType = "vacuum" + err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{ + JobType: jobType, + AdminRuntime: &plugin_pb.AdminRuntimeConfig{ + Enabled: true, + DetectionIntervalSeconds: 45, + DetectionTimeoutSeconds: 30, + MaxJobsPerDetection: 80, + GlobalExecutionConcurrency: 3, + PerWorkerExecutionConcurrency: 2, + RetryLimit: 1, + RetryBackoffSeconds: 9, + }, + }) + if err != nil { + t.Fatalf("SaveJobTypeConfig: %v", err) + } + + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, CanExecute: true}, + }, + }) + + nextDetectionAt := time.Now().UTC().Add(2 * time.Minute).Round(time.Second) + pluginSvc.schedulerMu.Lock() + pluginSvc.nextDetectionAt[jobType] = nextDetectionAt + pluginSvc.detectionInFlight[jobType] = true + pluginSvc.schedulerMu.Unlock() + + states, err := pluginSvc.ListSchedulerStates() + if err != nil { + t.Fatalf("ListSchedulerStates: %v", err) + } + + state := findSchedulerState(states, jobType) + if state == nil { + t.Fatalf("missing scheduler state for %s", jobType) + } + if !state.Enabled { + t.Fatalf("expected enabled scheduler state") + } + if state.PolicyError != "" { + t.Fatalf("unexpected policy error: %s", state.PolicyError) + } + if !state.DetectionInFlight { + t.Fatalf("expected detection in flight") + } + if state.NextDetectionAt == nil { + t.Fatalf("expected next detection time") + } + if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() { + t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt) + } + if state.DetectionIntervalSeconds != 45 { + t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds) + } + if state.DetectionTimeoutSeconds != 30 { + t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds) + } + if state.ExecutionTimeoutSeconds != 90 { + t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds) + } + if state.MaxJobsPerDetection != 80 { + t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection) + } + if state.GlobalExecutionConcurrency != 3 { + t.Fatalf("unexpected global execution concurrency: got=%d", state.GlobalExecutionConcurrency) + } + if state.PerWorkerExecutionConcurrency != 2 { + t.Fatalf("unexpected per worker execution concurrency: got=%d", state.PerWorkerExecutionConcurrency) + } + if state.RetryLimit != 1 { + t.Fatalf("unexpected retry limit: got=%d", state.RetryLimit) + } + if state.RetryBackoffSeconds != 9 { + t.Fatalf("unexpected retry backoff: got=%d", state.RetryBackoffSeconds) + } + if !state.DetectorAvailable || state.DetectorWorkerID != "worker-a" { + t.Fatalf("unexpected detector assignment: available=%v worker=%s", state.DetectorAvailable, state.DetectorWorkerID) + } + if state.ExecutorWorkerCount != 1 { + t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount) + } +} + +func TestListSchedulerStatesShowsDisabledWhenNoPolicy(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + const jobType = "balance" + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: jobType, CanDetect: true, CanExecute: true}, + }, + }) + + states, err := pluginSvc.ListSchedulerStates() + if err != nil { + t.Fatalf("ListSchedulerStates: %v", err) + } + + state := findSchedulerState(states, jobType) + if state == nil { + t.Fatalf("missing scheduler state for %s", jobType) + } + if state.Enabled { + t.Fatalf("expected disabled scheduler state") + } + if state.PolicyError != "" { + t.Fatalf("unexpected policy error: %s", state.PolicyError) + } + if !state.DetectorAvailable || state.DetectorWorkerID != "worker-b" { + t.Fatalf("unexpected detector details: available=%v worker=%s", state.DetectorAvailable, state.DetectorWorkerID) + } + if state.ExecutorWorkerCount != 1 { + t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount) + } +} + +func findSchedulerState(states []SchedulerJobTypeState, jobType string) *SchedulerJobTypeState { + for i := range states { + if states[i].JobType == jobType { + return &states[i] + } + } + return nil +} + +func TestPickDetectorPrefersLeasedWorker(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true}, + }, + }) + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true}, + }, + }) + + pluginSvc.setDetectorLease("vacuum", "worker-b") + + detector, err := pluginSvc.pickDetector("vacuum") + if err != nil { + t.Fatalf("pickDetector: %v", err) + } + if detector.WorkerID != "worker-b" { + t.Fatalf("expected leased detector worker-b, got=%s", detector.WorkerID) + } +} + +func TestPickDetectorReassignsWhenLeaseIsStale(t *testing.T) { + t.Parallel() + + pluginSvc, err := New(Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer pluginSvc.Shutdown() + + pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true}, + }, + }) + pluginSvc.setDetectorLease("vacuum", "worker-stale") + + detector, err := pluginSvc.pickDetector("vacuum") + if err != nil { + t.Fatalf("pickDetector: %v", err) + } + if detector.WorkerID != "worker-a" { + t.Fatalf("expected reassigned detector worker-a, got=%s", detector.WorkerID) + } + + lease := pluginSvc.getDetectorLease("vacuum") + if lease != "worker-a" { + t.Fatalf("expected detector lease to be updated to worker-a, got=%s", lease) + } +} diff --git a/weed/admin/plugin/plugin_schema_prefetch.go b/weed/admin/plugin/plugin_schema_prefetch.go new file mode 100644 index 000000000..13816f5d8 --- /dev/null +++ b/weed/admin/plugin/plugin_schema_prefetch.go @@ -0,0 +1,66 @@ +package plugin + +import ( + "context" + "sort" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +const descriptorPrefetchTimeout = 20 * time.Second + +func (r *Plugin) prefetchDescriptorsFromHello(hello *plugin_pb.WorkerHello) { + if hello == nil || len(hello.Capabilities) == 0 { + return + } + + jobTypeSet := make(map[string]struct{}) + for _, capability := range hello.Capabilities { + if capability == nil || capability.JobType == "" { + continue + } + if !capability.CanDetect && !capability.CanExecute { + continue + } + jobTypeSet[capability.JobType] = struct{}{} + } + + if len(jobTypeSet) == 0 { + return + } + + jobTypes := make([]string, 0, len(jobTypeSet)) + for jobType := range jobTypeSet { + jobTypes = append(jobTypes, jobType) + } + sort.Strings(jobTypes) + + for _, jobType := range jobTypes { + select { + case <-r.shutdownCh: + return + default: + } + + descriptor, err := r.store.LoadDescriptor(jobType) + if err != nil { + glog.Warningf("Plugin descriptor prefetch check failed for %s: %v", jobType, err) + continue + } + if descriptor != nil { + continue + } + + ctx, cancel := context.WithTimeout(r.ctx, descriptorPrefetchTimeout) + _, err = r.RequestConfigSchema(ctx, jobType, false) + cancel() + if err != nil { + glog.V(1).Infof("Plugin descriptor prefetch skipped for %s: %v", jobType, err) + continue + } + + glog.V(1).Infof("Plugin descriptor prefetched for job_type=%s", jobType) + } +} diff --git a/weed/admin/plugin/registry.go b/weed/admin/plugin/registry.go new file mode 100644 index 000000000..dafddc203 --- /dev/null +++ b/weed/admin/plugin/registry.go @@ -0,0 +1,465 @@ +package plugin + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +const defaultWorkerStaleTimeout = 2 * time.Minute + +// WorkerSession contains tracked worker metadata and plugin status. +type WorkerSession struct { + WorkerID string + WorkerInstance string + Address string + WorkerVersion string + ProtocolVersion string + ConnectedAt time.Time + LastSeenAt time.Time + Capabilities map[string]*plugin_pb.JobTypeCapability + Heartbeat *plugin_pb.WorkerHeartbeat +} + +// Registry tracks connected plugin workers and capability-based selection. +type Registry struct { + mu sync.RWMutex + sessions map[string]*WorkerSession + staleAfter time.Duration + detectorCursor map[string]int + executorCursor map[string]int +} + +func NewRegistry() *Registry { + return &Registry{ + sessions: make(map[string]*WorkerSession), + staleAfter: defaultWorkerStaleTimeout, + detectorCursor: make(map[string]int), + executorCursor: make(map[string]int), + } +} + +func (r *Registry) UpsertFromHello(hello *plugin_pb.WorkerHello) *WorkerSession { + now := time.Now() + caps := make(map[string]*plugin_pb.JobTypeCapability, len(hello.Capabilities)) + for _, c := range hello.Capabilities { + if c == nil || c.JobType == "" { + continue + } + caps[c.JobType] = cloneJobTypeCapability(c) + } + + r.mu.Lock() + defer r.mu.Unlock() + + session, ok := r.sessions[hello.WorkerId] + if !ok { + session = &WorkerSession{ + WorkerID: hello.WorkerId, + ConnectedAt: now, + } + r.sessions[hello.WorkerId] = session + } + + session.WorkerInstance = hello.WorkerInstanceId + session.Address = hello.Address + session.WorkerVersion = hello.WorkerVersion + session.ProtocolVersion = hello.ProtocolVersion + session.LastSeenAt = now + session.Capabilities = caps + + return cloneWorkerSession(session) +} + +func (r *Registry) Remove(workerID string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.sessions, workerID) +} + +func (r *Registry) UpdateHeartbeat(workerID string, heartbeat *plugin_pb.WorkerHeartbeat) { + r.mu.Lock() + defer r.mu.Unlock() + + session, ok := r.sessions[workerID] + if !ok { + return + } + session.Heartbeat = cloneWorkerHeartbeat(heartbeat) + session.LastSeenAt = time.Now() +} + +func (r *Registry) Get(workerID string) (*WorkerSession, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + session, ok := r.sessions[workerID] + if !ok || r.isSessionStaleLocked(session, time.Now()) { + return nil, false + } + return cloneWorkerSession(session), true +} + +func (r *Registry) List() []*WorkerSession { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]*WorkerSession, 0, len(r.sessions)) + now := time.Now() + for _, s := range r.sessions { + if r.isSessionStaleLocked(s, now) { + continue + } + out = append(out, cloneWorkerSession(s)) + } + sort.Slice(out, func(i, j int) bool { + return out[i].WorkerID < out[j].WorkerID + }) + return out +} + +// DetectableJobTypes returns sorted job types that currently have at least one detect-capable worker. +func (r *Registry) DetectableJobTypes() []string { + r.mu.RLock() + defer r.mu.RUnlock() + + jobTypes := make(map[string]struct{}) + now := time.Now() + for _, session := range r.sessions { + if r.isSessionStaleLocked(session, now) { + continue + } + for jobType, capability := range session.Capabilities { + if capability == nil || !capability.CanDetect { + continue + } + jobTypes[jobType] = struct{}{} + } + } + + out := make([]string, 0, len(jobTypes)) + for jobType := range jobTypes { + out = append(out, jobType) + } + sort.Strings(out) + return out +} + +// JobTypes returns sorted job types known by connected workers regardless of capability kind. +func (r *Registry) JobTypes() []string { + r.mu.RLock() + defer r.mu.RUnlock() + + jobTypes := make(map[string]struct{}) + now := time.Now() + for _, session := range r.sessions { + if r.isSessionStaleLocked(session, now) { + continue + } + for jobType := range session.Capabilities { + if jobType == "" { + continue + } + jobTypes[jobType] = struct{}{} + } + } + + out := make([]string, 0, len(jobTypes)) + for jobType := range jobTypes { + out = append(out, jobType) + } + sort.Strings(out) + return out +} + +// PickSchemaProvider picks one worker for schema requests. +// Preference order: +// 1) workers that can detect this job type +// 2) workers that can execute this job type +// tie-break: more free slots, then lexical worker ID. +func (r *Registry) PickSchemaProvider(jobType string) (*WorkerSession, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + var candidates []*WorkerSession + now := time.Now() + for _, s := range r.sessions { + if r.isSessionStaleLocked(s, now) { + continue + } + capability := s.Capabilities[jobType] + if capability == nil { + continue + } + if capability.CanDetect || capability.CanExecute { + candidates = append(candidates, s) + } + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("no worker available for schema job_type=%s", jobType) + } + + sort.Slice(candidates, func(i, j int) bool { + a := candidates[i] + b := candidates[j] + ac := a.Capabilities[jobType] + bc := b.Capabilities[jobType] + + // Prefer detect-capable providers first. + if ac.CanDetect != bc.CanDetect { + return ac.CanDetect + } + + aSlots := availableDetectionSlots(a, ac) + availableExecutionSlots(a, ac) + bSlots := availableDetectionSlots(b, bc) + availableExecutionSlots(b, bc) + if aSlots != bSlots { + return aSlots > bSlots + } + return a.WorkerID < b.WorkerID + }) + + return cloneWorkerSession(candidates[0]), nil +} + +// PickDetector picks one detector worker for a job type. +func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) { + return r.pickByKind(jobType, true) +} + +// PickExecutor picks one executor worker for a job type. +func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) { + return r.pickByKind(jobType, false) +} + +// ListExecutors returns sorted executor candidates for one job type. +// Ordering is by most available execution slots, then lexical worker ID. +// The top tie group is rotated round-robin to prevent sticky assignment. +func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) { + r.mu.Lock() + defer r.mu.Unlock() + + candidates := r.collectByKindLocked(jobType, false, time.Now()) + if len(candidates) == 0 { + return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType) + } + + sortByKind(candidates, jobType, false) + r.rotateTopCandidatesLocked(candidates, jobType, false) + + out := make([]*WorkerSession, 0, len(candidates)) + for _, candidate := range candidates { + out = append(out, cloneWorkerSession(candidate)) + } + return out, nil +} + +func (r *Registry) pickByKind(jobType string, detect bool) (*WorkerSession, error) { + r.mu.Lock() + defer r.mu.Unlock() + + candidates := r.collectByKindLocked(jobType, detect, time.Now()) + + if len(candidates) == 0 { + kind := "executor" + if detect { + kind = "detector" + } + return nil, fmt.Errorf("no %s worker available for job_type=%s", kind, jobType) + } + + sortByKind(candidates, jobType, detect) + r.rotateTopCandidatesLocked(candidates, jobType, detect) + + return cloneWorkerSession(candidates[0]), nil +} + +func (r *Registry) collectByKindLocked(jobType string, detect bool, now time.Time) []*WorkerSession { + var candidates []*WorkerSession + for _, session := range r.sessions { + if r.isSessionStaleLocked(session, now) { + continue + } + capability := session.Capabilities[jobType] + if capability == nil { + continue + } + if detect && capability.CanDetect { + candidates = append(candidates, session) + } + if !detect && capability.CanExecute { + candidates = append(candidates, session) + } + } + return candidates +} + +func (r *Registry) isSessionStaleLocked(session *WorkerSession, now time.Time) bool { + if session == nil { + return true + } + if r.staleAfter <= 0 { + return false + } + + lastSeen := session.LastSeenAt + if lastSeen.IsZero() { + lastSeen = session.ConnectedAt + } + if lastSeen.IsZero() { + return false + } + return now.Sub(lastSeen) > r.staleAfter +} + +func sortByKind(candidates []*WorkerSession, jobType string, detect bool) { + sort.Slice(candidates, func(i, j int) bool { + a := candidates[i] + b := candidates[j] + ac := a.Capabilities[jobType] + bc := b.Capabilities[jobType] + + aSlots := availableSlotsByKind(a, ac, detect) + bSlots := availableSlotsByKind(b, bc, detect) + + if aSlots != bSlots { + return aSlots > bSlots + } + return a.WorkerID < b.WorkerID + }) +} + +func (r *Registry) rotateTopCandidatesLocked(candidates []*WorkerSession, jobType string, detect bool) { + if len(candidates) < 2 { + return + } + + capability := candidates[0].Capabilities[jobType] + topSlots := availableSlotsByKind(candidates[0], capability, detect) + tieEnd := 1 + for tieEnd < len(candidates) { + nextCapability := candidates[tieEnd].Capabilities[jobType] + if availableSlotsByKind(candidates[tieEnd], nextCapability, detect) != topSlots { + break + } + tieEnd++ + } + if tieEnd <= 1 { + return + } + + cursorKey := strings.TrimSpace(jobType) + if cursorKey == "" { + cursorKey = "*" + } + + var offset int + if detect { + offset = r.detectorCursor[cursorKey] % tieEnd + r.detectorCursor[cursorKey] = (offset + 1) % tieEnd + } else { + offset = r.executorCursor[cursorKey] % tieEnd + r.executorCursor[cursorKey] = (offset + 1) % tieEnd + } + + if offset == 0 { + return + } + + prefix := append([]*WorkerSession(nil), candidates[:tieEnd]...) + for i := 0; i < tieEnd; i++ { + candidates[i] = prefix[(i+offset)%tieEnd] + } +} + +func availableSlotsByKind( + session *WorkerSession, + capability *plugin_pb.JobTypeCapability, + detect bool, +) int { + if detect { + return availableDetectionSlots(session, capability) + } + return availableExecutionSlots(session, capability) +} + +func availableDetectionSlots(session *WorkerSession, capability *plugin_pb.JobTypeCapability) int { + if session.Heartbeat != nil && session.Heartbeat.DetectionSlotsTotal > 0 { + free := int(session.Heartbeat.DetectionSlotsTotal - session.Heartbeat.DetectionSlotsUsed) + if free < 0 { + return 0 + } + return free + } + if capability.MaxDetectionConcurrency > 0 { + return int(capability.MaxDetectionConcurrency) + } + return 1 +} + +func availableExecutionSlots(session *WorkerSession, capability *plugin_pb.JobTypeCapability) int { + if session.Heartbeat != nil && session.Heartbeat.ExecutionSlotsTotal > 0 { + free := int(session.Heartbeat.ExecutionSlotsTotal - session.Heartbeat.ExecutionSlotsUsed) + if free < 0 { + return 0 + } + return free + } + if capability.MaxExecutionConcurrency > 0 { + return int(capability.MaxExecutionConcurrency) + } + return 1 +} + +func cloneWorkerSession(in *WorkerSession) *WorkerSession { + if in == nil { + return nil + } + out := *in + out.Capabilities = make(map[string]*plugin_pb.JobTypeCapability, len(in.Capabilities)) + for jobType, cap := range in.Capabilities { + out.Capabilities[jobType] = cloneJobTypeCapability(cap) + } + out.Heartbeat = cloneWorkerHeartbeat(in.Heartbeat) + return &out +} + +func cloneJobTypeCapability(in *plugin_pb.JobTypeCapability) *plugin_pb.JobTypeCapability { + if in == nil { + return nil + } + out := *in + return &out +} + +func cloneWorkerHeartbeat(in *plugin_pb.WorkerHeartbeat) *plugin_pb.WorkerHeartbeat { + if in == nil { + return nil + } + out := *in + if in.RunningWork != nil { + out.RunningWork = make([]*plugin_pb.RunningWork, 0, len(in.RunningWork)) + for _, rw := range in.RunningWork { + if rw == nil { + continue + } + clone := *rw + out.RunningWork = append(out.RunningWork, &clone) + } + } + if in.QueuedJobsByType != nil { + out.QueuedJobsByType = make(map[string]int32, len(in.QueuedJobsByType)) + for k, v := range in.QueuedJobsByType { + out.QueuedJobsByType[k] = v + } + } + if in.Metadata != nil { + out.Metadata = make(map[string]string, len(in.Metadata)) + for k, v := range in.Metadata { + out.Metadata[k] = v + } + } + return &out +} diff --git a/weed/admin/plugin/registry_test.go b/weed/admin/plugin/registry_test.go new file mode 100644 index 000000000..ff61b2b7a --- /dev/null +++ b/weed/admin/plugin/registry_test.go @@ -0,0 +1,321 @@ +package plugin + +import ( + "reflect" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" +) + +func TestRegistryPickDetectorPrefersMoreFreeSlots(t *testing.T) { + t.Parallel() + + r := NewRegistry() + + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 2, MaxExecutionConcurrency: 2}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 4, MaxExecutionConcurrency: 4}, + }, + }) + + r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-a", + DetectionSlotsUsed: 1, + DetectionSlotsTotal: 2, + }) + r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-b", + DetectionSlotsUsed: 1, + DetectionSlotsTotal: 4, + }) + + picked, err := r.PickDetector("vacuum") + if err != nil { + t.Fatalf("PickDetector: %v", err) + } + if picked.WorkerID != "worker-b" { + t.Fatalf("unexpected detector picked: got %s want worker-b", picked.WorkerID) + } +} + +func TestRegistryPickExecutorAllowsSameWorker(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-x", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanDetect: true, CanExecute: true, MaxDetectionConcurrency: 1, MaxExecutionConcurrency: 1}, + }, + }) + + detector, err := r.PickDetector("balance") + if err != nil { + t.Fatalf("PickDetector: %v", err) + } + executor, err := r.PickExecutor("balance") + if err != nil { + t.Fatalf("PickExecutor: %v", err) + } + + if detector.WorkerID != "worker-x" || executor.WorkerID != "worker-x" { + t.Fatalf("expected same worker for detect/execute, got detector=%s executor=%s", detector.WorkerID, executor.WorkerID) + } +} + +func TestRegistryDetectableJobTypes(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true, CanExecute: true}, + {JobType: "balance", CanDetect: false, CanExecute: true}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "ec", CanDetect: true, CanExecute: false}, + {JobType: "vacuum", CanDetect: true, CanExecute: false}, + }, + }) + + got := r.DetectableJobTypes() + want := []string{"ec", "vacuum"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected detectable job types: got=%v want=%v", got, want) + } +} + +func TestRegistryJobTypes(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true}, + {JobType: "balance", CanExecute: true}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "ec", CanDetect: true}, + }, + }) + + got := r.JobTypes() + want := []string{"balance", "ec", "vacuum"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected job types: got=%v want=%v", got, want) + } +} + +func TestRegistryListExecutorsSortedBySlots(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 4}, + }, + }) + + r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-a", + ExecutionSlotsUsed: 1, + ExecutionSlotsTotal: 2, + }) + r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-b", + ExecutionSlotsUsed: 1, + ExecutionSlotsTotal: 4, + }) + + executors, err := r.ListExecutors("balance") + if err != nil { + t.Fatalf("ListExecutors: %v", err) + } + if len(executors) != 2 { + t.Fatalf("unexpected candidate count: got=%d", len(executors)) + } + if executors[0].WorkerID != "worker-b" || executors[1].WorkerID != "worker-a" { + t.Fatalf("unexpected executor order: got=%s,%s", executors[0].WorkerID, executors[1].WorkerID) + } +} + +func TestRegistryPickExecutorRoundRobinForTopTie(t *testing.T) { + t.Parallel() + + r := NewRegistry() + for _, workerID := range []string{"worker-a", "worker-b", "worker-c"} { + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: workerID, + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1}, + }, + }) + } + + got := make([]string, 0, 6) + for i := 0; i < 6; i++ { + executor, err := r.PickExecutor("balance") + if err != nil { + t.Fatalf("PickExecutor: %v", err) + } + got = append(got, executor.WorkerID) + } + + want := []string{"worker-a", "worker-b", "worker-c", "worker-a", "worker-b", "worker-c"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected pick order: got=%v want=%v", got, want) + } +} + +func TestRegistryListExecutorsRoundRobinForTopTie(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-b", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 2}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-c", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "balance", CanExecute: true, MaxExecutionConcurrency: 1}, + }, + }) + + r.UpdateHeartbeat("worker-a", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-a", + ExecutionSlotsUsed: 0, + ExecutionSlotsTotal: 2, + }) + r.UpdateHeartbeat("worker-b", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-b", + ExecutionSlotsUsed: 0, + ExecutionSlotsTotal: 2, + }) + r.UpdateHeartbeat("worker-c", &plugin_pb.WorkerHeartbeat{ + WorkerId: "worker-c", + ExecutionSlotsUsed: 0, + ExecutionSlotsTotal: 1, + }) + + firstCall, err := r.ListExecutors("balance") + if err != nil { + t.Fatalf("ListExecutors first call: %v", err) + } + secondCall, err := r.ListExecutors("balance") + if err != nil { + t.Fatalf("ListExecutors second call: %v", err) + } + thirdCall, err := r.ListExecutors("balance") + if err != nil { + t.Fatalf("ListExecutors third call: %v", err) + } + + if firstCall[0].WorkerID != "worker-a" || firstCall[1].WorkerID != "worker-b" || firstCall[2].WorkerID != "worker-c" { + t.Fatalf("unexpected first executor order: got=%s,%s,%s", firstCall[0].WorkerID, firstCall[1].WorkerID, firstCall[2].WorkerID) + } + if secondCall[0].WorkerID != "worker-b" || secondCall[1].WorkerID != "worker-a" || secondCall[2].WorkerID != "worker-c" { + t.Fatalf("unexpected second executor order: got=%s,%s,%s", secondCall[0].WorkerID, secondCall[1].WorkerID, secondCall[2].WorkerID) + } + if thirdCall[0].WorkerID != "worker-a" || thirdCall[1].WorkerID != "worker-b" || thirdCall[2].WorkerID != "worker-c" { + t.Fatalf("unexpected third executor order: got=%s,%s,%s", thirdCall[0].WorkerID, thirdCall[1].WorkerID, thirdCall[2].WorkerID) + } +} + +func TestRegistrySkipsStaleWorkersForSelectionAndListing(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.staleAfter = 2 * time.Second + + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-stale", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true, CanExecute: true}, + }, + }) + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-fresh", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true, CanExecute: true}, + }, + }) + + r.mu.Lock() + r.sessions["worker-stale"].LastSeenAt = time.Now().Add(-10 * time.Second) + r.sessions["worker-fresh"].LastSeenAt = time.Now() + r.mu.Unlock() + + picked, err := r.PickDetector("vacuum") + if err != nil { + t.Fatalf("PickDetector: %v", err) + } + if picked.WorkerID != "worker-fresh" { + t.Fatalf("unexpected detector: got=%s want=worker-fresh", picked.WorkerID) + } + + if _, ok := r.Get("worker-stale"); ok { + t.Fatalf("expected stale worker to be hidden from Get") + } + if _, ok := r.Get("worker-fresh"); !ok { + t.Fatalf("expected fresh worker from Get") + } + + listed := r.List() + if len(listed) != 1 || listed[0].WorkerID != "worker-fresh" { + t.Fatalf("unexpected listed workers: %+v", listed) + } +} + +func TestRegistryReturnsNoDetectorWhenAllWorkersStale(t *testing.T) { + t.Parallel() + + r := NewRegistry() + r.staleAfter = 2 * time.Second + + r.UpsertFromHello(&plugin_pb.WorkerHello{ + WorkerId: "worker-a", + Capabilities: []*plugin_pb.JobTypeCapability{ + {JobType: "vacuum", CanDetect: true}, + }, + }) + + r.mu.Lock() + r.sessions["worker-a"].LastSeenAt = time.Now().Add(-10 * time.Second) + r.mu.Unlock() + + if _, err := r.PickDetector("vacuum"); err == nil { + t.Fatalf("expected no detector when all workers are stale") + } +} diff --git a/weed/admin/plugin/types.go b/weed/admin/plugin/types.go new file mode 100644 index 000000000..6a57a563c --- /dev/null +++ b/weed/admin/plugin/types.go @@ -0,0 +1,103 @@ +package plugin + +import "time" + +const ( + // Keep exactly the last 10 successful and last 10 error runs per job type. + MaxSuccessfulRunHistory = 10 + MaxErrorRunHistory = 10 +) + +type RunOutcome string + +const ( + RunOutcomeSuccess RunOutcome = "success" + RunOutcomeError RunOutcome = "error" +) + +type JobRunRecord struct { + RunID string `json:"run_id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + WorkerID string `json:"worker_id"` + Outcome RunOutcome `json:"outcome"` + Message string `json:"message,omitempty"` + DurationMs int64 `json:"duration_ms,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` +} + +type JobTypeRunHistory struct { + JobType string `json:"job_type"` + SuccessfulRuns []JobRunRecord `json:"successful_runs"` + ErrorRuns []JobRunRecord `json:"error_runs"` + LastUpdatedTime *time.Time `json:"last_updated_time,omitempty"` +} + +type TrackedJob struct { + JobID string `json:"job_id"` + JobType string `json:"job_type"` + RequestID string `json:"request_id"` + WorkerID string `json:"worker_id"` + DedupeKey string `json:"dedupe_key,omitempty"` + Summary string `json:"summary,omitempty"` + Detail string `json:"detail,omitempty"` + Parameters map[string]interface{} `json:"parameters,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + State string `json:"state"` + Progress float64 `json:"progress"` + Stage string `json:"stage,omitempty"` + Message string `json:"message,omitempty"` + Attempt int32 `json:"attempt,omitempty"` + CreatedAt *time.Time `json:"created_at,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ResultSummary string `json:"result_summary,omitempty"` + ResultOutputValues map[string]interface{} `json:"result_output_values,omitempty"` +} + +type JobActivity struct { + JobID string `json:"job_id"` + JobType string `json:"job_type"` + RequestID string `json:"request_id,omitempty"` + WorkerID string `json:"worker_id,omitempty"` + Source string `json:"source"` + Message string `json:"message"` + Stage string `json:"stage,omitempty"` + Details map[string]interface{} `json:"details,omitempty"` + OccurredAt *time.Time `json:"occurred_at,omitempty"` +} + +type JobDetail struct { + Job *TrackedJob `json:"job"` + RunRecord *JobRunRecord `json:"run_record,omitempty"` + Activities []JobActivity `json:"activities"` + RelatedJobs []TrackedJob `json:"related_jobs,omitempty"` + LastUpdated *time.Time `json:"last_updated,omitempty"` +} + +type SchedulerJobTypeState struct { + JobType string `json:"job_type"` + Enabled bool `json:"enabled"` + PolicyError string `json:"policy_error,omitempty"` + DetectionInFlight bool `json:"detection_in_flight"` + NextDetectionAt *time.Time `json:"next_detection_at,omitempty"` + DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"` + DetectionTimeoutSeconds int32 `json:"detection_timeout_seconds,omitempty"` + ExecutionTimeoutSeconds int32 `json:"execution_timeout_seconds,omitempty"` + MaxJobsPerDetection int32 `json:"max_jobs_per_detection,omitempty"` + GlobalExecutionConcurrency int `json:"global_execution_concurrency,omitempty"` + PerWorkerExecutionConcurrency int `json:"per_worker_execution_concurrency,omitempty"` + RetryLimit int `json:"retry_limit,omitempty"` + RetryBackoffSeconds int32 `json:"retry_backoff_seconds,omitempty"` + DetectorAvailable bool `json:"detector_available"` + DetectorWorkerID string `json:"detector_worker_id,omitempty"` + ExecutorWorkerCount int `json:"executor_worker_count"` +} + +func timeToPtr(t time.Time) *time.Time { + if t.IsZero() { + return nil + } + return &t +} diff --git a/weed/admin/static/js/admin.js b/weed/admin/static/js/admin.js index e20d653c4..27e1646c5 100644 --- a/weed/admin/static/js/admin.js +++ b/weed/admin/static/js/admin.js @@ -129,21 +129,6 @@ function setupSubmenuBehavior() { } } - // If we're on a maintenance page, expand the maintenance submenu - if (currentPath.startsWith('/maintenance')) { - const maintenanceSubmenu = document.getElementById('maintenanceSubmenu'); - if (maintenanceSubmenu) { - maintenanceSubmenu.classList.add('show'); - - // Update the parent toggle button state - const toggleButton = document.querySelector('[data-bs-target="#maintenanceSubmenu"]'); - if (toggleButton) { - toggleButton.classList.remove('collapsed'); - toggleButton.setAttribute('aria-expanded', 'true'); - } - } - } - // Prevent submenu from collapsing when clicking on submenu items const clusterSubmenuLinks = document.querySelectorAll('#clusterSubmenu .nav-link'); clusterSubmenuLinks.forEach(function (link) { @@ -161,14 +146,6 @@ function setupSubmenuBehavior() { }); }); - const maintenanceSubmenuLinks = document.querySelectorAll('#maintenanceSubmenu .nav-link'); - maintenanceSubmenuLinks.forEach(function (link) { - link.addEventListener('click', function (e) { - // Don't prevent the navigation, just stop the collapse behavior - e.stopPropagation(); - }); - }); - // Handle the main cluster toggle const clusterToggle = document.querySelector('[data-bs-target="#clusterSubmenu"]'); if (clusterToggle) { @@ -215,28 +192,6 @@ function setupSubmenuBehavior() { }); } - // Handle the main maintenance toggle - const maintenanceToggle = document.querySelector('[data-bs-target="#maintenanceSubmenu"]'); - if (maintenanceToggle) { - maintenanceToggle.addEventListener('click', function (e) { - e.preventDefault(); - - const submenu = document.getElementById('maintenanceSubmenu'); - const isExpanded = submenu.classList.contains('show'); - - if (isExpanded) { - // Collapse - submenu.classList.remove('show'); - this.classList.add('collapsed'); - this.setAttribute('aria-expanded', 'false'); - } else { - // Expand - submenu.classList.add('show'); - this.classList.remove('collapsed'); - this.setAttribute('aria-expanded', 'true'); - } - }); - } } // Loading indicator functions diff --git a/weed/admin/topology/capacity.go b/weed/admin/topology/capacity.go index 4b5e20843..6da8c228b 100644 --- a/weed/admin/topology/capacity.go +++ b/weed/admin/topology/capacity.go @@ -238,7 +238,7 @@ func (at *ActiveTopology) getPlanningCapacityUnsafe(disk *activeDisk) StorageSlo func (at *ActiveTopology) isDiskAvailableForPlanning(disk *activeDisk, taskType TaskType) bool { // Check total load including pending tasks totalLoad := len(disk.pendingTasks) + len(disk.assignedTasks) - if totalLoad >= MaxTotalTaskLoadPerDisk { + if MaxTotalTaskLoadPerDisk > 0 && totalLoad >= MaxTotalTaskLoadPerDisk { return false } @@ -299,6 +299,16 @@ func (at *ActiveTopology) getEffectiveAvailableCapacityUnsafe(disk *activeDisk) } baseAvailable := disk.DiskInfo.DiskInfo.MaxVolumeCount - disk.DiskInfo.DiskInfo.VolumeCount + if baseAvailable <= 0 && + disk.DiskInfo.DiskInfo.MaxVolumeCount == 0 && + disk.DiskInfo.DiskInfo.VolumeCount == 0 && + len(disk.DiskInfo.DiskInfo.VolumeInfos) == 0 && + len(disk.DiskInfo.DiskInfo.EcShardInfos) == 0 { + // Some empty volume servers can report max_volume_counts=0 before + // publishing concrete slot limits. Keep one provisional slot so EC + // detection still sees the disk for placement planning. + baseAvailable = 1 + } netImpact := at.getEffectiveCapacityUnsafe(disk) // Calculate available volume slots (negative impact reduces availability) diff --git a/weed/admin/topology/capacity_limits_test.go b/weed/admin/topology/capacity_limits_test.go new file mode 100644 index 000000000..203831e05 --- /dev/null +++ b/weed/admin/topology/capacity_limits_test.go @@ -0,0 +1,82 @@ +package topology + +import ( + "fmt" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +func TestGetDisksWithEffectiveCapacityNotCappedAtTenByLoad(t *testing.T) { + t.Parallel() + + activeTopology := NewActiveTopology(0) + if err := activeTopology.UpdateTopology(singleDiskTopologyInfoForCapacityTest()); err != nil { + t.Fatalf("UpdateTopology: %v", err) + } + + const pendingTasks = 32 + for i := 0; i < pendingTasks; i++ { + taskID := fmt.Sprintf("ec-capacity-%d", i) + err := activeTopology.AddPendingTask(TaskSpec{ + TaskID: taskID, + TaskType: TaskTypeErasureCoding, + VolumeID: uint32(i + 1), + VolumeSize: 1, + Sources: []TaskSourceSpec{ + { + ServerID: "node-a", + DiskID: 0, + StorageImpact: &StorageSlotChange{}, + }, + }, + Destinations: []TaskDestinationSpec{ + { + ServerID: "node-a", + DiskID: 0, + StorageImpact: &StorageSlotChange{}, + }, + }, + }) + if err != nil { + t.Fatalf("AddPendingTask(%s): %v", taskID, err) + } + } + + disks := activeTopology.GetDisksWithEffectiveCapacity(TaskTypeErasureCoding, "", 1) + if len(disks) != 1 { + t.Fatalf("expected disk to remain available after %d pending tasks, got %d", pendingTasks, len(disks)) + } + if disks[0].LoadCount != pendingTasks { + t.Fatalf("unexpected load count: got=%d want=%d", disks[0].LoadCount, pendingTasks) + } +} + +func singleDiskTopologyInfoForCapacityTest() *master_pb.TopologyInfo { + return &master_pb.TopologyInfo{ + Id: "topology-test", + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + Id: "dc1", + RackInfos: []*master_pb.RackInfo{ + { + Id: "rack1", + DataNodeInfos: []*master_pb.DataNodeInfo{ + { + Id: "node-a", + DiskInfos: map[string]*master_pb.DiskInfo{ + "hdd": { + DiskId: 0, + Type: "hdd", + VolumeCount: 0, + MaxVolumeCount: 1000, + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/weed/admin/topology/internal.go b/weed/admin/topology/internal.go index 72e37f6c1..96ef0ed20 100644 --- a/weed/admin/topology/internal.go +++ b/weed/admin/topology/internal.go @@ -68,7 +68,7 @@ func (at *ActiveTopology) assignTaskToDisk(task *taskState) { func (at *ActiveTopology) isDiskAvailable(disk *activeDisk, taskType TaskType) bool { // Check if disk has too many pending and active tasks activeLoad := len(disk.pendingTasks) + len(disk.assignedTasks) - if activeLoad >= MaxConcurrentTasksPerDisk { + if MaxConcurrentTasksPerDisk > 0 && activeLoad >= MaxConcurrentTasksPerDisk { return false } diff --git a/weed/admin/topology/storage_slot_test.go b/weed/admin/topology/storage_slot_test.go index 5a0ed3ce5..3bfd91bc9 100644 --- a/weed/admin/topology/storage_slot_test.go +++ b/weed/admin/topology/storage_slot_test.go @@ -317,6 +317,60 @@ func TestStorageSlotChangeCapacityCalculation(t *testing.T) { assert.Equal(t, int32(0), reservedShard, "Should show 0 reserved shard slots") } +func TestGetDisksWithEffectiveCapacity_UnknownEmptyDiskFallback(t *testing.T) { + activeTopology := NewActiveTopology(10) + + topologyInfo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + Id: "dc1", + RackInfos: []*master_pb.RackInfo{ + { + Id: "rack1", + DataNodeInfos: []*master_pb.DataNodeInfo{ + { + Id: "empty-node", + DiskInfos: map[string]*master_pb.DiskInfo{ + "hdd": { + DiskId: 0, + Type: "hdd", + VolumeCount: 0, + MaxVolumeCount: 0, + }, + }, + }, + { + Id: "used-node", + DiskInfos: map[string]*master_pb.DiskInfo{ + "hdd": { + DiskId: 0, + Type: "hdd", + VolumeCount: 1, + MaxVolumeCount: 0, + }, + }, + }, + }, + }, + }, + }, + }, + } + + err := activeTopology.UpdateTopology(topologyInfo) + assert.NoError(t, err) + + available := activeTopology.GetDisksWithEffectiveCapacity(TaskTypeErasureCoding, "", 1) + assert.Len(t, available, 1, "only the empty unknown-capacity disk should be treated as provisionally available") + if len(available) == 1 { + assert.Equal(t, "empty-node", available[0].NodeID) + assert.Equal(t, uint32(0), available[0].DiskID) + } + + assert.Equal(t, int64(1), activeTopology.GetEffectiveAvailableCapacity("empty-node", 0)) + assert.Equal(t, int64(0), activeTopology.GetEffectiveAvailableCapacity("used-node", 0)) +} + // TestECMultipleTargets demonstrates proper handling of EC operations with multiple targets func TestECMultipleTargets(t *testing.T) { activeTopology := NewActiveTopology(10) diff --git a/weed/admin/topology/types.go b/weed/admin/topology/types.go index 4b76de69f..054019271 100644 --- a/weed/admin/topology/types.go +++ b/weed/admin/topology/types.go @@ -26,17 +26,17 @@ const ( // Task and capacity management configuration constants const ( - // MaxConcurrentTasksPerDisk defines the maximum number of concurrent tasks per disk - // This prevents overloading a single disk with too many simultaneous operations - MaxConcurrentTasksPerDisk = 10 + // MaxConcurrentTasksPerDisk defines the maximum number of pending+assigned tasks per disk. + // Set to 0 to disable hard load capping and rely on effective capacity checks. + MaxConcurrentTasksPerDisk = 0 - // MaxTotalTaskLoadPerDisk defines the maximum total task load (pending + active) per disk - // This allows more tasks to be queued but limits the total pipeline depth - MaxTotalTaskLoadPerDisk = 20 + // MaxTotalTaskLoadPerDisk defines the maximum total planning load (pending + active) per disk. + // Set to 0 to disable hard load capping for planning. + MaxTotalTaskLoadPerDisk = 0 - // MaxTaskLoadForECPlacement defines the maximum task load to consider a disk for EC placement - // This threshold ensures disks aren't overloaded when planning EC operations - MaxTaskLoadForECPlacement = 10 + // MaxTaskLoadForECPlacement defines the maximum task load to consider a disk for EC placement. + // Set to 0 to disable this filter. + MaxTaskLoadForECPlacement = 0 ) // StorageSlotChange represents storage impact at both volume and shard levels diff --git a/weed/admin/view/app/ec_volume_details.templ b/weed/admin/view/app/ec_volume_details.templ index e09f20fe2..14c769d6c 100644 --- a/weed/admin/view/app/ec_volume_details.templ +++ b/weed/admin/view/app/ec_volume_details.templ @@ -3,6 +3,7 @@ package app import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) templ EcVolumeDetails(data dash.EcVolumeDetailsData) { @@ -61,11 +62,11 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) { if data.IsComplete { - Complete ({data.TotalShards}/14 shards) + Complete ({data.TotalShards}/{fmt.Sprintf("%d", erasure_coding.TotalShardsCount)} shards) } else { - Incomplete ({data.TotalShards}/14 shards) + Incomplete ({data.TotalShards}/{fmt.Sprintf("%d", erasure_coding.TotalShardsCount)} shards) } @@ -78,7 +79,7 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) { if i > 0 { , } - {fmt.Sprintf("%02d", shardID)} + @renderEcShardBadge(uint32(shardID), true) } @@ -145,14 +146,19 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) {
Present Shards:
for _, shard := range data.Shards { - {fmt.Sprintf("%02d", shard.ShardID)} + @renderEcShardBadge(shard.ShardID, false) }
+
+ Data + Parity + Data shards are blue, parity shards are yellow. +
if len(data.MissingShards) > 0 {
Missing Shards:
for _, shardID := range data.MissingShards { - {fmt.Sprintf("%02d", shardID)} + @renderEcShardBadge(uint32(shardID), true) }
} @@ -240,7 +246,7 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) { for _, shard := range data.Shards { - {fmt.Sprintf("%02d", shard.ShardID)} + @renderEcShardBadge(shard.ShardID, false) @@ -260,7 +266,7 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) { {bytesToHumanReadableUint64(shard.Size)} - + Volume Server @@ -298,6 +304,22 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) { } +templ renderEcShardBadge(shardID uint32, missing bool) { + if shardID < erasure_coding.DataShardsCount { + if missing { + { fmt.Sprintf("D%02d", shardID) } + } else { + { fmt.Sprintf("D%02d", shardID) } + } + } else { + if missing { + { fmt.Sprintf("P%02d", shardID) } + } else { + { fmt.Sprintf("P%02d", shardID) } + } + } +} + // Helper function to convert bytes to human readable format (uint64 version) func bytesToHumanReadableUint64(bytes uint64) string { const unit = 1024 @@ -310,4 +332,4 @@ func bytesToHumanReadableUint64(bytes uint64) string { exp++ } return fmt.Sprintf("%.1f%cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} \ No newline at end of file +} diff --git a/weed/admin/view/app/ec_volume_details_templ.go b/weed/admin/view/app/ec_volume_details_templ.go index a226b2634..3dda66e48 100644 --- a/weed/admin/view/app/ec_volume_details_templ.go +++ b/weed/admin/view/app/ec_volume_details_templ.go @@ -11,6 +11,7 @@ import templruntime "github.com/a-h/templ/runtime" import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component { @@ -41,7 +42,7 @@ func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component { var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.VolumeID)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 18, Col: 115} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 19, Col: 115} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) if templ_7745c5c3_Err != nil { @@ -54,7 +55,7 @@ func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.VolumeID)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 47, Col: 65} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 48, Col: 65} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { @@ -72,7 +73,7 @@ func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.Collection) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 53, Col: 80} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 54, Col: 80} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) if templ_7745c5c3_Err != nil { @@ -100,445 +101,585 @@ func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component { var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.TotalShards) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 64, Col: 100} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 65, Col: 100} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "/14 shards)") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "Incomplete (") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "/") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string - templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.TotalShards) + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", erasure_coding.TotalShardsCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 68, Col: 117} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 65, Col: 153} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "/14 shards)") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, " shards)") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if !data.IsComplete { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "Missing Shards:") + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "Incomplete (") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - for i, shardID := range data.MissingShards { - if i > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, ", ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%02d", shardID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 81, Col: 99} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.TotalShards) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 69, Col: 117} } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "Data Centers:") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for i, dc := range data.DataCenters { - if i > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, ", ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, " ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "/") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(dc) + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", erasure_coding.TotalShardsCount)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 93, Col: 70} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 69, Col: 170} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, " shards)") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "Servers:") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d servers", len(data.Servers))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 100, Col: 102} + if !data.IsComplete { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "Missing Shards:") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + for i, shardID := range data.MissingShards { + if i > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, ", ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = renderEcShardBadge(uint32(shardID), true).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "Data Centers:") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "Last Updated:") + for i, dc := range data.DataCenters { + if i > 0 { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, ", ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, " ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(dc) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 94, Col: 70} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "Servers:") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05")) + templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d servers", len(data.Servers))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 106, Col: 104} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 101, Col: 102} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
Shard Distribution

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "Last Updated:") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalShards)) + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05")) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 125, Col: 98} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 107, Col: 104} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "

Total Shards

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "

Shard Distribution

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var12 string - templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.DataCenters))) + templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalShards)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 131, Col: 103} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 126, Col: 98} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "

Data Centers

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "

Total Shards

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Servers))) + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.DataCenters))) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 137, Col: 96} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 132, Col: 103} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "

Servers
Present Shards:
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "Data Centers

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var14 string + templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Servers))) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 138, Col: 96} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "

Servers
Present Shards:
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, shard := range data.Shards { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var14 string - templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%02d", shard.ShardID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 148, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "") + templ_7745c5c3_Err = renderEcShardBadge(shard.ShardID, false).Render(ctx, templ_7745c5c3_Buffer) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
Data Parity Data shards are blue, parity shards are yellow.
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if len(data.MissingShards) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "
Missing Shards:
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "
Missing Shards:
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, shardID := range data.MissingShards { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var15 string - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%02d", shardID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 155, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") + templ_7745c5c3_Err = renderEcShardBadge(uint32(shardID), true).Render(ctx, templ_7745c5c3_Buffer) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "
Shard Details
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "
Shard Details
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if len(data.Shards) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "\">") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var24 string + templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("D%02d", shardID)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 310, Col: 142} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var26 string + templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("D%02d", shardID)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 312, Col: 123} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
Shard ID ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, shard := range data.Shards { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var22 string - templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(bytesToHumanReadableUint64(shard.Size)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 260, Col: 110} + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
Shard ID ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.SortBy == "shard_id" { if data.SortOrder == "asc" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "Server ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "Server ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.SortBy == "server" { if data.SortOrder == "asc" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "Data Center ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "Data Center ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.SortBy == "data_center" { if data.SortOrder == "asc" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "Rack ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "Rack ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } if data.SortBy == "rack" { if data.SortOrder == "asc" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "Disk TypeShard SizeActions
Disk TypeShard SizeActions
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = renderEcShardBadge(shard.ShardID, false).Render(ctx, templ_7745c5c3_Buffer) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var16 string - templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%02d", shard.ShardID)) + templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Server) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 243, Col: 110} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 253, Col: 81} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var17 templ.SafeURL - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL("/cluster/volume-servers/" + shard.Server)) + var templ_7745c5c3_Var17 string + templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(shard.DataCenter) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 246, Col: 106} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 257, Col: 103} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "\" class=\"text-primary text-decoration-none\">") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var18 string - templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Server) + templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Rack) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 247, Col: 81} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 260, Col: 99} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(shard.DataCenter) + templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(shard.DiskType) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 251, Col: 103} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 263, Col: 83} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var20 string - templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(shard.Rack) + templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(bytesToHumanReadableUint64(shard.Size)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 254, Col: 99} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 266, Col: 110} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "\" target=\"_blank\" rel=\"noopener noreferrer\" class=\"btn btn-sm btn-primary\">Volume Server
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
No EC shards found

This volume may not be EC encoded yet.

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +func renderEcShardBadge(shardID uint32, missing bool) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) + }() + } + ctx = templ.InitializeContext(ctx) + templ_7745c5c3_Var22 := templ.GetChildren(ctx) + if templ_7745c5c3_Var22 == nil { + templ_7745c5c3_Var22 = templ.NopComponent + } + ctx = templ.ClearChildren(ctx) + if shardID < erasure_coding.DataShardsCount { + if missing { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "		Volume Server
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
No EC shards found

This volume may not be EC encoded yet.

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err + if missing { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var28 string + templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("P%02d", shardID)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 316, Col: 154} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } else { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var30 string + templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("P%02d", shardID)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/ec_volume_details.templ`, Line: 318, Col: 135} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } return nil }) } diff --git a/weed/admin/view/app/maintenance_config.templ b/weed/admin/view/app/maintenance_config.templ deleted file mode 100644 index 4110486ba..000000000 --- a/weed/admin/view/app/maintenance_config.templ +++ /dev/null @@ -1,267 +0,0 @@ -package app - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" -) - -templ MaintenanceConfig(data *maintenance.MaintenanceConfigData) { -
-
-
-
-

- - Maintenance Configuration -

-
- - - Back to Queue - -
-
-
-
- -
-
-
-
-
System Settings
-
-
-
-
-
- - -
- - When enabled, the system will automatically scan for and execute maintenance tasks. - -
- -
- - - - How often to scan for maintenance tasks (1-1440 minutes). Default: 30 minutes - -
- -
- - - - How long to wait for worker heartbeat before considering it inactive (1-60 minutes). Default: 5 minutes - -
- -
- - - - Maximum time allowed for a single task to complete (1-24 hours). Default: 2 hours - -
- -
- - - - Maximum number of maintenance tasks that can run simultaneously across all workers (1-20). Default: 4 - -
- -
- - - - Default number of times to retry failed tasks (0-10). Default: 3 - -
- -
- - - - Time to wait before retrying failed tasks (1-120 minutes). Default: 15 minutes - -
- -
- - - - How long to keep completed/failed task records (1-30 days). Default: 7 days - -
- -
- - -
-
-
-
-
-
- - -
-
-
-
-
- - Task Configuration -
-
-
-

Configure specific settings for each maintenance task type.

-
- for _, menuItem := range data.MenuItems { - -
-
- - {menuItem.DisplayName} -
- if menuItem.IsEnabled { - Enabled - } else { - Disabled - } -
-

{menuItem.Description}

- - } -
-
-
-
-
- - -
-
-
-
-
System Statistics
-
-
-
-
-
-
Last Scan
-

{data.LastScanTime.Format("2006-01-02 15:04:05")}

-
-
-
-
-
Next Scan
-

{data.NextScanTime.Format("2006-01-02 15:04:05")}

-
-
-
-
-
Total Tasks
-

{fmt.Sprintf("%d", data.SystemStats.TotalTasks)}

-
-
-
-
-
Active Workers
-

{fmt.Sprintf("%d", data.SystemStats.ActiveWorkers)}

-
-
-
-
-
-
-
-
- - -} \ No newline at end of file diff --git a/weed/admin/view/app/maintenance_config_schema.templ b/weed/admin/view/app/maintenance_config_schema.templ deleted file mode 100644 index 5bd85b889..000000000 --- a/weed/admin/view/app/maintenance_config_schema.templ +++ /dev/null @@ -1,383 +0,0 @@ -package app - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" -) - -templ MaintenanceConfigSchema(data *maintenance.MaintenanceConfigData, schema *maintenance.MaintenanceConfigSchema) { -
-
-
-
-

- - Maintenance Configuration -

-
- - - View Tasks - -
-
-
-
- -
-
-
-
-
System Settings
-
-
-
- - for _, field := range schema.Fields { - @ConfigField(field, data.Config) - } - -
- - -
-
-
-
-
-
- - -
-
-
-
-
- - Volume Vacuum -
-
-
-

Reclaims disk space by removing deleted files from volumes.

- Configure -
-
-
-
-
-
-
- - Volume Balance -
-
-
-

Redistributes volumes across servers to optimize storage utilization.

- Configure -
-
-
-
-
-
-
- - Erasure Coding -
-
-
-

Converts volumes to erasure coded format for improved durability.

- Configure -
-
-
-
-
- - -} - -// ConfigField renders a single configuration field based on schema with typed value lookup -templ ConfigField(field *config.Field, config *maintenance.MaintenanceConfig) { - if field.InputType == "interval" { - -
- -
- - -
- if field.Description != "" { -
{ field.Description }
- } -
- } else if field.InputType == "checkbox" { - -
-
- - -
- if field.Description != "" { -
{ field.Description }
- } -
- } else { - -
- - - if field.Description != "" { -
{ field.Description }
- } -
- } -} - -// Helper functions for form field types - -func getNumberStep(field *config.Field) string { - if field.Type == config.FieldTypeFloat { - return "0.01" - } - return "1" -} - -// Typed field getters for MaintenanceConfig - no interface{} needed -func getMaintenanceInt32Field(config *maintenance.MaintenanceConfig, fieldName string) int32 { - if config == nil { - return 0 - } - - switch fieldName { - case "scan_interval_seconds": - return config.ScanIntervalSeconds - case "worker_timeout_seconds": - return config.WorkerTimeoutSeconds - case "task_timeout_seconds": - return config.TaskTimeoutSeconds - case "retry_delay_seconds": - return config.RetryDelaySeconds - case "max_retries": - return config.MaxRetries - case "cleanup_interval_seconds": - return config.CleanupIntervalSeconds - case "task_retention_seconds": - return config.TaskRetentionSeconds - case "global_max_concurrent": - if config.Policy != nil { - return config.Policy.GlobalMaxConcurrent - } - return 0 - default: - return 0 - } -} - -func getMaintenanceBoolField(config *maintenance.MaintenanceConfig, fieldName string) bool { - if config == nil { - return false - } - - switch fieldName { - case "enabled": - return config.Enabled - default: - return false - } -} - -// Helper function to convert schema to JSON for JavaScript -templ schemaToJSON(schema *maintenance.MaintenanceConfigSchema) { - {`{}`} -} \ No newline at end of file diff --git a/weed/admin/view/app/maintenance_config_schema_templ.go b/weed/admin/view/app/maintenance_config_schema_templ.go deleted file mode 100644 index b29f24644..000000000 --- a/weed/admin/view/app/maintenance_config_schema_templ.go +++ /dev/null @@ -1,622 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" -) - -func MaintenanceConfigSchema(data *maintenance.MaintenanceConfigData, schema *maintenance.MaintenanceConfigSchema) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Maintenance Configuration

View Tasks
System Settings
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, field := range schema.Fields { - templ_7745c5c3_Err = ConfigField(field, data.Config).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
Volume Vacuum

Reclaims disk space by removing deleted files from volumes.

Configure
Volume Balance

Redistributes volumes across servers to optimize storage utilization.

Configure
Erasure Coding

Converts volumes to erasure coded format for improved durability.

Configure
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -// ConfigField renders a single configuration field based on schema with typed value lookup -func ConfigField(field *config.Field, config *maintenance.MaintenanceConfig) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var2 := templ.GetChildren(ctx) - if templ_7745c5c3_Var2 == nil { - templ_7745c5c3_Var2 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - if field.InputType == "interval" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config_schema.templ`, Line: 269, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if field.InputType == "checkbox" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var15 string - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config_schema.templ`, Line: 290, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var25 string - templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config_schema.templ`, Line: 321, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - return nil - }) -} - -// Helper functions for form field types - -func getNumberStep(field *config.Field) string { - if field.Type == config.FieldTypeFloat { - return "0.01" - } - return "1" -} - -// Typed field getters for MaintenanceConfig - no interface{} needed -func getMaintenanceInt32Field(config *maintenance.MaintenanceConfig, fieldName string) int32 { - if config == nil { - return 0 - } - - switch fieldName { - case "scan_interval_seconds": - return config.ScanIntervalSeconds - case "worker_timeout_seconds": - return config.WorkerTimeoutSeconds - case "task_timeout_seconds": - return config.TaskTimeoutSeconds - case "retry_delay_seconds": - return config.RetryDelaySeconds - case "max_retries": - return config.MaxRetries - case "cleanup_interval_seconds": - return config.CleanupIntervalSeconds - case "task_retention_seconds": - return config.TaskRetentionSeconds - case "global_max_concurrent": - if config.Policy != nil { - return config.Policy.GlobalMaxConcurrent - } - return 0 - default: - return 0 - } -} - -func getMaintenanceBoolField(config *maintenance.MaintenanceConfig, fieldName string) bool { - if config == nil { - return false - } - - switch fieldName { - case "enabled": - return config.Enabled - default: - return false - } -} - -// Helper function to convert schema to JSON for JavaScript -func schemaToJSON(schema *maintenance.MaintenanceConfigSchema) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var26 := templ.GetChildren(ctx) - if templ_7745c5c3_Var26 == nil { - templ_7745c5c3_Var26 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - var templ_7745c5c3_Var27 string - templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(`{}`) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config_schema.templ`, Line: 382, Col: 9} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/maintenance_config_templ.go b/weed/admin/view/app/maintenance_config_templ.go deleted file mode 100644 index c5294cfd9..000000000 --- a/weed/admin/view/app/maintenance_config_templ.go +++ /dev/null @@ -1,284 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" -) - -func MaintenanceConfig(data *maintenance.MaintenanceConfigData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Maintenance Configuration

Back to Queue
System Settings
When enabled, the system will automatically scan for and execute maintenance tasks.
How often to scan for maintenance tasks (1-1440 minutes). Default: 30 minutes
How long to wait for worker heartbeat before considering it inactive (1-60 minutes). Default: 5 minutes
Maximum time allowed for a single task to complete (1-24 hours). Default: 2 hours
Maximum number of maintenance tasks that can run simultaneously across all workers (1-20). Default: 4
Default number of times to retry failed tasks (0-10). Default: 3
Time to wait before retrying failed tasks (1-120 minutes). Default: 15 minutes
How long to keep completed/failed task records (1-30 days). Default: 7 days
Task Configuration

Configure specific settings for each maintenance task type.

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, menuItem := range data.MenuItems { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var10 = []any{menuItem.Icon + " me-2"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var10...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var12 string - templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.DisplayName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 151, Col: 65} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if menuItem.IsEnabled { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "Enabled") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "Disabled") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 159, Col: 90} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
System Statistics
Last Scan

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var14 string - templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastScanTime.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 180, Col: 100} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "

Next Scan

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var15 string - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(data.NextScanTime.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 186, Col: 100} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "

Total Tasks

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var16 string - templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.SystemStats.TotalTasks)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 192, Col: 99} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "

Active Workers

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var17 string - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.SystemStats.ActiveWorkers)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_config.templ`, Line: 198, Col: 102} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/maintenance_queue.templ b/weed/admin/view/app/maintenance_queue.templ deleted file mode 100644 index 00650f80a..000000000 --- a/weed/admin/view/app/maintenance_queue.templ +++ /dev/null @@ -1,405 +0,0 @@ -package app - -import ( - "fmt" - "time" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" -) - -templ MaintenanceQueue(data *maintenance.MaintenanceQueueData) { -
- -
-
-
-

- - Maintenance Queue -

-
- - -
-
-
-
- - -
-
-
-
- -

{fmt.Sprintf("%d", data.Stats.PendingTasks)}

-

Pending Tasks

-
-
-
-
-
-
- -

{fmt.Sprintf("%d", data.Stats.RunningTasks)}

-

Running Tasks

-
-
-
-
-
-
- -

{fmt.Sprintf("%d", data.Stats.CompletedToday)}

-

Completed Today

-
-
-
-
-
-
- -

{fmt.Sprintf("%d", data.Stats.FailedToday)}

-

Failed Today

-
-
-
-
- - -
-
-
-
-
- - Completed Tasks -
-
-
- if data.Stats.CompletedToday == 0 && data.Stats.FailedToday == 0 { -
- -

No completed maintenance tasks today

- Completed tasks will appear here after workers finish processing them -
- } else { -
- - - - - - - - - - - - - for _, task := range data.Tasks { - if string(task.Status) == "completed" || string(task.Status) == "failed" || string(task.Status) == "cancelled" { - if string(task.Status) == "failed" { - - - - - - - - - } else { - - - - - - - - - } - } - } - -
TypeStatusVolumeWorkerDurationCompleted
- @TaskTypeIcon(task.Type) - {string(task.Type)} - @StatusBadge(task.Status){fmt.Sprintf("%d", task.VolumeID)} - if task.WorkerID != "" { - {task.WorkerID} - } else { - - - } - - if task.StartedAt != nil && task.CompletedAt != nil { - {formatDuration(task.CompletedAt.Sub(*task.StartedAt))} - } else { - - - } - - if task.CompletedAt != nil { - {task.CompletedAt.Format("2006-01-02 15:04")} - } else { - - - } -
- @TaskTypeIcon(task.Type) - {string(task.Type)} - @StatusBadge(task.Status){fmt.Sprintf("%d", task.VolumeID)} - if task.WorkerID != "" { - {task.WorkerID} - } else { - - - } - - if task.StartedAt != nil && task.CompletedAt != nil { - {formatDuration(task.CompletedAt.Sub(*task.StartedAt))} - } else { - - - } - - if task.CompletedAt != nil { - {task.CompletedAt.Format("2006-01-02 15:04")} - } else { - - - } -
-
- } -
-
-
-
- - -
-
-
-
-
- - Pending Tasks -
-
-
- if data.Stats.PendingTasks == 0 { -
- -

No pending maintenance tasks

- Pending tasks will appear here when the system detects maintenance needs -
- } else { -
- - - - - - - - - - - - - for _, task := range data.Tasks { - if string(task.Status) == "pending" { - - - - - - - - - } - } - -
TypePriorityVolumeServerReasonCreated
- @TaskTypeIcon(task.Type) - {string(task.Type)} - @PriorityBadge(task.Priority){fmt.Sprintf("%d", task.VolumeID)}{task.Server}{task.Reason}{task.CreatedAt.Format("2006-01-02 15:04")}
-
- } -
-
-
-
- - -
-
-
-
-
- - Active Tasks -
-
-
- if data.Stats.RunningTasks == 0 { -
- -

No active maintenance tasks

- Active tasks will appear here when workers start processing them -
- } else { -
- - - - - - - - - - - - - for _, task := range data.Tasks { - if string(task.Status) == "assigned" || string(task.Status) == "in_progress" { - - - - - - - - - } - } - -
TypeStatusProgressVolumeWorkerStarted
- @TaskTypeIcon(task.Type) - {string(task.Type)} - @StatusBadge(task.Status)@ProgressBar(task.Progress, task.Status){fmt.Sprintf("%d", task.VolumeID)} - if task.WorkerID != "" { - {task.WorkerID} - } else { - - - } - - if task.StartedAt != nil { - {task.StartedAt.Format("2006-01-02 15:04")} - } else { - - - } -
-
- } -
-
-
-
-
- - -} - -// Helper components -templ TaskTypeIcon(taskType maintenance.MaintenanceTaskType) { - -} - -templ PriorityBadge(priority maintenance.MaintenanceTaskPriority) { - switch priority { - case maintenance.PriorityCritical: - Critical - case maintenance.PriorityHigh: - High - case maintenance.PriorityNormal: - Normal - case maintenance.PriorityLow: - Low - default: - Unknown - } -} - -templ StatusBadge(status maintenance.MaintenanceTaskStatus) { - switch status { - case maintenance.TaskStatusPending: - Pending - case maintenance.TaskStatusAssigned: - Assigned - case maintenance.TaskStatusInProgress: - Running - case maintenance.TaskStatusCompleted: - Completed - case maintenance.TaskStatusFailed: - Failed - case maintenance.TaskStatusCancelled: - Cancelled - default: - Unknown - } -} - -templ ProgressBar(progress float64, status maintenance.MaintenanceTaskStatus) { - if status == maintenance.TaskStatusInProgress || status == maintenance.TaskStatusAssigned { -
-
-
-
- {fmt.Sprintf("%.1f%%", progress)} - } else if status == maintenance.TaskStatusCompleted { -
-
-
-
- 100% - } else { - - - } -} - -func formatDuration(d time.Duration) string { - if d < time.Minute { - return fmt.Sprintf("%.0fs", d.Seconds()) - } else if d < time.Hour { - return fmt.Sprintf("%.1fm", d.Minutes()) - } else { - return fmt.Sprintf("%.1fh", d.Hours()) - } -} - - \ No newline at end of file diff --git a/weed/admin/view/app/maintenance_queue_templ.go b/weed/admin/view/app/maintenance_queue_templ.go deleted file mode 100644 index 4fedbf704..000000000 --- a/weed/admin/view/app/maintenance_queue_templ.go +++ /dev/null @@ -1,860 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "time" -) - -func MaintenanceQueue(data *maintenance.MaintenanceQueueData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Maintenance Queue

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 string - templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Stats.PendingTasks)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 39, Col: 84} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

Pending Tasks

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var3 string - templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Stats.RunningTasks)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 48, Col: 84} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

Running Tasks

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Stats.CompletedToday)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 57, Col: 86} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "

Completed Today

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Stats.FailedToday)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 66, Col: 83} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "

Failed Today

Completed Tasks
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Stats.CompletedToday == 0 && data.Stats.FailedToday == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "

No completed maintenance tasks today

Completed tasks will appear here after workers finish processing them
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, task := range data.Tasks { - if string(task.Status) == "completed" || string(task.Status) == "failed" || string(task.Status) == "cancelled" { - if string(task.Status) == "failed" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "
TypeStatusVolumeWorkerDurationCompleted
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = TaskTypeIcon(task.Type).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(string(task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 110, Col: 78} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = StatusBadge(task.Status).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", task.VolumeID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 113, Col: 93} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.WorkerID != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(task.WorkerID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 116, Col: 85} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.StartedAt != nil && task.CompletedAt != nil { - var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(formatDuration(task.CompletedAt.Sub(*task.StartedAt))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 123, Col: 118} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.CompletedAt != nil { - var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(task.CompletedAt.Format("2006-01-02 15:04")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 130, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = TaskTypeIcon(task.Type).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(string(task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 140, Col: 78} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = StatusBadge(task.Status).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var14 string - templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", task.VolumeID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 143, Col: 93} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.WorkerID != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var15 string - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(task.WorkerID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 146, Col: 85} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.StartedAt != nil && task.CompletedAt != nil { - var templ_7745c5c3_Var16 string - templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(formatDuration(task.CompletedAt.Sub(*task.StartedAt))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 153, Col: 118} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.CompletedAt != nil { - var templ_7745c5c3_Var17 string - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(task.CompletedAt.Format("2006-01-02 15:04")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 160, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "
Pending Tasks
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Stats.PendingTasks == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "

No pending maintenance tasks

Pending tasks will appear here when the system detects maintenance needs
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, task := range data.Tasks { - if string(task.Status) == "pending" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "
TypePriorityVolumeServerReasonCreated
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = TaskTypeIcon(task.Type).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(string(task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 214, Col: 74} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = PriorityBadge(task.Priority).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var20 string - templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", task.VolumeID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 217, Col: 89} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var21 string - templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(task.Server) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 218, Col: 75} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var22 string - templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(task.Reason) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 219, Col: 75} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var23 string - templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(task.CreatedAt.Format("2006-01-02 15:04")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 220, Col: 98} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
Active Tasks
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Stats.RunningTasks == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "

No active maintenance tasks

Active tasks will appear here when workers start processing them
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, task := range data.Tasks { - if string(task.Status) == "assigned" || string(task.Status) == "in_progress" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
TypeStatusProgressVolumeWorkerStarted
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = TaskTypeIcon(task.Type).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var25 string - templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(string(task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 269, Col: 74} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = StatusBadge(task.Status).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = ProgressBar(task.Progress, task.Status).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var26 string - templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", task.VolumeID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 273, Col: 89} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.WorkerID != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var27 string - templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(task.WorkerID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 276, Col: 81} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if task.StartedAt != nil { - var templ_7745c5c3_Var28 string - templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(task.StartedAt.Format("2006-01-02 15:04")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 283, Col: 102} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -// Helper components -func TaskTypeIcon(taskType maintenance.MaintenanceTaskType) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var29 := templ.GetChildren(ctx) - if templ_7745c5c3_Var29 == nil { - templ_7745c5c3_Var29 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - var templ_7745c5c3_Var30 = []any{maintenance.GetTaskIcon(taskType) + " me-1"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var30...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -func PriorityBadge(priority maintenance.MaintenanceTaskPriority) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var32 := templ.GetChildren(ctx) - if templ_7745c5c3_Var32 == nil { - templ_7745c5c3_Var32 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - switch priority { - case maintenance.PriorityCritical: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "Critical") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.PriorityHigh: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "High") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.PriorityNormal: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "Normal") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.PriorityLow: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "Low") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - default: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "Unknown") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - return nil - }) -} - -func StatusBadge(status maintenance.MaintenanceTaskStatus) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var33 := templ.GetChildren(ctx) - if templ_7745c5c3_Var33 == nil { - templ_7745c5c3_Var33 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - switch status { - case maintenance.TaskStatusPending: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "Pending") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.TaskStatusAssigned: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "Assigned") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.TaskStatusInProgress: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "Running") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.TaskStatusCompleted: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "Completed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.TaskStatusFailed: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "Failed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - case maintenance.TaskStatusCancelled: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "Cancelled") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - default: - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "Unknown") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - return nil - }) -} - -func ProgressBar(progress float64, status maintenance.MaintenanceTaskStatus) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var34 := templ.GetChildren(ctx) - if templ_7745c5c3_Var34 == nil { - templ_7745c5c3_Var34 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - if status == maintenance.TaskStatusInProgress || status == maintenance.TaskStatusAssigned { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var36 string - templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f%%", progress)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_queue.templ`, Line: 383, Col: 66} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if status == maintenance.TaskStatusCompleted { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, "
100%") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - return nil - }) -} - -func formatDuration(d time.Duration) string { - if d < time.Minute { - return fmt.Sprintf("%.0fs", d.Seconds()) - } else if d < time.Hour { - return fmt.Sprintf("%.1fm", d.Minutes()) - } else { - return fmt.Sprintf("%.1fh", d.Hours()) - } -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/maintenance_workers.templ b/weed/admin/view/app/maintenance_workers.templ deleted file mode 100644 index e4af71470..000000000 --- a/weed/admin/view/app/maintenance_workers.templ +++ /dev/null @@ -1,343 +0,0 @@ -package app - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/dash" - "time" -) - -templ MaintenanceWorkers(data *dash.MaintenanceWorkersData) { -
-
-
-
-
-

Maintenance Workers

-

Monitor and manage maintenance workers

-
-
- Last updated: { data.LastUpdated.Format("2006-01-02 15:04:05") } -
-
-
-
- - -
-
-
-
-
-
-
- Total Workers -
-
{ fmt.Sprintf("%d", len(data.Workers)) }
-
-
- -
-
-
-
-
- -
-
-
-
-
-
- Active Workers -
-
- { fmt.Sprintf("%d", data.ActiveWorkers) } -
-
-
- -
-
-
-
-
- -
-
-
-
-
-
- Busy Workers -
-
- { fmt.Sprintf("%d", data.BusyWorkers) } -
-
-
- -
-
-
-
-
- -
-
-
-
-
-
- Total Load -
-
- { fmt.Sprintf("%d", data.TotalLoad) } -
-
-
- -
-
-
-
-
-
- - -
-
-
-
-
Worker Details
-
-
- if len(data.Workers) == 0 { -
- -
No Workers Found
-

No maintenance workers are currently registered.

-
- Tip: To start a worker, run: -
weed worker -admin=<admin_server> -capabilities=vacuum,ec,balance -
-
- } else { -
- - - - - - - - - - - - - - - - for _, worker := range data.Workers { - - - - - - - - - - - - } - -
Worker IDAddressStatusCapabilitiesLoadCurrent TasksPerformanceLast HeartbeatActions
- { worker.Worker.ID } - - { worker.Worker.Address } - - if worker.Worker.Status == "active" { - Active - } else if worker.Worker.Status == "busy" { - Busy - } else { - Inactive - } - -
- for _, capability := range worker.Worker.Capabilities { - { string(capability) } - } -
- 		-
- if worker.Worker.MaxConcurrent > 0 { -
- { fmt.Sprintf("%d/%d", worker.Worker.CurrentLoad, worker.Worker.MaxConcurrent) } -
- } else { -
0/0
- } -
- 		- { fmt.Sprintf("%d", len(worker.CurrentTasks)) } - - -
Completed: { fmt.Sprintf("%d", worker.Performance.TasksCompleted) }
-
Failed: { fmt.Sprintf("%d", worker.Performance.TasksFailed) }
-
Success Rate: { fmt.Sprintf("%.1f%%", worker.Performance.SuccessRate) }
- - 		- if time.Since(worker.Worker.LastHeartbeat) < 2*time.Minute { - - - { worker.Worker.LastHeartbeat.Format("15:04:05") } - - } else { - - - { worker.Worker.LastHeartbeat.Format("15:04:05") } - - } - -
- - if worker.Worker.Status == "active" { - - } -
-
-
- } -
-
-
-
-
- - - - - -} diff --git a/weed/admin/view/app/maintenance_workers_templ.go b/weed/admin/view/app/maintenance_workers_templ.go deleted file mode 100644 index 4232ba3dd..000000000 --- a/weed/admin/view/app/maintenance_workers_templ.go +++ /dev/null @@ -1,401 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/dash" - "time" -) - -func MaintenanceWorkers(data *dash.MaintenanceWorkersData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Maintenance Workers

Monitor and manage maintenance workers

Last updated: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 string - templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 19, Col: 112} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
Total Workers
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var3 string - templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Workers))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 35, Col: 122} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "
Active Workers
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.ActiveWorkers)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 54, Col: 75} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
Busy Workers
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.BusyWorkers)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 74, Col: 73} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
Total Load
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var6 string - templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalLoad)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 94, Col: 71} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Worker Details
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.Workers) == 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "
No Workers Found

No maintenance workers are currently registered.

Tip: To start a worker, run:
weed worker -admin=<admin_server> -capabilities=vacuum,ec,balance
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, worker := range data.Workers { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "
Worker IDAddressStatusCapabilitiesLoadCurrent TasksPerformanceLast HeartbeatActions
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(worker.Worker.ID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 144, Col: 76} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(worker.Worker.Address) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 147, Col: 81} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if worker.Worker.Status == "active" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "Active") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if worker.Worker.Status == "busy" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "Busy") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "Inactive") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, capability := range worker.Worker.Capabilities { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(string(capability)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 161, Col: 126} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if worker.Worker.MaxConcurrent > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d/%d", worker.Worker.CurrentLoad, worker.Worker.MaxConcurrent)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 173, Col: 142} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
0/0
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var14 string - templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(worker.CurrentTasks))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 181, Col: 97} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "
Completed: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var15 string - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", worker.Performance.TasksCompleted)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 185, Col: 122} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "
Failed: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var16 string - templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", worker.Performance.TasksFailed)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 186, Col: 116} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
Success Rate: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var17 string - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f%%", worker.Performance.SuccessRate)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 187, Col: 126} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if time.Since(worker.Worker.LastHeartbeat) < 2*time.Minute { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var18 string - templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(worker.Worker.LastHeartbeat.Format("15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 194, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(worker.Worker.LastHeartbeat.Format("15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/maintenance_workers.templ`, Line: 199, Col: 108} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if worker.Worker.Status == "active" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "
Worker Details
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/plugin.templ b/weed/admin/view/app/plugin.templ new file mode 100644 index 000000000..b7cdf60d7 --- /dev/null +++ b/weed/admin/view/app/plugin.templ @@ -0,0 +1,2876 @@ +package app + +templ Plugin(page string) { + {{ + currentPage := page + if currentPage == "" { + currentPage = "overview" + } + }} +
+
+
+
+
+

Workers

+

Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.

+
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
+
Workers
+

0

+
+
+
+
+
+
+
Active Jobs
+

0

+
+
+
+
+
+
+
Activities (recent)
+

0

+
+
+
+
+
+
+
+
+
Per Job Type Summary
+
+
+
+ + + + + + + + + + + +
Job TypeActive JobsRecent Activities
Loading...
+
+
+
+
+
+
+
+
+
+
Scheduler State
+ Per job type detection schedule and execution limits +
+
+
+ + + + + + + + + + + + + + + + + + +
Job TypeEnabledDetectorIn FlightNext DetectionIntervalExec GlobalExec/WorkerExecutor WorkersEffective Exec
Loading...
+
+
+
+
+
+
+
+
+
+
Workers
+
+
+
+ + + + + + + + + + + + +
WorkerAddressCapabilitiesLoad
Loading...
+
+
+
+
+
+
+ +
+
+
+
+
+
Job Type Configuration
+ Not loaded +
+
+
+
Selected Job Type
+
-
+
+ +
+
+
Descriptor
+ +
+
Select a job type to load schema and config.
+
+ +
+
Admin Config Form
+
+
No admin form loaded.
+
+
+ +
+
Worker Config Form
+
+
No worker form loaded.
+
+
+ +
+ + + +
+
+
+
+ +
+
+
+
Job Scheduling Settings
+
+
+
+
+ +
+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+
+ +
+
+
+
+
Run History
+ Keep last 10 success + last 10 errors +
+
+
+
+
Successful Runs
+
+ + + + + + + + + + + + +
TimeJob IDWorkerDuration
No data
+
+
+
+
Error Runs
+
+ + + + + + + + + + + + +
TimeJob IDWorkerError
No data
+
+
+
+
+
+
+ +
+
+
+
Detection Results
+
+
+
Run detection to see proposals.
+
+
+
+
+
+ +
+
+
+
+
+
Job Queue
+
+ States: pending/assigned/running +
+
+
+
+ + + + + + + + + + + + + + + +
Job IDTypeStateProgressWorkerUpdatedMessage
Loading...
+
+
+
+
+
+
+ +
+
+
+
+
+
Detection Jobs
+
Detection activities for selected job type
+
+
+
+ + + + + + + + + + + + + + + +
TimeJob TypeRequest IDWorkerStageSourceMessage
Loading...
+
+
+
+
+
+
+ +
+
+
+
+
+
Execution Jobs
+
+ +
+
+
+
+ + + + + + + + + + + + + + + +
Job IDTypeStateProgressWorkerUpdatedMessage
Loading...
+
+
+
+
+
+ +
+
+
+
+
Execution Activities
+ Non-detection events only +
+
+
+ + + + + + + + + + + + + + +
TimeJob TypeJob IDSourceStageMessage
Loading...
+
+
+
+
+
+
+
+ + + + + + +} diff --git a/weed/admin/view/app/plugin_templ.go b/weed/admin/view/app/plugin_templ.go new file mode 100644 index 000000000..99b89181b --- /dev/null +++ b/weed/admin/view/app/plugin_templ.go @@ -0,0 +1,57 @@ +// Code generated by templ - DO NOT EDIT. + +// templ: version: v0.3.977 +package app + +//lint:file-ignore SA4006 This context is only used if a nested component is present. + +import "github.com/a-h/templ" +import templruntime "github.com/a-h/templ/runtime" + +func Plugin(page string) templ.Component { + return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { + templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context + if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { + return templ_7745c5c3_CtxErr + } + templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) + if !templ_7745c5c3_IsBuffer { + defer func() { + templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) + if templ_7745c5c3_Err == nil { + templ_7745c5c3_Err = templ_7745c5c3_BufErr + } + }() + } + ctx = templ.InitializeContext(ctx) + templ_7745c5c3_Var1 := templ.GetChildren(ctx) + if templ_7745c5c3_Var1 == nil { + templ_7745c5c3_Var1 = templ.NopComponent + } + ctx = templ.ClearChildren(ctx) + currentPage := page + if currentPage == "" { + currentPage = "overview" + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Workers

Cluster-wide worker status, per-job configuration, detection, queue, and execution workflows.

Workers

0

Active Jobs

0

Activities (recent)

0

Per Job Type Summary
Job TypeActive JobsRecent Activities
Loading...
Scheduler State
Per job type detection schedule and execution limits
Job TypeEnabledDetectorIn FlightNext DetectionIntervalExec GlobalExec/WorkerExecutor WorkersEffective Exec
Loading...
Workers
WorkerAddressCapabilitiesLoad
Loading...
Job Type Configuration
Not loaded
Selected Job Type
-
Descriptor
Select a job type to load schema and config.
Admin Config Form
No admin form loaded.
Worker Config Form
No worker form loaded.
Job Scheduling Settings
Run History
Keep last 10 success + last 10 errors
Successful Runs
TimeJob IDWorkerDuration
No data
Error Runs
TimeJob IDWorkerError
No data
Detection Results
Run detection to see proposals.
Job Queue
States: pending/assigned/running
Job IDTypeStateProgressWorkerUpdatedMessage
Loading...
Detection Jobs
Detection activities for selected job type
TimeJob TypeRequest IDWorkerStageSourceMessage
Loading...
Execution Jobs
Job IDTypeStateProgressWorkerUpdatedMessage
Loading...
Execution Activities
Non-detection events only
TimeJob TypeJob IDSourceStageMessage
Loading...
Job Detail
Select a job to view details.
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + return nil + }) +} + +var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/task_config.templ b/weed/admin/view/app/task_config.templ deleted file mode 100644 index 41034cd09..000000000 --- a/weed/admin/view/app/task_config.templ +++ /dev/null @@ -1,160 +0,0 @@ -package app - -import ( - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" -) - -templ TaskConfig(data *maintenance.TaskConfigData) { -
-
-
-
-

- - {data.TaskName} Configuration -

-
- - - Back to Configuration - - - - View Queue - -
-
-
-
- -
-
-
-
-
- - {data.TaskName} Settings -
-
-
-

{data.Description}

- - -
-
- @templ.Raw(string(data.ConfigFormHTML)) -
- -
- -
- - - - - Cancel - -
-
-
-
-
-
- - -
-
-
-
-
- - Task Information -
-
-
-
-
-
Task Type
-

- {string(data.TaskType)} -

-
-
-
Display Name
-

{data.TaskName}

-
-
-
-
-
Description
-

{data.Description}

-
-
-
-
-
-
-
- - -} \ No newline at end of file diff --git a/weed/admin/view/app/task_config_schema.templ b/weed/admin/view/app/task_config_schema.templ deleted file mode 100644 index 1c3393854..000000000 --- a/weed/admin/view/app/task_config_schema.templ +++ /dev/null @@ -1,487 +0,0 @@ -package app - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "reflect" - "strings" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks" - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" -) - -// Helper function to convert task schema to JSON string -func taskSchemaToJSON(schema *tasks.TaskConfigSchema) string { - if schema == nil { - return "{}" - } - - data := map[string]interface{}{ - "fields": schema.Fields, - } - - jsonBytes, err := json.Marshal(data) - if err != nil { - return "{}" - } - - return string(jsonBytes) -} - -// Helper function to base64 encode the JSON to avoid HTML escaping issues -func taskSchemaToBase64JSON(schema *tasks.TaskConfigSchema) string { - jsonStr := taskSchemaToJSON(schema) - return base64.StdEncoding.EncodeToString([]byte(jsonStr)) -} - -templ TaskConfigSchema(data *maintenance.TaskConfigData, schema *tasks.TaskConfigSchema, config interface{}) { -
-
-
-
-

- - {schema.DisplayName} Configuration -

-
- - - Back to System Config - -
-
-
-
- - -
-
-
-
-
- - Task Configuration -
-

{schema.Description}

-
-
-
- - for _, field := range schema.Fields { - @TaskConfigField(field, config) - } - -
- - -
-
-
-
-
-
- - -
-
-
-
-
- - Important Notes -
-
-
- -
-
-
-
-
- - - - -
- - -} - -// TaskConfigField renders a single task configuration field based on schema with typed field lookup -templ TaskConfigField(field *config.Field, config interface{}) { - if field.InputType == "interval" { - -
- -
- - -
- if field.Description != "" { -
{ field.Description }
- } -
- } else if field.InputType == "checkbox" { - -
-
- - -
- if field.Description != "" { -
{ field.Description }
- } -
- } else if field.InputType == "text" { - -
- - - if field.Description != "" { -
{ field.Description }
- } -
- } else { - -
- - - if field.Description != "" { -
{ field.Description }
- } -
- } -} - -// Typed field getters for task configs - avoiding interface{} where possible -func getTaskConfigBoolField(config interface{}, fieldName string) bool { - switch fieldName { - case "enabled": - // Use reflection only for the common 'enabled' field in BaseConfig - if value := getTaskFieldValue(config, fieldName); value != nil { - if boolVal, ok := value.(bool); ok { - return boolVal - } - } - return false - default: - // For other boolean fields, use reflection - if value := getTaskFieldValue(config, fieldName); value != nil { - if boolVal, ok := value.(bool); ok { - return boolVal - } - } - return false - } -} - -func getTaskConfigInt32Field(config interface{}, fieldName string) int32 { - switch fieldName { - case "scan_interval_seconds", "max_concurrent": - // Common fields that should be int/int32 - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case int32: - return v - case int: - return int32(v) - case int64: - return int32(v) - } - } - return 0 - default: - // For other int fields, use reflection - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case int32: - return v - case int: - return int32(v) - case int64: - return int32(v) - case float64: - return int32(v) - } - } - return 0 - } -} - -func getTaskConfigFloatField(config interface{}, fieldName string) float64 { - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case float64: - return v - case float32: - return float64(v) - case int: - return float64(v) - case int32: - return float64(v) - case int64: - return float64(v) - } - } - return 0.0 -} - -func getTaskConfigStringField(config interface{}, fieldName string) string { - if value := getTaskFieldValue(config, fieldName); value != nil { - if strVal, ok := value.(string); ok { - return strVal - } - // Convert numbers to strings for form display - switch v := value.(type) { - case int: - return fmt.Sprintf("%d", v) - case int32: - return fmt.Sprintf("%d", v) - case int64: - return fmt.Sprintf("%d", v) - case float64: - return fmt.Sprintf("%.6g", v) - case float32: - return fmt.Sprintf("%.6g", v) - } - } - return "" -} - -func getTaskNumberStep(field *config.Field) string { - if field.Type == config.FieldTypeFloat { - return "any" - } - return "1" -} - -func getTaskFieldValue(config interface{}, fieldName string) interface{} { - if config == nil { - return nil - } - - // Use reflection to get the field value from the config struct - configValue := reflect.ValueOf(config) - if configValue.Kind() == reflect.Ptr { - configValue = configValue.Elem() - } - - if configValue.Kind() != reflect.Struct { - return nil - } - - configType := configValue.Type() - - for i := 0; i < configValue.NumField(); i++ { - field := configValue.Field(i) - fieldType := configType.Field(i) - - // Handle embedded structs recursively (before JSON tag check) - if field.Kind() == reflect.Struct && fieldType.Anonymous { - if value := getTaskFieldValue(field.Interface(), fieldName); value != nil { - return value - } - continue - } - - // Get JSON tag name - jsonTag := fieldType.Tag.Get("json") - if jsonTag == "" { - continue - } - - // Remove options like ",omitempty" - if commaIdx := strings.Index(jsonTag, ","); commaIdx > 0 { - jsonTag = jsonTag[:commaIdx] - } - - // Check if this is the field we're looking for - if jsonTag == fieldName { - return field.Interface() - } - } - - return nil -} - - \ No newline at end of file diff --git a/weed/admin/view/app/task_config_schema_templ.go b/weed/admin/view/app/task_config_schema_templ.go deleted file mode 100644 index fa03a3d3f..000000000 --- a/weed/admin/view/app/task_config_schema_templ.go +++ /dev/null @@ -1,948 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/config" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" - "github.com/seaweedfs/seaweedfs/weed/worker/tasks" - "reflect" - "strings" -) - -// Helper function to convert task schema to JSON string -func taskSchemaToJSON(schema *tasks.TaskConfigSchema) string { - if schema == nil { - return "{}" - } - - data := map[string]interface{}{ - "fields": schema.Fields, - } - - jsonBytes, err := json.Marshal(data) - if err != nil { - return "{}" - } - - return string(jsonBytes) -} - -// Helper function to base64 encode the JSON to avoid HTML escaping issues -func taskSchemaToBase64JSON(schema *tasks.TaskConfigSchema) string { - jsonStr := taskSchemaToJSON(schema) - return base64.StdEncoding.EncodeToString([]byte(jsonStr)) -} - -func TaskConfigSchema(data *maintenance.TaskConfigData, schema *tasks.TaskConfigSchema, config interface{}) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 = []any{schema.Icon + " me-2"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var2...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(schema.DisplayName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 47, Col: 43} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, " Configuration

Back to System Config
Task Configuration

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(schema.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 68, Col: 76} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, field := range schema.Fields { - templ_7745c5c3_Err = TaskConfigField(field, config).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
Important Notes
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if schema.TaskName == "vacuum" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "
Vacuum Operations:

Performance: Vacuum operations are I/O intensive and may impact cluster performance.

Safety: Only volumes meeting age and garbage thresholds will be processed.

Recommendation: Monitor cluster load and adjust concurrent limits accordingly.

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if schema.TaskName == "balance" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "
Balance Operations:

Performance: Volume balancing involves data movement and can impact cluster performance.

Safety: Requires adequate server count to ensure data safety during moves.

Recommendation: Run during off-peak hours to minimize impact on production workloads.

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if schema.TaskName == "erasure_coding" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "
Erasure Coding Operations:

Performance: Erasure coding is CPU and I/O intensive. Consider running during off-peak hours.

Durability: With ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var6 string - templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 118, Col: 170} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, " configuration, can tolerate up to ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", erasure_coding.ParityShardsCount)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 118, Col: 260} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, " shard failures.

Configuration: Fullness ratio should be between 0.5 and 1.0 (e.g., 0.90 for 90%).

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -// TaskConfigField renders a single task configuration field based on schema with typed field lookup -func TaskConfigField(field *config.Field, config interface{}) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var9 := templ.GetChildren(ctx) - if templ_7745c5c3_Var9 == nil { - templ_7745c5c3_Var9 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - if field.InputType == "interval" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var17 string - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 253, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if field.InputType == "checkbox" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var22 string - templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 275, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if field.InputType == "text" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var29 string - templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 299, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if field.Description != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var39 string - templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(field.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_schema.templ`, Line: 330, Col: 69} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - return nil - }) -} - -// Typed field getters for task configs - avoiding interface{} where possible -func getTaskConfigBoolField(config interface{}, fieldName string) bool { - switch fieldName { - case "enabled": - // Use reflection only for the common 'enabled' field in BaseConfig - if value := getTaskFieldValue(config, fieldName); value != nil { - if boolVal, ok := value.(bool); ok { - return boolVal - } - } - return false - default: - // For other boolean fields, use reflection - if value := getTaskFieldValue(config, fieldName); value != nil { - if boolVal, ok := value.(bool); ok { - return boolVal - } - } - return false - } -} - -func getTaskConfigInt32Field(config interface{}, fieldName string) int32 { - switch fieldName { - case "scan_interval_seconds", "max_concurrent": - // Common fields that should be int/int32 - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case int32: - return v - case int: - return int32(v) - case int64: - return int32(v) - } - } - return 0 - default: - // For other int fields, use reflection - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case int32: - return v - case int: - return int32(v) - case int64: - return int32(v) - case float64: - return int32(v) - } - } - return 0 - } -} - -func getTaskConfigFloatField(config interface{}, fieldName string) float64 { - if value := getTaskFieldValue(config, fieldName); value != nil { - switch v := value.(type) { - case float64: - return v - case float32: - return float64(v) - case int: - return float64(v) - case int32: - return float64(v) - case int64: - return float64(v) - } - } - return 0.0 -} - -func getTaskConfigStringField(config interface{}, fieldName string) string { - if value := getTaskFieldValue(config, fieldName); value != nil { - if strVal, ok := value.(string); ok { - return strVal - } - // Convert numbers to strings for form display - switch v := value.(type) { - case int: - return fmt.Sprintf("%d", v) - case int32: - return fmt.Sprintf("%d", v) - case int64: - return fmt.Sprintf("%d", v) - case float64: - return fmt.Sprintf("%.6g", v) - case float32: - return fmt.Sprintf("%.6g", v) - } - } - return "" -} - -func getTaskNumberStep(field *config.Field) string { - if field.Type == config.FieldTypeFloat { - return "any" - } - return "1" -} - -func getTaskFieldValue(config interface{}, fieldName string) interface{} { - if config == nil { - return nil - } - - // Use reflection to get the field value from the config struct - configValue := reflect.ValueOf(config) - if configValue.Kind() == reflect.Ptr { - configValue = configValue.Elem() - } - - if configValue.Kind() != reflect.Struct { - return nil - } - - configType := configValue.Type() - - for i := 0; i < configValue.NumField(); i++ { - field := configValue.Field(i) - fieldType := configType.Field(i) - - // Handle embedded structs recursively (before JSON tag check) - if field.Kind() == reflect.Struct && fieldType.Anonymous { - if value := getTaskFieldValue(field.Interface(), fieldName); value != nil { - return value - } - continue - } - - // Get JSON tag name - jsonTag := fieldType.Tag.Get("json") - if jsonTag == "" { - continue - } - - // Remove options like ",omitempty" - if commaIdx := strings.Index(jsonTag, ","); commaIdx > 0 { - jsonTag = jsonTag[:commaIdx] - } - - // Check if this is the field we're looking for - if jsonTag == fieldName { - return field.Interface() - } - } - - return nil -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/task_config_schema_test.go b/weed/admin/view/app/task_config_schema_test.go deleted file mode 100644 index a4e2a8bc4..000000000 --- a/weed/admin/view/app/task_config_schema_test.go +++ /dev/null @@ -1,232 +0,0 @@ -package app - -import ( - "testing" -) - -// Test structs that mirror the actual configuration structure -type TestBaseConfigForTemplate struct { - Enabled bool `json:"enabled"` - ScanIntervalSeconds int `json:"scan_interval_seconds"` - MaxConcurrent int `json:"max_concurrent"` -} - -type TestTaskConfigForTemplate struct { - TestBaseConfigForTemplate - TaskSpecificField float64 `json:"task_specific_field"` - AnotherSpecificField string `json:"another_specific_field"` -} - -func TestGetTaskFieldValue_EmbeddedStructFields(t *testing.T) { - config := &TestTaskConfigForTemplate{ - TestBaseConfigForTemplate: TestBaseConfigForTemplate{ - Enabled: true, - ScanIntervalSeconds: 2400, - MaxConcurrent: 5, - }, - TaskSpecificField: 0.18, - AnotherSpecificField: "test_value", - } - - // Test embedded struct fields - tests := []struct { - fieldName string - expectedValue interface{} - description string - }{ - {"enabled", true, "BaseConfig boolean field"}, - {"scan_interval_seconds", 2400, "BaseConfig integer field"}, - {"max_concurrent", 5, "BaseConfig integer field"}, - {"task_specific_field", 0.18, "Task-specific float field"}, - {"another_specific_field", "test_value", "Task-specific string field"}, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - result := getTaskFieldValue(config, test.fieldName) - - if result != test.expectedValue { - t.Errorf("Field %s: expected %v (%T), got %v (%T)", - test.fieldName, test.expectedValue, test.expectedValue, result, result) - } - }) - } -} - -func TestGetTaskFieldValue_NonExistentField(t *testing.T) { - config := &TestTaskConfigForTemplate{ - TestBaseConfigForTemplate: TestBaseConfigForTemplate{ - Enabled: true, - ScanIntervalSeconds: 1800, - MaxConcurrent: 3, - }, - } - - result := getTaskFieldValue(config, "non_existent_field") - - if result != nil { - t.Errorf("Expected nil for non-existent field, got %v", result) - } -} - -func TestGetTaskFieldValue_NilConfig(t *testing.T) { - var config *TestTaskConfigForTemplate = nil - - result := getTaskFieldValue(config, "enabled") - - if result != nil { - t.Errorf("Expected nil for nil config, got %v", result) - } -} - -func TestGetTaskFieldValue_EmptyStruct(t *testing.T) { - config := &TestTaskConfigForTemplate{} - - // Test that we can extract zero values - tests := []struct { - fieldName string - expectedValue interface{} - description string - }{ - {"enabled", false, "Zero value boolean"}, - {"scan_interval_seconds", 0, "Zero value integer"}, - {"max_concurrent", 0, "Zero value integer"}, - {"task_specific_field", 0.0, "Zero value float"}, - {"another_specific_field", "", "Zero value string"}, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - result := getTaskFieldValue(config, test.fieldName) - - if result != test.expectedValue { - t.Errorf("Field %s: expected %v (%T), got %v (%T)", - test.fieldName, test.expectedValue, test.expectedValue, result, result) - } - }) - } -} - -func TestGetTaskFieldValue_NonStructConfig(t *testing.T) { - var config interface{} = "not a struct" - - result := getTaskFieldValue(config, "enabled") - - if result != nil { - t.Errorf("Expected nil for non-struct config, got %v", result) - } -} - -func TestGetTaskFieldValue_PointerToStruct(t *testing.T) { - config := &TestTaskConfigForTemplate{ - TestBaseConfigForTemplate: TestBaseConfigForTemplate{ - Enabled: false, - ScanIntervalSeconds: 900, - MaxConcurrent: 2, - }, - TaskSpecificField: 0.35, - } - - // Test that pointers are handled correctly - enabledResult := getTaskFieldValue(config, "enabled") - if enabledResult != false { - t.Errorf("Expected false for enabled field, got %v", enabledResult) - } - - intervalResult := getTaskFieldValue(config, "scan_interval_seconds") - if intervalResult != 900 { - t.Errorf("Expected 900 for scan_interval_seconds field, got %v", intervalResult) - } -} - -func TestGetTaskFieldValue_FieldsWithJSONOmitempty(t *testing.T) { - // Test struct with omitempty tags - type TestConfigWithOmitempty struct { - TestBaseConfigForTemplate - OptionalField string `json:"optional_field,omitempty"` - } - - config := &TestConfigWithOmitempty{ - TestBaseConfigForTemplate: TestBaseConfigForTemplate{ - Enabled: true, - ScanIntervalSeconds: 1200, - MaxConcurrent: 4, - }, - OptionalField: "optional_value", - } - - // Test that fields with omitempty are still found - result := getTaskFieldValue(config, "optional_field") - if result != "optional_value" { - t.Errorf("Expected 'optional_value' for optional_field, got %v", result) - } - - // Test embedded fields still work - enabledResult := getTaskFieldValue(config, "enabled") - if enabledResult != true { - t.Errorf("Expected true for enabled field, got %v", enabledResult) - } -} - -func TestGetTaskFieldValue_DeepEmbedding(t *testing.T) { - // Test with multiple levels of embedding - type DeepBaseConfig struct { - DeepField string `json:"deep_field"` - } - - type MiddleConfig struct { - DeepBaseConfig - MiddleField int `json:"middle_field"` - } - - type TopConfig struct { - MiddleConfig - TopField bool `json:"top_field"` - } - - config := &TopConfig{ - MiddleConfig: MiddleConfig{ - DeepBaseConfig: DeepBaseConfig{ - DeepField: "deep_value", - }, - MiddleField: 123, - }, - TopField: true, - } - - // Test that deeply embedded fields are found - deepResult := getTaskFieldValue(config, "deep_field") - if deepResult != "deep_value" { - t.Errorf("Expected 'deep_value' for deep_field, got %v", deepResult) - } - - middleResult := getTaskFieldValue(config, "middle_field") - if middleResult != 123 { - t.Errorf("Expected 123 for middle_field, got %v", middleResult) - } - - topResult := getTaskFieldValue(config, "top_field") - if topResult != true { - t.Errorf("Expected true for top_field, got %v", topResult) - } -} - -// Benchmark to ensure performance is reasonable -func BenchmarkGetTaskFieldValue(b *testing.B) { - config := &TestTaskConfigForTemplate{ - TestBaseConfigForTemplate: TestBaseConfigForTemplate{ - Enabled: true, - ScanIntervalSeconds: 1800, - MaxConcurrent: 3, - }, - TaskSpecificField: 0.25, - AnotherSpecificField: "benchmark_test", - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Test both embedded and regular fields - _ = getTaskFieldValue(config, "enabled") - _ = getTaskFieldValue(config, "task_specific_field") - } -} diff --git a/weed/admin/view/app/task_config_templ.go b/weed/admin/view/app/task_config_templ.go deleted file mode 100644 index 0d2975b32..000000000 --- a/weed/admin/view/app/task_config_templ.go +++ /dev/null @@ -1,174 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" -) - -func TaskConfig(data *maintenance.TaskConfigData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 = []any{data.TaskIcon + " me-2"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var2...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.TaskName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 14, Col: 38} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, " Configuration

Back to Configuration View Queue
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 = []any{data.TaskIcon + " me-2"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var5...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.TaskName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 36, Col: 42} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, " Settings

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(data.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 40, Col: 68} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templ.Raw(string(data.ConfigFormHTML)).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "
Cancel
Task Information
Task Type

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(string(data.TaskType)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 85, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "

Display Name

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(data.TaskName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 90, Col: 62} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "

Description

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(data.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config.templ`, Line: 96, Col: 65} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/task_config_templ.templ b/weed/admin/view/app/task_config_templ.templ deleted file mode 100644 index 37d236868..000000000 --- a/weed/admin/view/app/task_config_templ.templ +++ /dev/null @@ -1,160 +0,0 @@ -package app - -import ( - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" -) - -// TaskConfigTemplData represents data for templ-based task configuration -type TaskConfigTemplData struct { - TaskType maintenance.MaintenanceTaskType - TaskName string - TaskIcon string - Description string - ConfigSections []components.ConfigSectionData -} - -templ TaskConfigTempl(data *TaskConfigTemplData) { -
-
-
-
-

- - {data.TaskName} Configuration -

-
- - - Back to Configuration - - - - View Queue - -
-
-
-
- -
-
- -
-
- -
- - for _, section := range data.ConfigSections { - @components.ConfigSection(section) - } - - -
-
-
-
-
-
- - -
-
- -
-
-
-
-
-
-
-
- - -} \ No newline at end of file diff --git a/weed/admin/view/app/task_config_templ_templ.go b/weed/admin/view/app/task_config_templ_templ.go deleted file mode 100644 index f0e9a19cb..000000000 --- a/weed/admin/view/app/task_config_templ_templ.go +++ /dev/null @@ -1,112 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/admin/view/components" -) - -// TaskConfigTemplData represents data for templ-based task configuration -type TaskConfigTemplData struct { - TaskType maintenance.MaintenanceTaskType - TaskName string - TaskIcon string - Description string - ConfigSections []components.ConfigSectionData -} - -func TaskConfigTempl(data *TaskConfigTemplData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 = []any{data.TaskIcon + " me-2"} - templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var2...) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(data.TaskName) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_templ.templ`, Line: 24, Col: 38} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, " Configuration

Back to Configuration View Queue
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Description) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_config_templ.templ`, Line: 44, Col: 37} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, section := range data.ConfigSections { - templ_7745c5c3_Err = components.ConfigSection(section).Render(ctx, templ_7745c5c3_Buffer) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - return nil - }) -} - -var _ = templruntime.GeneratedTemplate diff --git a/weed/admin/view/app/task_detail.templ b/weed/admin/view/app/task_detail.templ deleted file mode 100644 index 0e46c713d..000000000 --- a/weed/admin/view/app/task_detail.templ +++ /dev/null @@ -1,1161 +0,0 @@ -package app - -import ( - "fmt" - "sort" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" -) - -// sortedKeys returns the sorted keys for a string map -func sortedKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -templ TaskDetail(data *maintenance.TaskDetailData) { -
- -
-
-
-
- -

- - Task Detail: {data.Task.ID} -

-
-
- - -
-
-
-
- - -
-
-
-
-
- - Task Overview -
-
-
-
-
-
-
Task ID:
-
{data.Task.ID}
- -
Type:
-
- {string(data.Task.Type)} -
- -
Status:
-
- if data.Task.Status == maintenance.TaskStatusPending { - Pending - } else if data.Task.Status == maintenance.TaskStatusAssigned { - Assigned - } else if data.Task.Status == maintenance.TaskStatusInProgress { - In Progress - } else if data.Task.Status == maintenance.TaskStatusCompleted { - Completed - } else if data.Task.Status == maintenance.TaskStatusFailed { - Failed - } else if data.Task.Status == maintenance.TaskStatusCancelled { - Cancelled - } -
- -
Priority:
-
- if data.Task.Priority == maintenance.PriorityHigh { - High - } else if data.Task.Priority == maintenance.PriorityCritical { - Critical - } else if data.Task.Priority == maintenance.PriorityNormal { - Normal - } else { - Low - } -
- - if data.Task.Reason != "" { -
Reason:
-
- {data.Task.Reason} -
- } -
-
-
- -
-
- Task Timeline -
-
-
-
-
- -
-
-
- Created - {data.Task.CreatedAt.Format("01-02 15:04:05")} -
-
- -
-
- -
- if data.Task.StartedAt != nil { -
- } else { -
- } -
- Scheduled - {data.Task.ScheduledAt.Format("01-02 15:04:05")} -
-
- -
- if data.Task.StartedAt != nil { -
- -
- } else { -
- -
- } - if data.Task.CompletedAt != nil { -
- } else { -
- } -
- Started - - if data.Task.StartedAt != nil { - {data.Task.StartedAt.Format("01-02 15:04:05")} - } else { - — - } - -
-
- -
- if data.Task.CompletedAt != nil { -
- if data.Task.Status == maintenance.TaskStatusCompleted { - - } else if data.Task.Status == maintenance.TaskStatusFailed { - - } else { - - } -
- } else { -
- -
- } -
- - if data.Task.Status == maintenance.TaskStatusCompleted { - Completed - } else if data.Task.Status == maintenance.TaskStatusFailed { - Failed - } else if data.Task.Status == maintenance.TaskStatusCancelled { - Cancelled - } else { - Pending - } - - - if data.Task.CompletedAt != nil { - {data.Task.CompletedAt.Format("01-02 15:04:05")} - } else { - — - } - -
-
-
-
-
- - - if data.Task.WorkerID != "" { -
-
Worker:
-
{data.Task.WorkerID}
-
- } - -
- if data.Task.TypedParams != nil && data.Task.TypedParams.VolumeSize > 0 { -
Volume Size:
-
- {formatBytes(int64(data.Task.TypedParams.VolumeSize))} -
- } - - if data.Task.TypedParams != nil && data.Task.TypedParams.Collection != "" { -
Collection:
-
- {data.Task.TypedParams.Collection} -
- } - - if data.Task.TypedParams != nil && data.Task.TypedParams.DataCenter != "" { -
Data Center:
-
- {data.Task.TypedParams.DataCenter} -
- } - - if data.Task.Progress > 0 { -
Progress:
-
-
-
- {fmt.Sprintf("%.1f%%", data.Task.Progress)} -
-
-
- } -
-
-
- - - - if data.Task.DetailedReason != "" { -
-
-
Detailed Reason:
-

{data.Task.DetailedReason}

-
-
- } - - if data.Task.Error != "" { -
-
-
Error:
-
- {data.Task.Error} -
-
-
- } -
-
-
-
- - - if data.Task.TypedParams != nil { -
-
-
-
-
- - Task Configuration -
-
-
- - if len(data.Task.TypedParams.Sources) > 0 { -
-
- - Source Servers - {fmt.Sprintf("%d", len(data.Task.TypedParams.Sources))} -
-
-
- for i, source := range data.Task.TypedParams.Sources { -
- {fmt.Sprintf("#%d", i+1)} - {source.Node} -
- if source.DataCenter != "" { - - {source.DataCenter} - - } -
-
- if source.Rack != "" { - - {source.Rack} - - } -
-
- if source.VolumeId > 0 { - - Vol:{fmt.Sprintf("%d", source.VolumeId)} - - } -
-
- if len(source.ShardIds) > 0 { - - Shards: - for j, shardId := range source.ShardIds { - if j > 0 { - , - } - if shardId < erasure_coding.DataShardsCount { - {fmt.Sprintf("%d", shardId)} - } else { - {fmt.Sprintf("P%d", shardId-erasure_coding.DataShardsCount)} - } - } - - } -
-
- } -
-
-
- } - - - if len(data.Task.TypedParams.Sources) > 0 || len(data.Task.TypedParams.Targets) > 0 { -
- -
- Task: {string(data.Task.Type)} -
- } - - - if len(data.Task.TypedParams.Targets) > 0 { -
-
- - Target Servers - {fmt.Sprintf("%d", len(data.Task.TypedParams.Targets))} -
-
-
- for i, target := range data.Task.TypedParams.Targets { -
- {fmt.Sprintf("#%d", i+1)} - {target.Node} -
- if target.DataCenter != "" { - - {target.DataCenter} - - } -
-
- if target.Rack != "" { - - {target.Rack} - - } -
-
- if target.VolumeId > 0 { - - Vol:{fmt.Sprintf("%d", target.VolumeId)} - - } -
-
- if len(target.ShardIds) > 0 { - - Shards: - for j, shardId := range target.ShardIds { - if j > 0 { - , - } - if shardId < erasure_coding.DataShardsCount { - {fmt.Sprintf("%d", shardId)} - } else { - {fmt.Sprintf("P%d", shardId-erasure_coding.DataShardsCount)} - } - } - - } -
-
- } -
-
-
- } -
-
-
-
- } - - - if data.WorkerInfo != nil { -
-
-
-
-
- - Worker Information -
-
-
-
-
-
-
Worker ID:
-
{data.WorkerInfo.ID}
- -
Address:
-
{data.WorkerInfo.Address}
- -
Status:
-
- if data.WorkerInfo.Status == "active" { - Active - } else if data.WorkerInfo.Status == "busy" { - Busy - } else { - Inactive - } -
-
-
-
-
-
Last Heartbeat:
-
{data.WorkerInfo.LastHeartbeat.Format("2006-01-02 15:04:05")}
- -
Current Load:
-
{fmt.Sprintf("%d/%d", data.WorkerInfo.CurrentLoad, data.WorkerInfo.MaxConcurrent)}
- -
Capabilities:
-
- for _, capability := range data.WorkerInfo.Capabilities { - {string(capability)} - } -
-
-
-
-
-
-
-
- } - - - if len(data.AssignmentHistory) > 0 { -
-
-
-
-
- - Assignment History -
-
-
-
- - - - - - - - - - - - for _, assignment := range data.AssignmentHistory { - - - - - - - - } - -
Worker IDWorker AddressAssigned AtUnassigned AtReason
{assignment.WorkerID}{assignment.WorkerAddress}{assignment.AssignedAt.Format("2006-01-02 15:04:05")} - if assignment.UnassignedAt != nil { - {assignment.UnassignedAt.Format("2006-01-02 15:04:05")} - } else { - — - } - {assignment.Reason}
-
-
-
-
-
- } - - - if len(data.ExecutionLogs) > 0 { -
-
-
-
-
- - Execution Logs -
-
-
-
- - - - - - - - - - - for _, log := range data.ExecutionLogs { - - - - - - - } - -
TimestampLevelMessageDetails
{log.Timestamp.Format("15:04:05")} - if log.Level == "error" { - {log.Level} - } else if log.Level == "warn" { - {log.Level} - } else if log.Level == "info" { - {log.Level} - } else { - {log.Level} - } - {log.Message} - if log.Fields != nil && len(log.Fields) > 0 { - - for _, k := range sortedKeys(log.Fields) { - {k}={log.Fields[k]} - } - - } else if log.Progress != nil || log.Status != "" { - - if log.Progress != nil { - progress={fmt.Sprintf("%.0f%%", *log.Progress)} - } - if log.Status != "" { - status={log.Status} - } - - } else { - - - } -
-
-
-
-
-
- } - - - if len(data.RelatedTasks) > 0 { -
-
-
-
-
- - Related Tasks -
-
-
-
- - - - - - - - - - - - - for _, relatedTask := range data.RelatedTasks { - - - - - - - - - } - -
Task IDTypeStatusVolume IDServerCreated
- - {relatedTask.ID} - - {string(relatedTask.Type)} - if relatedTask.Status == maintenance.TaskStatusCompleted { - Completed - } else if relatedTask.Status == maintenance.TaskStatusFailed { - Failed - } else if relatedTask.Status == maintenance.TaskStatusInProgress { - In Progress - } else { - {string(relatedTask.Status)} - } - - if relatedTask.VolumeID != 0 { - {fmt.Sprintf("%d", relatedTask.VolumeID)} - } else { - - - } - - if relatedTask.Server != "" { - {relatedTask.Server} - } else { - - - } - {relatedTask.CreatedAt.Format("2006-01-02 15:04:05")}
-
-
-
-
-
- } - - -
-
-
-
-
- - Actions -
-
-
- if data.Task.Status == maintenance.TaskStatusPending || data.Task.Status == maintenance.TaskStatusAssigned { - - } - if data.Task.WorkerID != "" { - - } - -
-
-
-
-
- - - - - - - -} diff --git a/weed/admin/view/app/task_detail_templ.go b/weed/admin/view/app/task_detail_templ.go deleted file mode 100644 index ed53c28d6..000000000 --- a/weed/admin/view/app/task_detail_templ.go +++ /dev/null @@ -1,1628 +0,0 @@ -// Code generated by templ - DO NOT EDIT. - -// templ: version: v0.3.977 -package app - -//lint:file-ignore SA4006 This context is only used if a nested component is present. - -import "github.com/a-h/templ" -import templruntime "github.com/a-h/templ/runtime" - -import ( - "fmt" - "github.com/seaweedfs/seaweedfs/weed/admin/maintenance" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" - "sort" -) - -// sortedKeys returns the sorted keys for a string map -func sortedKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -func TaskDetail(data *maintenance.TaskDetailData) templ.Component { - return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { - templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context - if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { - return templ_7745c5c3_CtxErr - } - templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W) - if !templ_7745c5c3_IsBuffer { - defer func() { - templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer) - if templ_7745c5c3_Err == nil { - templ_7745c5c3_Err = templ_7745c5c3_BufErr - } - }() - } - ctx = templ.InitializeContext(ctx) - templ_7745c5c3_Var1 := templ.GetChildren(ctx) - if templ_7745c5c3_Var1 == nil { - templ_7745c5c3_Var1 = templ.NopComponent - } - ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "

Task Detail: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var2 string - templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.ID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 35, Col: 54} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

Task Overview
Task ID:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var3 string - templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.ID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 67, Col: 76} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "
Type:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(string(data.Task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 71, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "
Status:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Status == maintenance.TaskStatusPending { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "Pending") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusAssigned { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "Assigned") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusInProgress { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "In Progress") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusCompleted { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "Completed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusFailed { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "Failed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusCancelled { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "Cancelled") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "
Priority:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Priority == maintenance.PriorityHigh { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "High") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Priority == maintenance.PriorityCritical { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "Critical") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Priority == maintenance.PriorityNormal { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "Normal") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "Low") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Reason != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "
Reason:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.Reason) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 107, Col: 86} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "
Task Timeline
Created ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var6 string - templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.CreatedAt.Format("01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 127, Col: 131} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.StartedAt != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "
Scheduled ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.ScheduledAt.Format("01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 142, Col: 133} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.StartedAt != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.CompletedAt != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "
Started ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.StartedAt != nil { - var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.StartedAt.Format("01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 165, Col: 105} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "—") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.CompletedAt != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Status == maintenance.TaskStatusCompleted { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusFailed { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Status == maintenance.TaskStatusCompleted { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "Completed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusFailed { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "Failed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.Task.Status == maintenance.TaskStatusCancelled { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "Cancelled") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "Pending") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.CompletedAt != nil { - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.CompletedAt.Format("01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 203, Col: 107} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "—") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.WorkerID != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "
Worker:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.WorkerID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 218, Col: 86} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.TypedParams != nil && data.Task.TypedParams.VolumeSize > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
Volume Size:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(formatBytes(int64(data.Task.TypedParams.VolumeSize))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 226, Col: 128} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.TypedParams != nil && data.Task.TypedParams.Collection != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "
Collection:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var12 string - templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.TypedParams.Collection) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 233, Col: 139} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.TypedParams != nil && data.Task.TypedParams.DataCenter != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "
Data Center:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var13 string - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.TypedParams.DataCenter) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 240, Col: 146} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.Progress > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
Progress:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var16 string - templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f%%", data.Task.Progress)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 252, Col: 94} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.DetailedReason != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "
Detailed Reason:

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var17 string - templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.DetailedReason) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 267, Col: 83} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "

") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.Error != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
Error:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var18 string - templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(data.Task.Error) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 277, Col: 62} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.TypedParams != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "
Task Configuration
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.Task.TypedParams.Sources) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, "
Source Servers ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var19 string - templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Task.TypedParams.Sources))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 305, Col: 127} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for i, source := range data.Task.TypedParams.Sources { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var20 string - templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("#%d", i+1)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 311, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var21 string - templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(source.Node) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 312, Col: 54} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if source.DataCenter != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 71, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var22 string - templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinStringErrs(source.DataCenter) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 316, Col: 102} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 72, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 73, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if source.Rack != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 74, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var23 string - templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(source.Rack) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 323, Col: 94} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 75, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 76, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if source.VolumeId > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 77, "Vol:") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var24 string - templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", source.VolumeId)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 330, Col: 118} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 78, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 79, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(source.ShardIds) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 80, "Shards: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for j, shardId := range source.ShardIds { - if j > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 81, ", ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 82, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if shardId < erasure_coding.DataShardsCount { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 83, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var26 string - templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", shardId)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 343, Col: 202} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 85, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 86, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var28 string - templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("P%d", shardId-erasure_coding.DataShardsCount)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 345, Col: 246} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 88, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 90, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 91, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 92, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.Task.TypedParams.Sources) > 0 || len(data.Task.TypedParams.Targets) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 93, "

Task: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var29 string - templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(string(data.Task.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 363, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 94, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 95, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.Task.TypedParams.Targets) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "
Target Servers ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var30 string - templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Task.TypedParams.Targets))) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 373, Col: 130} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 97, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for i, target := range data.Task.TypedParams.Targets { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 98, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var31 string - templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("#%d", i+1)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 379, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 99, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var32 string - templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(target.Node) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 380, Col: 54} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 100, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if target.DataCenter != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 101, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var33 string - templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(target.DataCenter) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 384, Col: 102} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 102, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 103, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if target.Rack != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 104, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var34 string - templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(target.Rack) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 391, Col: 94} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 105, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 106, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if target.VolumeId > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 107, "Vol:") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var35 string - templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", target.VolumeId)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 398, Col: 118} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 108, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 109, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(target.ShardIds) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 110, "Shards: ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for j, shardId := range target.ShardIds { - if j > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 111, ", ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 112, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if shardId < erasure_coding.DataShardsCount { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 113, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var37 string - templ_7745c5c3_Var37, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", shardId)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 411, Col: 202} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var37)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 115, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 116, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var39 string - templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("P%d", shardId-erasure_coding.DataShardsCount)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 413, Col: 246} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 118, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 119, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 120, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 121, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 122, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 123, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.WorkerInfo != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 124, "
Worker Information
Worker ID:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var40 string - templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(data.WorkerInfo.ID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 447, Col: 86} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var40)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 125, "
Address:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var41 string - templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(data.WorkerInfo.Address) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 450, Col: 91} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 126, "
Status:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.WorkerInfo.Status == "active" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 127, "Active") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if data.WorkerInfo.Status == "busy" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 128, "Busy") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 129, "Inactive") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 130, "
Last Heartbeat:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var42 string - templ_7745c5c3_Var42, templ_7745c5c3_Err = templ.JoinStringErrs(data.WorkerInfo.LastHeartbeat.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 467, Col: 121} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var42)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 131, "
Current Load:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var43 string - templ_7745c5c3_Var43, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d/%d", data.WorkerInfo.CurrentLoad, data.WorkerInfo.MaxConcurrent)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 470, Col: 142} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var43)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 132, "
Capabilities:
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, capability := range data.WorkerInfo.Capabilities { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 133, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var44 string - templ_7745c5c3_Var44, templ_7745c5c3_Err = templ.JoinStringErrs(string(capability)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 475, Col: 100} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var44)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 134, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 135, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 136, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.AssignmentHistory) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 137, "
Assignment History
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, assignment := range data.AssignmentHistory { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 138, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 145, "
Worker IDWorker AddressAssigned AtUnassigned AtReason
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var45 string - templ_7745c5c3_Var45, templ_7745c5c3_Err = templ.JoinStringErrs(assignment.WorkerID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 513, Col: 78} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var45)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 139, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var46 string - templ_7745c5c3_Var46, templ_7745c5c3_Err = templ.JoinStringErrs(assignment.WorkerAddress) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 514, Col: 83} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var46)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 140, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var47 string - templ_7745c5c3_Var47, templ_7745c5c3_Err = templ.JoinStringErrs(assignment.AssignedAt.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 515, Col: 104} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var47)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 141, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if assignment.UnassignedAt != nil { - var templ_7745c5c3_Var48 string - templ_7745c5c3_Var48, templ_7745c5c3_Err = templ.JoinStringErrs(assignment.UnassignedAt.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 518, Col: 110} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var48)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 142, "—") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 143, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var49 string - templ_7745c5c3_Var49, templ_7745c5c3_Err = templ.JoinStringErrs(assignment.Reason) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 523, Col: 70} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var49)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 144, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 146, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.ExecutionLogs) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 147, "
Execution Logs
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, log := range data.ExecutionLogs { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 148, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 173, "
TimestampLevelMessageDetails
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var50 string - templ_7745c5c3_Var50, templ_7745c5c3_Err = templ.JoinStringErrs(log.Timestamp.Format("15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 560, Col: 92} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var50)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 149, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if log.Level == "error" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 150, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var51 string - templ_7745c5c3_Var51, templ_7745c5c3_Err = templ.JoinStringErrs(log.Level) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 563, Col: 96} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var51)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 151, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if log.Level == "warn" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 152, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var52 string - templ_7745c5c3_Var52, templ_7745c5c3_Err = templ.JoinStringErrs(log.Level) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 565, Col: 97} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var52)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 153, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if log.Level == "info" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 154, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var53 string - templ_7745c5c3_Var53, templ_7745c5c3_Err = templ.JoinStringErrs(log.Level) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 567, Col: 94} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var53)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 155, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 156, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var54 string - templ_7745c5c3_Var54, templ_7745c5c3_Err = templ.JoinStringErrs(log.Level) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 569, Col: 99} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var54)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 157, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 158, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var55 string - templ_7745c5c3_Var55, templ_7745c5c3_Err = templ.JoinStringErrs(log.Message) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 572, Col: 70} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var55)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 159, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if log.Fields != nil && len(log.Fields) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 160, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, k := range sortedKeys(log.Fields) { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 161, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var56 string - templ_7745c5c3_Var56, templ_7745c5c3_Err = templ.JoinStringErrs(k) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 577, Col: 110} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var56)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 162, "=") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var57 string - templ_7745c5c3_Var57, templ_7745c5c3_Err = templ.JoinStringErrs(log.Fields[k]) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 577, Col: 129} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var57)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 163, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 164, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if log.Progress != nil || log.Status != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 165, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if log.Progress != nil { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 166, "progress=") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var58 string - templ_7745c5c3_Var58, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.0f%%", *log.Progress)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 583, Col: 151} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var58)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 167, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if log.Status != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 168, "status=") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var59 string - templ_7745c5c3_Var59, templ_7745c5c3_Err = templ.JoinStringErrs(log.Status) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 586, Col: 118} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var59)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 169, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 170, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 171, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 172, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 174, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if len(data.RelatedTasks) > 0 { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 175, "
Related Tasks
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - for _, relatedTask := range data.RelatedTasks { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 176, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 193, "
Task IDTypeStatusVolume IDServerCreated
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var61 string - templ_7745c5c3_Var61, templ_7745c5c3_Err = templ.JoinStringErrs(relatedTask.ID) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 633, Col: 77} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var61)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 178, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var62 string - templ_7745c5c3_Var62, templ_7745c5c3_Err = templ.JoinStringErrs(string(relatedTask.Type)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 636, Col: 105} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var62)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 179, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if relatedTask.Status == maintenance.TaskStatusCompleted { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 180, "Completed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if relatedTask.Status == maintenance.TaskStatusFailed { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 181, "Failed") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else if relatedTask.Status == maintenance.TaskStatusInProgress { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 182, "In Progress") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 183, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var63 string - templ_7745c5c3_Var63, templ_7745c5c3_Err = templ.JoinStringErrs(string(relatedTask.Status)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 645, Col: 116} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var63)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 184, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 185, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if relatedTask.VolumeID != 0 { - var templ_7745c5c3_Var64 string - templ_7745c5c3_Var64, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", relatedTask.VolumeID)) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 650, Col: 96} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var64)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 186, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 187, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if relatedTask.Server != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 188, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var65 string - templ_7745c5c3_Var65, templ_7745c5c3_Err = templ.JoinStringErrs(relatedTask.Server) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 657, Col: 81} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var65)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 189, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 190, "-") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 191, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var66 string - templ_7745c5c3_Var66, templ_7745c5c3_Err = templ.JoinStringErrs(relatedTask.CreatedAt.Format("2006-01-02 15:04:05")) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/task_detail.templ`, Line: 662, Col: 111} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var66)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 192, "
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 194, "
Actions
") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - if data.Task.Status == maintenance.TaskStatusPending || data.Task.Status == maintenance.TaskStatusAssigned { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 195, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - if data.Task.WorkerID != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 197, " ") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 200, "
Task Logs

Fetching logs from worker...

Task: | Worker: | Entries:
Log Entries (Last 100) Newest entries first
")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		return nil
-	})
-}
-
-var _ = templruntime.GeneratedTemplate
diff --git a/weed/admin/view/layout/layout.templ b/weed/admin/view/layout/layout.templ
index 9c87e824d..0627b1b58 100644
--- a/weed/admin/view/layout/layout.templ
+++ b/weed/admin/view/layout/layout.templ
@@ -16,13 +16,14 @@ templ Layout(c *gin.Context, content templ.Component) {
   }
   csrfToken := c.GetString("csrf_token")
   
-  // Detect if we're on a configuration page to keep submenu expanded
   currentPath := c.Request.URL.Path
-  isConfigPage := strings.HasPrefix(currentPath, "/maintenance/config") || currentPath == "/config"
   
   // Detect if we're on a message queue page to keep submenu expanded
   isMQPage := strings.HasPrefix(currentPath, "/mq/")
 
+  // Detect if we're on plugin page.
+  isPluginPage := strings.HasPrefix(currentPath, "/plugin")
+
   // Detect if we're on a storage page to keep submenu expanded
   isStoragePage := strings.HasPrefix(currentPath, "/storage/volumes") || strings.HasPrefix(currentPath, "/storage/ec-shards") || strings.HasPrefix(currentPath, "/storage/collections")
   
@@ -258,75 +259,61 @@ templ Layout(c *gin.Context, content templ.Component) {
                     
 
                     

-                        MAINTENANCE
+                        WORKERS
                     

                     
    
                         
  • 
-                            if isConfigPage {
-                                
-                                    Configuration
-                                    
+                            if isPluginPage {
+                                
+                                    Workers
                                 
                             } else {
-                                
+                                    Workers
                                 
                             }
-                            if isConfigPage {
-                                
    
-                                    
-                                
    
-                            } else {
-                                
    
-                                    
-                                
    
-                            }
                         
    • 
                         
  • 
-                            if currentPath == "/maintenance" {
-                                
+                            if currentPath == "/plugin/detection" {
+                                
+                                    Job Detection
+                                
+                            } else {
+                                
+                                    Job Detection
+                                
+                            }
+                        
    • 
+                        
  • 
+                            if currentPath == "/plugin/queue" {
+                                
                                     Job Queue
                                 
                             } else {
-                                
+                                
                                     Job Queue
                                 
                             }
                         
    • 
                         
  • 
-                            if currentPath == "/maintenance/workers" {
-                                
-                                    Workers
+                            if currentPath == "/plugin/execution" {
+                                
+                                    Job Execution
                                 
                             } else {
-                                
-                                    Workers
+                                
+                                    Job Execution
+                                
+                            }
+                        
    • 
+                        
  • 
+                            if currentPath == "/plugin/configuration" {
+                                
+                                    Configuration
+                                
+                            } else {
+                                
+                                    Configuration
                                 
                             }
                         
    • 
diff --git a/weed/admin/view/layout/layout_templ.go b/weed/admin/view/layout/layout_templ.go
index ec7019879..ec42543d7 100644
--- a/weed/admin/view/layout/layout_templ.go
+++ b/weed/admin/view/layout/layout_templ.go
@@ -43,13 +43,14 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		}
 		csrfToken := c.GetString("csrf_token")
 
-		// Detect if we're on a configuration page to keep submenu expanded
 		currentPath := c.Request.URL.Path
-		isConfigPage := strings.HasPrefix(currentPath, "/maintenance/config") || currentPath == "/config"
 
 		// Detect if we're on a message queue page to keep submenu expanded
 		isMQPage := strings.HasPrefix(currentPath, "/mq/")
 
+		// Detect if we're on plugin page.
+		isPluginPage := strings.HasPrefix(currentPath, "/plugin")
+
 		// Detect if we're on a storage page to keep submenu expanded
 		isStoragePage := strings.HasPrefix(currentPath, "/storage/volumes") || strings.HasPrefix(currentPath, "/storage/ec-shards") || strings.HasPrefix(currentPath, "/storage/collections")
 
@@ -62,7 +63,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var2 string
 		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(csrfToken)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 38, Col: 47}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 39, Col: 47}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
 		if templ_7745c5c3_Err != nil {
@@ -75,7 +76,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var3 string
 		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(username)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 69, Col: 73}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 70, Col: 73}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
 		if templ_7745c5c3_Err != nil {
@@ -110,7 +111,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var6 string
 		templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 96, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 97, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
 		if templ_7745c5c3_Err != nil {
@@ -167,7 +168,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		var templ_7745c5c3_Var11 string
 		templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 121, Col: 207}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 122, Col: 207}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
 		if templ_7745c5c3_Err != nil {
@@ -251,243 +252,82 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
MAINTENANCE
  • ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "
WORKERS
")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "
  • ")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		if currentPath == "/plugin/execution" {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "Job Execution")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "Job Execution")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
  • ")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		if currentPath == "/plugin/configuration" {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "Configuration")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "Configuration")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "
    • ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -495,43 +335,43 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "
    © ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "
    © ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		var templ_7745c5c3_Var26 string
-		templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
+		var templ_7745c5c3_Var14 string
+		templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 350, Col: 60}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 337, Col: 60}
 		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, " SeaweedFS Admin v")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, " SeaweedFS Admin v")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		var templ_7745c5c3_Var27 string
-		templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
+		var templ_7745c5c3_Var15 string
+		templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 350, Col: 102}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 337, Col: 102}
 		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, " ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, " ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if !strings.Contains(version.VERSION, "enterprise") {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "• Enterprise Version Available")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "• Enterprise Version Available")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "
    ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -555,61 +395,61 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
 			}()
 		}
 		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var28 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var28 == nil {
-			templ_7745c5c3_Var28 = templ.NopComponent
+		templ_7745c5c3_Var16 := templ.GetChildren(ctx)
+		if templ_7745c5c3_Var16 == nil {
+			templ_7745c5c3_Var16 = templ.NopComponent
 		}
 		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		var templ_7745c5c3_Var29 string
-		templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(title)
+		var templ_7745c5c3_Var17 string
+		templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 378, Col: 17}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 365, Col: 17}
 		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " - Login
    ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, " - Login
    ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		var templ_7745c5c3_Var30 string
-		templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(title)
+		var templ_7745c5c3_Var18 string
+		templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(title)
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 392, Col: 57}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 379, Col: 57}
 		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "
    Please sign in to continue
    ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "
    Please sign in to continue
    ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if errorMessage != "" {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "
     ")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "
     ")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var31 string
-			templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
+			var templ_7745c5c3_Var19 string
+			templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 399, Col: 45}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 386, Col: 45}
 			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "
    ")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
    ")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "
     
     
    ")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "
     
     
    ")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
diff --git a/weed/admin/view/layout/menu_helper.go b/weed/admin/view/layout/menu_helper.go
deleted file mode 100644
index fc8954423..000000000
--- a/weed/admin/view/layout/menu_helper.go
+++ /dev/null
@@ -1,47 +0,0 @@
-package layout
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
-
-	// Import task packages to trigger their auto-registration
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
-)
-
-// MenuItemData represents a menu item
-type MenuItemData struct {
-	Name        string
-	URL         string
-	Icon        string
-	Description string
-}
-
-// GetConfigurationMenuItems returns the dynamic configuration menu items
-func GetConfigurationMenuItems() []*MenuItemData {
-	var menuItems []*MenuItemData
-
-	// Add system configuration item
-	menuItems = append(menuItems, &MenuItemData{
-		Name:        "System",
-		URL:         "/maintenance/config",
-		Icon:        "fas fa-cogs",
-		Description: "System-level configuration",
-	})
-
-	// Get all registered task types and add them as submenu items
-	registeredTypes := maintenance.GetRegisteredMaintenanceTaskTypes()
-
-	for _, taskType := range registeredTypes {
-		menuItem := &MenuItemData{
-			Name:        maintenance.GetTaskDisplayName(taskType),
-			URL:         "/maintenance/config/" + string(taskType),
-			Icon:        maintenance.GetTaskIcon(taskType),
-			Description: maintenance.GetTaskDescription(taskType),
-		}
-
-		menuItems = append(menuItems, menuItem)
-	}
-
-	return menuItems
-}
diff --git a/weed/command/admin.go b/weed/command/admin.go
index 4aa8e35b3..8f90b8f32 100644
--- a/weed/command/admin.go
+++ b/weed/command/admin.go
@@ -117,6 +117,12 @@ var cmdAdmin = &Command{
     - TLS is automatically used if certificates are configured
     - Workers fall back to insecure connections if TLS is unavailable
 
+  Plugin:
+    - Always enabled on the worker gRPC port
+    - Registers plugin.proto gRPC service on the same worker gRPC port
+    - External workers connect with: weed worker -admin=
+    - Persists plugin metadata under dataDir/plugin when dataDir is configured
+
   Configuration File:
     - The security.toml file is read from ".", "$HOME/.seaweedfs/", 
       "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
@@ -197,6 +203,7 @@ func runAdmin(cmd *Command, args []string) bool {
 	} else {
 		fmt.Printf("Authentication: Disabled\n")
 	}
+	fmt.Printf("Plugin: Enabled\n")
 
 	// Set up graceful shutdown
 	ctx, cancel := context.WithCancel(context.Background())
@@ -295,7 +302,7 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
 		r.StaticFS("/static", http.FS(staticFS))
 	}
 
-	// Create admin server
+	// Create admin server (plugin is always enabled)
 	adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort)
 
 	// Show discovered filers
diff --git a/weed/command/maintenance_capabilities.go b/weed/command/maintenance_capabilities.go
new file mode 100644
index 000000000..738538006
--- /dev/null
+++ b/weed/command/maintenance_capabilities.go
@@ -0,0 +1,48 @@
+package command
+
+import (
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
+)
+
+// parseCapabilities converts comma-separated legacy maintenance capabilities to task types.
+// This remains for mini-mode maintenance worker wiring.
+func parseCapabilities(capabilityStr string) []types.TaskType {
+	if capabilityStr == "" {
+		return nil
+	}
+
+	capabilityMap := map[string]types.TaskType{}
+
+	typesRegistry := tasks.GetGlobalTypesRegistry()
+	for taskType := range typesRegistry.GetAllDetectors() {
+		capabilityMap[strings.ToLower(string(taskType))] = taskType
+	}
+
+	if taskType, exists := capabilityMap["erasure_coding"]; exists {
+		capabilityMap["ec"] = taskType
+	}
+	if taskType, exists := capabilityMap["remote_upload"]; exists {
+		capabilityMap["remote"] = taskType
+	}
+	if taskType, exists := capabilityMap["fix_replication"]; exists {
+		capabilityMap["replication"] = taskType
+	}
+
+	var capabilities []types.TaskType
+	parts := strings.Split(capabilityStr, ",")
+
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if taskType, exists := capabilityMap[part]; exists {
+			capabilities = append(capabilities, taskType)
+		} else {
+			glog.Warningf("Unknown capability: %s", part)
+		}
+	}
+
+	return capabilities
+}
diff --git a/weed/command/mini.go b/weed/command/mini.go
index 6c8b758a7..bb7b6ebc3 100644
--- a/weed/command/mini.go
+++ b/weed/command/mini.go
@@ -13,11 +13,13 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"github.com/seaweedfs/seaweedfs/weed/worker"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
 
@@ -43,6 +45,7 @@ const (
 	defaultMiniVolumeSizeMB       = 128         // Default volume size for mini mode
 	maxVolumeSizeMB               = 1024        // Maximum volume size in MB (1GB)
 	GrpcPortOffset                = 10000       // Offset used to calculate gRPC port from HTTP port
+	defaultMiniPluginJobTypes     = "vacuum,volume_balance,erasure_coding"
 )
 
 var (
@@ -1028,6 +1031,7 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
 
 	// Start worker after admin server is ready
 	startMiniWorker()
+	startMiniPluginWorker(ctx)
 
 	// Wait for worker to be ready by polling its gRPC port
 	workerGrpcAddr := fmt.Sprintf("%s:%d", bindIp, *miniAdminOptions.grpcPort)
@@ -1165,6 +1169,62 @@ func startMiniWorker() {
 	glog.Infof("Maintenance worker %s started successfully", workerInstance.ID())
 }
 
+func startMiniPluginWorker(ctx context.Context) {
+	glog.Infof("Starting plugin worker for admin server")
+
+	adminAddr := fmt.Sprintf("%s:%d", *miniIp, *miniAdminOptions.port)
+	resolvedAdminAddr := resolvePluginWorkerAdminServer(adminAddr)
+	if resolvedAdminAddr != adminAddr {
+		glog.Infof("Resolved mini plugin worker admin endpoint: %s -> %s", adminAddr, resolvedAdminAddr)
+	}
+
+	workerDir := filepath.Join(*miniDataFolders, "plugin_worker")
+	if err := os.MkdirAll(workerDir, 0755); err != nil {
+		glog.Fatalf("Failed to create plugin worker directory: %v", err)
+	}
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.worker")
+
+	handlers, err := buildPluginWorkerHandlers(defaultMiniPluginJobTypes, grpcDialOption)
+	if err != nil {
+		glog.Fatalf("Failed to build mini plugin worker handlers: %v", err)
+	}
+
+	workerID, err := resolvePluginWorkerID("", workerDir)
+	if err != nil {
+		glog.Fatalf("Failed to resolve mini plugin worker ID: %v", err)
+	}
+
+	pluginRuntime, err := pluginworker.NewWorker(pluginworker.WorkerOptions{
+		AdminServer:             resolvedAdminAddr,
+		WorkerID:                workerID,
+		WorkerVersion:           version.Version(),
+		WorkerAddress:           *miniIp,
+		HeartbeatInterval:       15 * time.Second,
+		ReconnectDelay:          5 * time.Second,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 2,
+		GrpcDialOption:          grpcDialOption,
+		Handlers:                handlers,
+	})
+	if err != nil {
+		glog.Fatalf("Failed to create mini plugin worker: %v", err)
+	}
+
+	go func() {
+		runCtx := ctx
+		if runCtx == nil {
+			runCtx = context.Background()
+		}
+		if runErr := pluginRuntime.Run(runCtx); runErr != nil && runCtx.Err() == nil {
+			glog.Errorf("Mini plugin worker stopped with error: %v", runErr)
+		}
+	}()
+
+	glog.Infof("Plugin worker %s started successfully with job types: %s", workerID, defaultMiniPluginJobTypes)
+}
+
 const credentialsInstructionTemplate = `
   To create S3 credentials, you have two options:
 
diff --git a/weed/command/mini_plugin_test.go b/weed/command/mini_plugin_test.go
new file mode 100644
index 000000000..30aafcaf3
--- /dev/null
+++ b/weed/command/mini_plugin_test.go
@@ -0,0 +1,13 @@
+package command
+
+import "testing"
+
+func TestMiniDefaultPluginJobTypes(t *testing.T) {
+	jobTypes, err := parsePluginWorkerJobTypes(defaultMiniPluginJobTypes)
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err)
+	}
+	if len(jobTypes) != 3 {
+		t.Fatalf("expected mini default job types to include 3 handlers, got %v", jobTypes)
+	}
+}
diff --git a/weed/command/plugin_worker_test.go b/weed/command/plugin_worker_test.go
new file mode 100644
index 000000000..7c357cbc0
--- /dev/null
+++ b/weed/command/plugin_worker_test.go
@@ -0,0 +1,238 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestBuildPluginWorkerHandler(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+
+	handler, err := buildPluginWorkerHandler("vacuum", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(vacuum) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil handler")
+	}
+
+	handler, err = buildPluginWorkerHandler("", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(default) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil default handler")
+	}
+
+	handler, err = buildPluginWorkerHandler("volume_balance", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(volume_balance) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil volume_balance handler")
+	}
+
+	handler, err = buildPluginWorkerHandler("balance", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(balance alias) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil balance alias handler")
+	}
+
+	handler, err = buildPluginWorkerHandler("erasure_coding", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(erasure_coding) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil erasure_coding handler")
+	}
+
+	handler, err = buildPluginWorkerHandler("ec", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandler(ec alias) err = %v", err)
+	}
+	if handler == nil {
+		t.Fatalf("expected non-nil ec alias handler")
+	}
+
+	_, err = buildPluginWorkerHandler("unknown", dialOption)
+	if err == nil {
+		t.Fatalf("expected unsupported job type error")
+	}
+}
+
+func TestBuildPluginWorkerHandlers(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+
+	handlers, err := buildPluginWorkerHandlers("vacuum,volume_balance,erasure_coding", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandlers(list) err = %v", err)
+	}
+	if len(handlers) != 3 {
+		t.Fatalf("expected 3 handlers, got %d", len(handlers))
+	}
+
+	handlers, err = buildPluginWorkerHandlers("balance,ec,vacuum,balance", dialOption)
+	if err != nil {
+		t.Fatalf("buildPluginWorkerHandlers(aliases) err = %v", err)
+	}
+	if len(handlers) != 3 {
+		t.Fatalf("expected deduped 3 handlers, got %d", len(handlers))
+	}
+
+	_, err = buildPluginWorkerHandlers("unknown,vacuum", dialOption)
+	if err == nil {
+		t.Fatalf("expected unsupported job type error")
+	}
+}
+
+func TestParsePluginWorkerJobTypes(t *testing.T) {
+	jobTypes, err := parsePluginWorkerJobTypes("")
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(default) err = %v", err)
+	}
+	if len(jobTypes) != 1 || jobTypes[0] != "vacuum" {
+		t.Fatalf("expected default [vacuum], got %v", jobTypes)
+	}
+
+	jobTypes, err = parsePluginWorkerJobTypes(" volume_balance , ec , vacuum , volume_balance ")
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(list) err = %v", err)
+	}
+	if len(jobTypes) != 3 {
+		t.Fatalf("expected 3 deduped job types, got %d (%v)", len(jobTypes), jobTypes)
+	}
+	if jobTypes[0] != "volume_balance" || jobTypes[1] != "erasure_coding" || jobTypes[2] != "vacuum" {
+		t.Fatalf("unexpected parsed order %v", jobTypes)
+	}
+
+	if _, err = parsePluginWorkerJobTypes(" , "); err != nil {
+		t.Fatalf("expected empty list to resolve to default vacuum: %v", err)
+	}
+}
+
+func TestPluginWorkerDefaultJobTypes(t *testing.T) {
+	jobTypes, err := parsePluginWorkerJobTypes(defaultPluginWorkerJobTypes)
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(default setting) err = %v", err)
+	}
+	if len(jobTypes) != 3 {
+		t.Fatalf("expected default job types to include 3 handlers, got %v", jobTypes)
+	}
+}
+
+func TestResolvePluginWorkerID(t *testing.T) {
+	dir := t.TempDir()
+
+	explicit, err := resolvePluginWorkerID("worker-x", dir)
+	if err != nil {
+		t.Fatalf("resolvePluginWorkerID(explicit) err = %v", err)
+	}
+	if explicit != "worker-x" {
+		t.Fatalf("expected explicit id, got %q", explicit)
+	}
+
+	generated, err := resolvePluginWorkerID("", dir)
+	if err != nil {
+		t.Fatalf("resolvePluginWorkerID(generate) err = %v", err)
+	}
+	if generated == "" {
+		t.Fatalf("expected generated id")
+	}
+	if len(generated) < 7 || generated[:7] != "plugin-" {
+		t.Fatalf("expected generated id prefix plugin-, got %q", generated)
+	}
+
+	persistedPath := filepath.Join(dir, "plugin.worker.id")
+	if _, statErr := os.Stat(persistedPath); statErr != nil {
+		t.Fatalf("expected persisted worker id file: %v", statErr)
+	}
+
+	reused, err := resolvePluginWorkerID("", dir)
+	if err != nil {
+		t.Fatalf("resolvePluginWorkerID(reuse) err = %v", err)
+	}
+	if reused != generated {
+		t.Fatalf("expected reused id %q, got %q", generated, reused)
+	}
+}
+
+func TestParsePluginWorkerAdminAddress(t *testing.T) {
+	host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress("localhost:23646")
+	if err != nil {
+		t.Fatalf("parsePluginWorkerAdminAddress(localhost:23646) err = %v", err)
+	}
+	if host != "localhost" || httpPort != 23646 || hasExplicitGrpcPort {
+		t.Fatalf("unexpected parse result: host=%q httpPort=%d hasExplicit=%v", host, httpPort, hasExplicitGrpcPort)
+	}
+
+	host, httpPort, hasExplicitGrpcPort, err = parsePluginWorkerAdminAddress("localhost:23646.33646")
+	if err != nil {
+		t.Fatalf("parsePluginWorkerAdminAddress(localhost:23646.33646) err = %v", err)
+	}
+	if host != "localhost" || httpPort != 23646 || !hasExplicitGrpcPort {
+		t.Fatalf("unexpected dotted parse result: host=%q httpPort=%d hasExplicit=%v", host, httpPort, hasExplicitGrpcPort)
+	}
+
+	if _, _, _, err = parsePluginWorkerAdminAddress("localhost"); err == nil {
+		t.Fatalf("expected parse error for invalid address")
+	}
+}
+
+func TestResolvePluginWorkerAdminServerUsesStatusGrpcPort(t *testing.T) {
+	const grpcPort = 35432
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/plugin/status" {
+			http.NotFound(w, r)
+			return
+		}
+		_, _ = w.Write([]byte(fmt.Sprintf(`{"worker_grpc_port":%d}`, grpcPort)))
+	}))
+	defer server.Close()
+
+	adminAddress := strings.TrimPrefix(server.URL, "http://")
+	host, httpPort, _, err := parsePluginWorkerAdminAddress(adminAddress)
+	if err != nil {
+		t.Fatalf("parsePluginWorkerAdminAddress(%s) err = %v", adminAddress, err)
+	}
+
+	resolved := resolvePluginWorkerAdminServer(adminAddress)
+	expected := fmt.Sprintf("%s:%d.%d", host, httpPort, grpcPort)
+	if resolved != expected {
+		t.Fatalf("unexpected resolved admin address: got=%q want=%q", resolved, expected)
+	}
+}
+
+func TestResolvePluginWorkerAdminServerKeepsDefaultGrpcOffset(t *testing.T) {
+	var server *httptest.Server
+	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/api/plugin/status" {
+			http.NotFound(w, r)
+			return
+		}
+		address := strings.TrimPrefix(server.URL, "http://")
+		_, httpPort, _, parseErr := parsePluginWorkerAdminAddress(address)
+		if parseErr != nil {
+			http.Error(w, parseErr.Error(), http.StatusInternalServerError)
+			return
+		}
+		_, _ = w.Write([]byte(fmt.Sprintf(`{"worker_grpc_port":%d}`, httpPort+10000)))
+	}))
+	defer server.Close()
+
+	adminAddress := strings.TrimPrefix(server.URL, "http://")
+	resolved := resolvePluginWorkerAdminServer(adminAddress)
+	if resolved != adminAddress {
+		t.Fatalf("expected admin address to remain unchanged, got=%q want=%q", resolved, adminAddress)
+	}
+}
diff --git a/weed/command/worker.go b/weed/command/worker.go
index 16c14c738..685e79780 100644
--- a/weed/command/worker.go
+++ b/weed/command/worker.go
@@ -1,76 +1,54 @@
 package command
 
 import (
-	"net/http"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"strings"
-	"syscall"
 	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-	statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
-	"github.com/seaweedfs/seaweedfs/weed/worker"
-	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
-	"github.com/seaweedfs/seaweedfs/weed/worker/types"
-
-	// Import task packages to trigger their auto-registration
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
-	_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
-
-	// TODO: Implement additional task packages (add to default capabilities when ready):
-	// _ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/remote" - for uploading volumes to remote/cloud storage
-	// _ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/replication" - for fixing replication issues and maintaining data consistency
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var cmdWorker = &Command{
-	UsageLine: "worker -admin= [-capabilities=] [-maxConcurrent=] [-workingDir=] [-metricsPort=] [-debug]",
-	Short:     "start a maintenance worker to process cluster maintenance tasks",
-	Long: `Start a maintenance worker that connects to an admin server to process
-maintenance tasks like vacuum, erasure coding, remote upload, and replication fixes.
+	UsageLine: "worker -admin= [-id=] [-jobType=vacuum,volume_balance,erasure_coding] [-workingDir=] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=] [-metricsIp=] [-debug]",
+	Short:     "start a plugin.proto worker process",
+	Long: `Start an external plugin worker using weed/pb/plugin.proto over gRPC.
 
-The worker ID and address are automatically generated.
-The worker connects to the admin server via gRPC (admin HTTP port + 10000).
+This command provides vacuum, volume_balance, and erasure_coding job type
+contracts with the plugin stream runtime, including descriptor delivery,
+heartbeat/load reporting, detection, and execution.
+
+Behavior:
+  - Use -jobType to choose one or more plugin job handlers (comma-separated list)
+  - Use -workingDir to persist plugin.worker.id for stable worker identity across restarts
+  - Use -metricsPort/-metricsIp to expose /health, /ready, and /metrics
 
 Examples:
   weed worker -admin=localhost:23646
-  weed worker -admin=admin.example.com:23646
-  weed worker -admin=localhost:23646 -capabilities=vacuum,replication
-  weed worker -admin=localhost:23646 -maxConcurrent=4
-  weed worker -admin=localhost:23646 -workingDir=/tmp/worker
-  weed worker -admin=localhost:23646 -metricsPort=9327
-  weed worker -admin=localhost:23646 -debug -debug.port=6060
+  weed worker -admin=localhost:23646 -jobType=volume_balance
+  weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance
+  weed worker -admin=localhost:23646 -jobType=erasure_coding
+  weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s
+  weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin
+  weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0
 `,
 }
 
 var (
-	workerAdminServer         = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address")
-	workerCapabilities        = cmdWorker.Flag.String("capabilities", "vacuum,ec,balance", "comma-separated list of task types this worker can handle")
-	workerMaxConcurrent       = cmdWorker.Flag.Int("maxConcurrent", 2, "maximum number of concurrent tasks")
-	workerHeartbeatInterval   = cmdWorker.Flag.Duration("heartbeat", 30*time.Second, "heartbeat interval")
-	workerTaskRequestInterval = cmdWorker.Flag.Duration("taskInterval", 5*time.Second, "task request interval")
-	workerWorkingDir          = cmdWorker.Flag.String("workingDir", "", "working directory for the worker")
-	workerMetricsPort         = cmdWorker.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
-	workerMetricsIp           = cmdWorker.Flag.String("metricsIp", "0.0.0.0", "Prometheus metrics listen IP")
-	workerDebug               = cmdWorker.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
-	workerDebugPort           = cmdWorker.Flag.Int("debug.port", 6060, "http port for debugging")
-
-	workerServerHeader = "SeaweedFS Worker " + version.VERSION
+	workerAdminServer = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address")
+	workerID          = cmdWorker.Flag.String("id", "", "worker ID (auto-generated when empty)")
+	workerWorkingDir  = cmdWorker.Flag.String("workingDir", "", "working directory for persistent worker state")
+	workerJobType     = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types to serve (comma-separated list)")
+	workerHeartbeat   = cmdWorker.Flag.Duration("heartbeat", 15*time.Second, "heartbeat interval")
+	workerReconnect   = cmdWorker.Flag.Duration("reconnect", 5*time.Second, "reconnect delay")
+	workerMaxDetect   = cmdWorker.Flag.Int("maxDetect", 1, "max concurrent detection requests")
+	workerMaxExecute  = cmdWorker.Flag.Int("maxExecute", 4, "max concurrent execute requests")
+	workerAddress     = cmdWorker.Flag.String("address", "", "worker address advertised to admin")
+	workerMetricsPort = cmdWorker.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
+	workerMetricsIp   = cmdWorker.Flag.String("metricsIp", "0.0.0.0", "Prometheus metrics listen IP")
+	workerDebug       = cmdWorker.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
+	workerDebugPort   = cmdWorker.Flag.Int("debug.port", 6060, "http port for debugging")
 )
 
 func init() {
 	cmdWorker.Run = runWorker
-
-	// Set default capabilities from registered task types
-	// This happens after package imports have triggered auto-registration
-	tasks.SetDefaultCapabilitiesFromRegistry()
 }
 
 func runWorker(cmd *Command, args []string) bool {
@@ -78,218 +56,17 @@ func runWorker(cmd *Command, args []string) bool {
 		grace.StartDebugServer(*workerDebugPort)
 	}
 
-	util.LoadConfiguration("security", false)
-
-	glog.Infof("Starting maintenance worker")
-	glog.Infof("Admin server: %s", *workerAdminServer)
-	glog.Infof("Capabilities: %s", *workerCapabilities)
-
-	// Parse capabilities
-	capabilities := parseCapabilities(*workerCapabilities)
-	if len(capabilities) == 0 {
-		glog.Fatalf("No valid capabilities specified")
-		return false
-	}
-
-	// Set working directory and create task-specific subdirectories
-	var baseWorkingDir string
-	if *workerWorkingDir != "" {
-		glog.Infof("Setting working directory to: %s", *workerWorkingDir)
-		if err := os.Chdir(*workerWorkingDir); err != nil {
-			glog.Fatalf("Failed to change working directory: %v", err)
-			return false
-		}
-		wd, err := os.Getwd()
-		if err != nil {
-			glog.Fatalf("Failed to get working directory: %v", err)
-			return false
-		}
-		baseWorkingDir = wd
-		glog.Infof("Current working directory: %s", baseWorkingDir)
-	} else {
-		// Use default working directory when not specified
-		wd, err := os.Getwd()
-		if err != nil {
-			glog.Fatalf("Failed to get current working directory: %v", err)
-			return false
-		}
-		baseWorkingDir = wd
-		glog.Infof("Using current working directory: %s", baseWorkingDir)
-	}
-
-	// Create task-specific subdirectories
-	for _, capability := range capabilities {
-		taskDir := filepath.Join(baseWorkingDir, string(capability))
-		if err := os.MkdirAll(taskDir, 0755); err != nil {
-			glog.Fatalf("Failed to create task directory %s: %v", taskDir, err)
-			return false
-		}
-		glog.Infof("Created task directory: %s", taskDir)
-	}
-
-	// Create gRPC dial option using TLS configuration
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.worker")
-
-	// Create worker configuration
-	config := &types.WorkerConfig{
-		AdminServer:         *workerAdminServer,
-		Capabilities:        capabilities,
-		MaxConcurrent:       *workerMaxConcurrent,
-		HeartbeatInterval:   *workerHeartbeatInterval,
-		TaskRequestInterval: *workerTaskRequestInterval,
-		BaseWorkingDir:      baseWorkingDir,
-		GrpcDialOption:      grpcDialOption,
-	}
-
-	// Create worker instance
-	workerInstance, err := worker.NewWorker(config)
-	if err != nil {
-		glog.Fatalf("Failed to create worker: %v", err)
-		return false
-	}
-	adminClient, err := worker.CreateAdminClient(*workerAdminServer, workerInstance.ID(), grpcDialOption)
-	if err != nil {
-		glog.Fatalf("Failed to create admin client: %v", err)
-		return false
-	}
-
-	// Set admin client
-	workerInstance.SetAdminClient(adminClient)
-
-	// Set working directory
-	if *workerWorkingDir != "" {
-		glog.Infof("Setting working directory to: %s", *workerWorkingDir)
-		if err := os.Chdir(*workerWorkingDir); err != nil {
-			glog.Fatalf("Failed to change working directory: %v", err)
-			return false
-		}
-		wd, err := os.Getwd()
-		if err != nil {
-			glog.Fatalf("Failed to get working directory: %v", err)
-			return false
-		}
-		glog.Infof("Current working directory: %s", wd)
-	}
-
-	// Start metrics HTTP server if port is specified
-	if *workerMetricsPort > 0 {
-		go startWorkerMetricsServer(*workerMetricsIp, *workerMetricsPort, workerInstance)
-	}
-
-	// Start the worker
-	err = workerInstance.Start()
-	if err != nil {
-		glog.Errorf("Failed to start worker: %v", err)
-		return false
-	}
-
-	// Set up signal handling
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
-
-	glog.Infof("Maintenance worker %s started successfully", workerInstance.ID())
-	glog.Infof("Press Ctrl+C to stop the worker")
-
-	// Wait for shutdown signal
-	<-sigChan
-	glog.Infof("Shutdown signal received, stopping worker...")
-
-	// Gracefully stop the worker
-	err = workerInstance.Stop()
-	if err != nil {
-		glog.Errorf("Error stopping worker: %v", err)
-	}
-	glog.Infof("Worker stopped")
-
-	return true
-}
-
-// parseCapabilities converts comma-separated capability string to task types
-func parseCapabilities(capabilityStr string) []types.TaskType {
-	if capabilityStr == "" {
-		return nil
-	}
-
-	capabilityMap := map[string]types.TaskType{}
-
-	// Populate capabilityMap with registered task types
-	typesRegistry := tasks.GetGlobalTypesRegistry()
-	for taskType := range typesRegistry.GetAllDetectors() {
-		// Use the task type string directly as the key
-		capabilityMap[strings.ToLower(string(taskType))] = taskType
-	}
-
-	// Add common aliases for convenience
-	if taskType, exists := capabilityMap["erasure_coding"]; exists {
-		capabilityMap["ec"] = taskType
-	}
-	if taskType, exists := capabilityMap["remote_upload"]; exists {
-		capabilityMap["remote"] = taskType
-	}
-	if taskType, exists := capabilityMap["fix_replication"]; exists {
-		capabilityMap["replication"] = taskType
-	}
-
-	var capabilities []types.TaskType
-	parts := strings.Split(capabilityStr, ",")
-
-	for _, part := range parts {
-		part = strings.TrimSpace(part)
-		if taskType, exists := capabilityMap[part]; exists {
-			capabilities = append(capabilities, taskType)
-		} else {
-			glog.Warningf("Unknown capability: %s", part)
-		}
-	}
-
-	return capabilities
-}
-
-// Legacy compatibility types for backward compatibility
-// These will be deprecated in future versions
-
-// WorkerStatus represents the current status of a worker (deprecated)
-type WorkerStatus struct {
-	WorkerID       string           `json:"worker_id"`
-	Address        string           `json:"address"`
-	Status         string           `json:"status"`
-	Capabilities   []types.TaskType `json:"capabilities"`
-	MaxConcurrent  int              `json:"max_concurrent"`
-	CurrentLoad    int              `json:"current_load"`
-	LastHeartbeat  time.Time        `json:"last_heartbeat"`
-	CurrentTasks   []types.Task     `json:"current_tasks"`
-	Uptime         time.Duration    `json:"uptime"`
-	TasksCompleted int              `json:"tasks_completed"`
-	TasksFailed    int              `json:"tasks_failed"`
-}
-
-func workerHealthHandler(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Server", workerServerHeader)
-	w.WriteHeader(http.StatusOK)
-}
-
-func workerReadyHandler(workerInstance *worker.Worker) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Server", workerServerHeader)
-
-		admin := workerInstance.GetAdmin()
-		if admin == nil || !admin.IsConnected() {
-			w.WriteHeader(http.StatusServiceUnavailable)
-			return
-		}
-
-		w.WriteHeader(http.StatusOK)
-	}
-}
-
-func startWorkerMetricsServer(ip string, port int, w *worker.Worker) {
-	mux := http.NewServeMux()
-	mux.HandleFunc("/health", workerHealthHandler)
-	mux.HandleFunc("/ready", workerReadyHandler(w))
-	mux.Handle("/metrics", promhttp.HandlerFor(statsCollect.Gather, promhttp.HandlerOpts{}))
-
-	glog.V(0).Infof("Starting worker metrics server at %s", statsCollect.JoinHostPort(ip, port))
-	if err := http.ListenAndServe(statsCollect.JoinHostPort(ip, port), mux); err != nil {
-		glog.Errorf("Worker metrics server failed to start: %v", err)
-	}
+	return runPluginWorkerWithOptions(pluginWorkerRunOptions{
+		AdminServer: *workerAdminServer,
+		WorkerID:    *workerID,
+		WorkingDir:  *workerWorkingDir,
+		JobTypes:    *workerJobType,
+		Heartbeat:   *workerHeartbeat,
+		Reconnect:   *workerReconnect,
+		MaxDetect:   *workerMaxDetect,
+		MaxExecute:  *workerMaxExecute,
+		Address:     *workerAddress,
+		MetricsPort: *workerMetricsPort,
+		MetricsIP:   *workerMetricsIp,
+	})
 }
diff --git a/weed/command/worker_runtime.go b/weed/command/worker_runtime.go
new file mode 100644
index 000000000..20069a14d
--- /dev/null
+++ b/weed/command/worker_runtime.go
@@ -0,0 +1,348 @@
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+	"google.golang.org/grpc"
+)
+
+const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding"
+
+type pluginWorkerRunOptions struct {
+	AdminServer string
+	WorkerID    string
+	WorkingDir  string
+	JobTypes    string
+	Heartbeat   time.Duration
+	Reconnect   time.Duration
+	MaxDetect   int
+	MaxExecute  int
+	Address     string
+	MetricsPort int
+	MetricsIP   string
+}
+
+func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool {
+	util.LoadConfiguration("security", false)
+
+	options.AdminServer = strings.TrimSpace(options.AdminServer)
+	if options.AdminServer == "" {
+		options.AdminServer = "localhost:23646"
+	}
+
+	options.JobTypes = strings.TrimSpace(options.JobTypes)
+	if options.JobTypes == "" {
+		options.JobTypes = defaultPluginWorkerJobTypes
+	}
+
+	if options.Heartbeat <= 0 {
+		options.Heartbeat = 15 * time.Second
+	}
+	if options.Reconnect <= 0 {
+		options.Reconnect = 5 * time.Second
+	}
+	if options.MaxDetect <= 0 {
+		options.MaxDetect = 1
+	}
+	if options.MaxExecute <= 0 {
+		options.MaxExecute = 4
+	}
+	options.MetricsIP = strings.TrimSpace(options.MetricsIP)
+	if options.MetricsIP == "" {
+		options.MetricsIP = "0.0.0.0"
+	}
+
+	resolvedAdminServer := resolvePluginWorkerAdminServer(options.AdminServer)
+	if resolvedAdminServer != options.AdminServer {
+		fmt.Printf("Resolved admin worker gRPC endpoint: %s -> %s\n", options.AdminServer, resolvedAdminServer)
+	}
+
+	dialOption := security.LoadClientTLS(util.GetViper(), "grpc.worker")
+	workerID, err := resolvePluginWorkerID(options.WorkerID, options.WorkingDir)
+	if err != nil {
+		glog.Errorf("Failed to resolve plugin worker ID: %v", err)
+		return false
+	}
+
+	handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption)
+	if err != nil {
+		glog.Errorf("Failed to build plugin worker handlers: %v", err)
+		return false
+	}
+	worker, err := pluginworker.NewWorker(pluginworker.WorkerOptions{
+		AdminServer:             resolvedAdminServer,
+		WorkerID:                workerID,
+		WorkerVersion:           version.Version(),
+		WorkerAddress:           options.Address,
+		HeartbeatInterval:       options.Heartbeat,
+		ReconnectDelay:          options.Reconnect,
+		MaxDetectionConcurrency: options.MaxDetect,
+		MaxExecutionConcurrency: options.MaxExecute,
+		GrpcDialOption:          dialOption,
+		Handlers:                handlers,
+	})
+	if err != nil {
+		glog.Errorf("Failed to create plugin worker: %v", err)
+		return false
+	}
+
+	if options.MetricsPort > 0 {
+		go startPluginWorkerMetricsServer(options.MetricsIP, options.MetricsPort, worker)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	defer signal.Stop(sigCh)
+
+	go func() {
+		sig := <-sigCh
+		fmt.Printf("\nReceived signal %v, stopping plugin worker...\n", sig)
+		cancel()
+	}()
+
+	fmt.Printf("Starting plugin worker (admin=%s)\n", resolvedAdminServer)
+	if err := worker.Run(ctx); err != nil {
+		glog.Errorf("Plugin worker stopped with error: %v", err)
+		return false
+	}
+	fmt.Println("Plugin worker stopped")
+	return true
+}
+
+func resolvePluginWorkerID(explicitID string, workingDir string) (string, error) {
+	id := strings.TrimSpace(explicitID)
+	if id != "" {
+		return id, nil
+	}
+
+	workingDir = strings.TrimSpace(workingDir)
+	if workingDir == "" {
+		return "", nil
+	}
+	if err := os.MkdirAll(workingDir, 0755); err != nil {
+		return "", err
+	}
+
+	workerIDPath := filepath.Join(workingDir, "plugin.worker.id")
+	if data, err := os.ReadFile(workerIDPath); err == nil {
+		if persisted := strings.TrimSpace(string(data)); persisted != "" {
+			return persisted, nil
+		}
+	}
+
+	generated := fmt.Sprintf("plugin-%d", time.Now().UnixNano())
+	if err := os.WriteFile(workerIDPath, []byte(generated+"\n"), 0644); err != nil {
+		return "", err
+	}
+	return generated, nil
+}
+
+func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption) (pluginworker.JobHandler, error) {
+	canonicalJobType, err := canonicalPluginWorkerJobType(jobType)
+	if err != nil {
+		return nil, err
+	}
+
+	switch canonicalJobType {
+	case "vacuum":
+		return pluginworker.NewVacuumHandler(dialOption), nil
+	case "volume_balance":
+		return pluginworker.NewVolumeBalanceHandler(dialOption), nil
+	case "erasure_coding":
+		return pluginworker.NewErasureCodingHandler(dialOption), nil
+	default:
+		return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
+	}
+}
+
+func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption) ([]pluginworker.JobHandler, error) {
+	parsedJobTypes, err := parsePluginWorkerJobTypes(jobTypes)
+	if err != nil {
+		return nil, err
+	}
+
+	handlers := make([]pluginworker.JobHandler, 0, len(parsedJobTypes))
+	for _, jobType := range parsedJobTypes {
+		handler, buildErr := buildPluginWorkerHandler(jobType, dialOption)
+		if buildErr != nil {
+			return nil, buildErr
+		}
+		handlers = append(handlers, handler)
+	}
+	return handlers, nil
+}
+
+func parsePluginWorkerJobTypes(jobTypes string) ([]string, error) {
+	jobTypes = strings.TrimSpace(jobTypes)
+	if jobTypes == "" {
+		return []string{"vacuum"}, nil
+	}
+
+	parts := strings.Split(jobTypes, ",")
+	parsed := make([]string, 0, len(parts))
+	seen := make(map[string]struct{}, len(parts))
+
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		canonical, err := canonicalPluginWorkerJobType(part)
+		if err != nil {
+			return nil, err
+		}
+		if _, found := seen[canonical]; found {
+			continue
+		}
+		seen[canonical] = struct{}{}
+		parsed = append(parsed, canonical)
+	}
+
+	if len(parsed) == 0 {
+		return []string{"vacuum"}, nil
+	}
+	return parsed, nil
+}
+
+func canonicalPluginWorkerJobType(jobType string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(jobType)) {
+	case "", "vacuum":
+		return "vacuum", nil
+	case "volume_balance", "balance", "volume.balance", "volume-balance":
+		return "volume_balance", nil
+	case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
+		return "erasure_coding", nil
+	default:
+		return "", fmt.Errorf("unsupported plugin job type %q", jobType)
+	}
+}
+
+func resolvePluginWorkerAdminServer(adminServer string) string {
+	adminServer = strings.TrimSpace(adminServer)
+	host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress(adminServer)
+	if err != nil || hasExplicitGrpcPort {
+		return adminServer
+	}
+
+	workerGrpcPort, err := fetchPluginWorkerGrpcPort(host, httpPort)
+	if err != nil || workerGrpcPort <= 0 {
+		return adminServer
+	}
+
+	// Keep canonical host:http form when admin gRPC follows the default +10000 rule.
+	if workerGrpcPort == httpPort+10000 {
+		return adminServer
+	}
+
+	return fmt.Sprintf("%s:%d.%d", host, httpPort, workerGrpcPort)
+}
+
+func parsePluginWorkerAdminAddress(adminServer string) (host string, httpPort int, hasExplicitGrpcPort bool, err error) {
+	adminServer = strings.TrimSpace(adminServer)
+	colonIndex := strings.LastIndex(adminServer, ":")
+	if colonIndex <= 0 || colonIndex >= len(adminServer)-1 {
+		return "", 0, false, fmt.Errorf("invalid admin address %q", adminServer)
+	}
+
+	host = adminServer[:colonIndex]
+	portPart := adminServer[colonIndex+1:]
+	if dotIndex := strings.LastIndex(portPart, "."); dotIndex > 0 && dotIndex < len(portPart)-1 {
+		if _, parseErr := strconv.Atoi(portPart[dotIndex+1:]); parseErr == nil {
+			hasExplicitGrpcPort = true
+			portPart = portPart[:dotIndex]
+		}
+	}
+
+	httpPort, err = strconv.Atoi(portPart)
+	if err != nil || httpPort <= 0 {
+		return "", 0, false, fmt.Errorf("invalid admin http port in %q", adminServer)
+	}
+	return host, httpPort, hasExplicitGrpcPort, nil
+}
+
+func fetchPluginWorkerGrpcPort(host string, httpPort int) (int, error) {
+	client := &http.Client{Timeout: 2 * time.Second}
+	address := util.JoinHostPort(host, httpPort)
+	var lastErr error
+
+	for _, scheme := range []string{"http", "https"} {
+		statusURL := fmt.Sprintf("%s://%s/api/plugin/status", scheme, address)
+		resp, err := client.Get(statusURL)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+
+		var payload struct {
+			WorkerGrpcPort int `json:"worker_grpc_port"`
+		}
+		decodeErr := json.NewDecoder(resp.Body).Decode(&payload)
+		resp.Body.Close()
+
+		if resp.StatusCode != http.StatusOK {
+			lastErr = fmt.Errorf("status code %d from %s", resp.StatusCode, statusURL)
+			continue
+		}
+		if decodeErr != nil {
+			lastErr = fmt.Errorf("decode plugin status from %s: %w", statusURL, decodeErr)
+			continue
+		}
+		if payload.WorkerGrpcPort <= 0 {
+			lastErr = fmt.Errorf("plugin status from %s returned empty worker_grpc_port", statusURL)
+			continue
+		}
+
+		return payload.WorkerGrpcPort, nil
+	}
+
+	if lastErr == nil {
+		lastErr = fmt.Errorf("plugin status endpoint unavailable")
+	}
+	return 0, lastErr
+}
+
+func pluginWorkerHealthHandler(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func pluginWorkerReadyHandler(pluginRuntime *pluginworker.Worker) http.HandlerFunc {
+	return func(w http.ResponseWriter, _ *http.Request) {
+		if pluginRuntime == nil || !pluginRuntime.IsConnected() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+func startPluginWorkerMetricsServer(ip string, port int, pluginRuntime *pluginworker.Worker) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", pluginWorkerHealthHandler)
+	mux.HandleFunc("/ready", pluginWorkerReadyHandler(pluginRuntime))
+	mux.Handle("/metrics", promhttp.HandlerFor(statsCollect.Gather, promhttp.HandlerOpts{}))
+
+	glog.V(0).Infof("Starting plugin worker metrics server at %s", statsCollect.JoinHostPort(ip, port))
+	if err := http.ListenAndServe(statsCollect.JoinHostPort(ip, port), mux); err != nil {
+		glog.Errorf("Plugin worker metrics server failed to start: %v", err)
+	}
+}
diff --git a/weed/command/worker_test.go b/weed/command/worker_test.go
new file mode 100644
index 000000000..54867893e
--- /dev/null
+++ b/weed/command/worker_test.go
@@ -0,0 +1,13 @@
+package command
+
+import "testing"
+
+func TestWorkerDefaultJobTypes(t *testing.T) {
+	jobTypes, err := parsePluginWorkerJobTypes(*workerJobType)
+	if err != nil {
+		t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err)
+	}
+	if len(jobTypes) != 3 {
+		t.Fatalf("expected default worker job types to include 3 handlers, got %v", jobTypes)
+	}
+}
diff --git a/weed/pb/Makefile b/weed/pb/Makefile
index e5db76426..ad90e1fe5 100644
--- a/weed/pb/Makefile
+++ b/weed/pb/Makefile
@@ -14,6 +14,8 @@ gen:
 	protoc mq_schema.proto --go_out=./schema_pb --go-grpc_out=./schema_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	protoc mq_agent.proto --go_out=./mq_agent_pb --go-grpc_out=./mq_agent_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	protoc worker.proto --go_out=./worker_pb --go-grpc_out=./worker_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
+	mkdir -p ./plugin_pb
+	protoc plugin.proto --go_out=./plugin_pb --go-grpc_out=./plugin_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
 	# protoc filer.proto --java_out=../../other/java/client/src/main/java
 	cp filer.proto ../../other/java/client/src/main/proto
 
diff --git a/weed/pb/plugin.proto b/weed/pb/plugin.proto
new file mode 100644
index 000000000..d3c92fd4b
--- /dev/null
+++ b/weed/pb/plugin.proto
@@ -0,0 +1,443 @@
+syntax = "proto3";
+
+package plugin;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb";
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+service PluginControlService {
+  rpc WorkerStream(stream WorkerToAdminMessage) returns (stream AdminToWorkerMessage);
+}
+
+// WorkerToAdminMessage carries worker-originated events and responses.
+message WorkerToAdminMessage {
+  string worker_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    WorkerHello hello = 10;
+    WorkerHeartbeat heartbeat = 11;
+    WorkerAcknowledge acknowledge = 12;
+    ConfigSchemaResponse config_schema_response = 13;
+    DetectionProposals detection_proposals = 14;
+    DetectionComplete detection_complete = 15;
+    JobProgressUpdate job_progress_update = 16;
+    JobCompleted job_completed = 17;
+  }
+}
+
+// AdminToWorkerMessage carries commands and lifecycle notifications from admin.
+message AdminToWorkerMessage {
+  string request_id = 1;
+  google.protobuf.Timestamp sent_at = 2;
+
+  oneof body {
+    AdminHello hello = 10;
+    RequestConfigSchema request_config_schema = 11;
+    RunDetectionRequest run_detection_request = 12;
+    ExecuteJobRequest execute_job_request = 13;
+    CancelRequest cancel_request = 14;
+    AdminShutdown shutdown = 15;
+  }
+}
+
+message WorkerHello {
+  string worker_id = 1;
+  string worker_instance_id = 2;
+  string address = 3;
+  string worker_version = 4;
+  string protocol_version = 5;
+  repeated JobTypeCapability capabilities = 6;
+  map metadata = 7;
+}
+
+message AdminHello {
+  bool accepted = 1;
+  string message = 2;
+  int32 heartbeat_interval_seconds = 3;
+  int32 reconnect_delay_seconds = 4;
+}
+
+message WorkerHeartbeat {
+  string worker_id = 1;
+  repeated RunningWork running_work = 2;
+  int32 detection_slots_used = 3;
+  int32 detection_slots_total = 4;
+  int32 execution_slots_used = 5;
+  int32 execution_slots_total = 6;
+  map queued_jobs_by_type = 7;
+  map metadata = 8;
+}
+
+message WorkerAcknowledge {
+  string request_id = 1;
+  bool accepted = 2;
+  string message = 3;
+}
+
+message RunningWork {
+  string work_id = 1;
+  WorkKind kind = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+}
+
+message JobTypeCapability {
+  string job_type = 1;
+  bool can_detect = 2;
+  bool can_execute = 3;
+  int32 max_detection_concurrency = 4;
+  int32 max_execution_concurrency = 5;
+  string display_name = 6;
+  string description = 7;
+}
+
+message RequestConfigSchema {
+  string job_type = 1;
+  bool force_refresh = 2;
+}
+
+message ConfigSchemaResponse {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  JobTypeDescriptor job_type_descriptor = 5;
+}
+
+// JobTypeDescriptor defines one job type contract, including UI schema and defaults.
+message JobTypeDescriptor {
+  string job_type = 1;
+  string display_name = 2;
+  string description = 3;
+  string icon = 4;
+  uint32 descriptor_version = 5;
+
+  // Admin-owned options such as detection frequency and dispatch concurrency.
+  ConfigForm admin_config_form = 6;
+  // Worker-owned options used during detection and execution.
+  ConfigForm worker_config_form = 7;
+
+  AdminRuntimeDefaults admin_runtime_defaults = 8;
+  map worker_default_values = 9;
+}
+
+message ConfigForm {
+  string form_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigSection sections = 4;
+  map default_values = 5;
+}
+
+message ConfigSection {
+  string section_id = 1;
+  string title = 2;
+  string description = 3;
+  repeated ConfigField fields = 4;
+}
+
+message ConfigField {
+  string name = 1;
+  string label = 2;
+  string description = 3;
+  string help_text = 4;
+  string placeholder = 5;
+
+  ConfigFieldType field_type = 6;
+  ConfigWidget widget = 7;
+
+  bool required = 8;
+  bool read_only = 9;
+  bool sensitive = 10;
+
+  ConfigValue min_value = 11;
+  ConfigValue max_value = 12;
+
+  repeated ConfigOption options = 13;
+  repeated ValidationRule validation_rules = 14;
+
+  // Simple visibility dependency: show this field when the referenced field equals value.
+  string visible_when_field = 15;
+  ConfigValue visible_when_equals = 16;
+}
+
+message ConfigOption {
+  string value = 1;
+  string label = 2;
+  string description = 3;
+  bool disabled = 4;
+}
+
+message ValidationRule {
+  ValidationRuleType type = 1;
+  string expression = 2;
+  string error_message = 3;
+}
+
+message ConfigValue {
+  oneof kind {
+    bool bool_value = 1;
+    int64 int64_value = 2;
+    double double_value = 3;
+    string string_value = 4;
+    bytes bytes_value = 5;
+    google.protobuf.Duration duration_value = 6;
+    StringList string_list = 7;
+    Int64List int64_list = 8;
+    DoubleList double_list = 9;
+    BoolList bool_list = 10;
+    ValueList list_value = 11;
+    ValueMap map_value = 12;
+  }
+}
+
+message StringList {
+  repeated string values = 1;
+}
+
+message Int64List {
+  repeated int64 values = 1;
+}
+
+message DoubleList {
+  repeated double values = 1;
+}
+
+message BoolList {
+  repeated bool values = 1;
+}
+
+message ValueList {
+  repeated ConfigValue values = 1;
+}
+
+message ValueMap {
+  map fields = 1;
+}
+
+message AdminRuntimeDefaults {
+  bool enabled = 1;
+  int32 detection_interval_seconds = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+}
+
+message AdminRuntimeConfig {
+  bool enabled = 1;
+  int32 detection_interval_seconds = 2;
+  int32 detection_timeout_seconds = 3;
+  int32 max_jobs_per_detection = 4;
+  int32 global_execution_concurrency = 5;
+  int32 per_worker_execution_concurrency = 6;
+  int32 retry_limit = 7;
+  int32 retry_backoff_seconds = 8;
+}
+
+message RunDetectionRequest {
+  string request_id = 1;
+  string job_type = 2;
+  int64 detection_sequence = 3;
+
+  AdminRuntimeConfig admin_runtime = 4;
+  map admin_config_values = 5;
+  map worker_config_values = 6;
+
+  ClusterContext cluster_context = 7;
+  google.protobuf.Timestamp last_successful_run = 8;
+  int32 max_results = 9;
+}
+
+message DetectionProposals {
+  string request_id = 1;
+  string job_type = 2;
+  repeated JobProposal proposals = 3;
+  bool has_more = 4;
+}
+
+message DetectionComplete {
+  string request_id = 1;
+  string job_type = 2;
+  bool success = 3;
+  string error_message = 4;
+  int32 total_proposals = 5;
+}
+
+message JobProposal {
+  string proposal_id = 1;
+  string dedupe_key = 2;
+  string job_type = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map parameters = 7;
+  map labels = 8;
+  google.protobuf.Timestamp not_before = 9;
+  google.protobuf.Timestamp expires_at = 10;
+}
+
+message ExecuteJobRequest {
+  string request_id = 1;
+  JobSpec job = 2;
+  AdminRuntimeConfig admin_runtime = 3;
+  map admin_config_values = 4;
+  map worker_config_values = 5;
+  ClusterContext cluster_context = 6;
+  int32 attempt = 7;
+}
+
+message JobSpec {
+  string job_id = 1;
+  string job_type = 2;
+  string dedupe_key = 3;
+  JobPriority priority = 4;
+  string summary = 5;
+  string detail = 6;
+  map parameters = 7;
+  map labels = 8;
+  google.protobuf.Timestamp created_at = 9;
+  google.protobuf.Timestamp scheduled_at = 10;
+}
+
+message JobProgressUpdate {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  JobState state = 4;
+  double progress_percent = 5;
+  string stage = 6;
+  string message = 7;
+  map metrics = 8;
+  repeated ActivityEvent activities = 9;
+  google.protobuf.Timestamp updated_at = 10;
+}
+
+message JobCompleted {
+  string request_id = 1;
+  string job_id = 2;
+  string job_type = 3;
+  bool success = 4;
+  string error_message = 5;
+  JobResult result = 6;
+  repeated ActivityEvent activities = 7;
+  google.protobuf.Timestamp completed_at = 8;
+}
+
+message JobResult {
+  map output_values = 1;
+  string summary = 2;
+}
+
+message ClusterContext {
+  repeated string master_grpc_addresses = 1;
+  repeated string filer_grpc_addresses = 2;
+  repeated string volume_grpc_addresses = 3;
+  map metadata = 4;
+}
+
+message ActivityEvent {
+  ActivitySource source = 1;
+  string message = 2;
+  string stage = 3;
+  map details = 4;
+  google.protobuf.Timestamp created_at = 5;
+}
+
+message CancelRequest {
+  string target_id = 1;
+  WorkKind target_kind = 2;
+  string reason = 3;
+  bool force = 4;
+}
+
+message AdminShutdown {
+  string reason = 1;
+  int32 grace_period_seconds = 2;
+}
+
+// PersistedJobTypeConfig is the admin-side on-disk model per job type.
+message PersistedJobTypeConfig {
+  string job_type = 1;
+  uint32 descriptor_version = 2;
+  map admin_config_values = 3;
+  map worker_config_values = 4;
+  AdminRuntimeConfig admin_runtime = 5;
+  google.protobuf.Timestamp updated_at = 6;
+  string updated_by = 7;
+}
+
+enum WorkKind {
+  WORK_KIND_UNSPECIFIED = 0;
+  WORK_KIND_DETECTION = 1;
+  WORK_KIND_EXECUTION = 2;
+}
+
+enum JobPriority {
+  JOB_PRIORITY_UNSPECIFIED = 0;
+  JOB_PRIORITY_LOW = 1;
+  JOB_PRIORITY_NORMAL = 2;
+  JOB_PRIORITY_HIGH = 3;
+  JOB_PRIORITY_CRITICAL = 4;
+}
+
+enum JobState {
+  JOB_STATE_UNSPECIFIED = 0;
+  JOB_STATE_PENDING = 1;
+  JOB_STATE_ASSIGNED = 2;
+  JOB_STATE_RUNNING = 3;
+  JOB_STATE_SUCCEEDED = 4;
+  JOB_STATE_FAILED = 5;
+  JOB_STATE_CANCELED = 6;
+}
+
+enum ConfigFieldType {
+  CONFIG_FIELD_TYPE_UNSPECIFIED = 0;
+  CONFIG_FIELD_TYPE_BOOL = 1;
+  CONFIG_FIELD_TYPE_INT64 = 2;
+  CONFIG_FIELD_TYPE_DOUBLE = 3;
+  CONFIG_FIELD_TYPE_STRING = 4;
+  CONFIG_FIELD_TYPE_BYTES = 5;
+  CONFIG_FIELD_TYPE_DURATION = 6;
+  CONFIG_FIELD_TYPE_ENUM = 7;
+  CONFIG_FIELD_TYPE_LIST = 8;
+  CONFIG_FIELD_TYPE_OBJECT = 9;
+}
+
+enum ConfigWidget {
+  CONFIG_WIDGET_UNSPECIFIED = 0;
+  CONFIG_WIDGET_TOGGLE = 1;
+  CONFIG_WIDGET_TEXT = 2;
+  CONFIG_WIDGET_TEXTAREA = 3;
+  CONFIG_WIDGET_NUMBER = 4;
+  CONFIG_WIDGET_SELECT = 5;
+  CONFIG_WIDGET_MULTI_SELECT = 6;
+  CONFIG_WIDGET_DURATION = 7;
+  CONFIG_WIDGET_PASSWORD = 8;
+}
+
+enum ValidationRuleType {
+  VALIDATION_RULE_TYPE_UNSPECIFIED = 0;
+  VALIDATION_RULE_TYPE_REGEX = 1;
+  VALIDATION_RULE_TYPE_MIN_LENGTH = 2;
+  VALIDATION_RULE_TYPE_MAX_LENGTH = 3;
+  VALIDATION_RULE_TYPE_MIN_ITEMS = 4;
+  VALIDATION_RULE_TYPE_MAX_ITEMS = 5;
+  VALIDATION_RULE_TYPE_CUSTOM = 6;
+}
+
+enum ActivitySource {
+  ACTIVITY_SOURCE_UNSPECIFIED = 0;
+  ACTIVITY_SOURCE_ADMIN = 1;
+  ACTIVITY_SOURCE_DETECTOR = 2;
+  ACTIVITY_SOURCE_EXECUTOR = 3;
+}
diff --git a/weed/pb/plugin_pb/plugin.pb.go b/weed/pb/plugin_pb/plugin.pb.go
new file mode 100644
index 000000000..d11dee176
--- /dev/null
+++ b/weed/pb/plugin_pb/plugin.pb.go
@@ -0,0 +1,4566 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: plugin.proto
+
+package plugin_pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	durationpb "google.golang.org/protobuf/types/known/durationpb"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type WorkKind int32
+
+const (
+	WorkKind_WORK_KIND_UNSPECIFIED WorkKind = 0
+	WorkKind_WORK_KIND_DETECTION   WorkKind = 1
+	WorkKind_WORK_KIND_EXECUTION   WorkKind = 2
+)
+
+// Enum value maps for WorkKind.
+var (
+	WorkKind_name = map[int32]string{
+		0: "WORK_KIND_UNSPECIFIED",
+		1: "WORK_KIND_DETECTION",
+		2: "WORK_KIND_EXECUTION",
+	}
+	WorkKind_value = map[string]int32{
+		"WORK_KIND_UNSPECIFIED": 0,
+		"WORK_KIND_DETECTION":   1,
+		"WORK_KIND_EXECUTION":   2,
+	}
+)
+
+func (x WorkKind) Enum() *WorkKind {
+	p := new(WorkKind)
+	*p = x
+	return p
+}
+
+func (x WorkKind) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (WorkKind) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[0].Descriptor()
+}
+
+func (WorkKind) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[0]
+}
+
+func (x WorkKind) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use WorkKind.Descriptor instead.
+func (WorkKind) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{0}
+}
+
+type JobPriority int32
+
+const (
+	JobPriority_JOB_PRIORITY_UNSPECIFIED JobPriority = 0
+	JobPriority_JOB_PRIORITY_LOW         JobPriority = 1
+	JobPriority_JOB_PRIORITY_NORMAL      JobPriority = 2
+	JobPriority_JOB_PRIORITY_HIGH        JobPriority = 3
+	JobPriority_JOB_PRIORITY_CRITICAL    JobPriority = 4
+)
+
+// Enum value maps for JobPriority.
+var (
+	JobPriority_name = map[int32]string{
+		0: "JOB_PRIORITY_UNSPECIFIED",
+		1: "JOB_PRIORITY_LOW",
+		2: "JOB_PRIORITY_NORMAL",
+		3: "JOB_PRIORITY_HIGH",
+		4: "JOB_PRIORITY_CRITICAL",
+	}
+	JobPriority_value = map[string]int32{
+		"JOB_PRIORITY_UNSPECIFIED": 0,
+		"JOB_PRIORITY_LOW":         1,
+		"JOB_PRIORITY_NORMAL":      2,
+		"JOB_PRIORITY_HIGH":        3,
+		"JOB_PRIORITY_CRITICAL":    4,
+	}
+)
+
+func (x JobPriority) Enum() *JobPriority {
+	p := new(JobPriority)
+	*p = x
+	return p
+}
+
+func (x JobPriority) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (JobPriority) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[1].Descriptor()
+}
+
+func (JobPriority) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[1]
+}
+
+func (x JobPriority) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use JobPriority.Descriptor instead.
+func (JobPriority) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{1}
+}
+
+type JobState int32
+
+const (
+	JobState_JOB_STATE_UNSPECIFIED JobState = 0
+	JobState_JOB_STATE_PENDING     JobState = 1
+	JobState_JOB_STATE_ASSIGNED    JobState = 2
+	JobState_JOB_STATE_RUNNING     JobState = 3
+	JobState_JOB_STATE_SUCCEEDED   JobState = 4
+	JobState_JOB_STATE_FAILED      JobState = 5
+	JobState_JOB_STATE_CANCELED    JobState = 6
+)
+
+// Enum value maps for JobState.
+var (
+	JobState_name = map[int32]string{
+		0: "JOB_STATE_UNSPECIFIED",
+		1: "JOB_STATE_PENDING",
+		2: "JOB_STATE_ASSIGNED",
+		3: "JOB_STATE_RUNNING",
+		4: "JOB_STATE_SUCCEEDED",
+		5: "JOB_STATE_FAILED",
+		6: "JOB_STATE_CANCELED",
+	}
+	JobState_value = map[string]int32{
+		"JOB_STATE_UNSPECIFIED": 0,
+		"JOB_STATE_PENDING":     1,
+		"JOB_STATE_ASSIGNED":    2,
+		"JOB_STATE_RUNNING":     3,
+		"JOB_STATE_SUCCEEDED":   4,
+		"JOB_STATE_FAILED":      5,
+		"JOB_STATE_CANCELED":    6,
+	}
+)
+
+func (x JobState) Enum() *JobState {
+	p := new(JobState)
+	*p = x
+	return p
+}
+
+func (x JobState) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (JobState) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[2].Descriptor()
+}
+
+func (JobState) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[2]
+}
+
+func (x JobState) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use JobState.Descriptor instead.
+func (JobState) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{2}
+}
+
+type ConfigFieldType int32
+
+const (
+	ConfigFieldType_CONFIG_FIELD_TYPE_UNSPECIFIED ConfigFieldType = 0
+	ConfigFieldType_CONFIG_FIELD_TYPE_BOOL        ConfigFieldType = 1
+	ConfigFieldType_CONFIG_FIELD_TYPE_INT64       ConfigFieldType = 2
+	ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE      ConfigFieldType = 3
+	ConfigFieldType_CONFIG_FIELD_TYPE_STRING      ConfigFieldType = 4
+	ConfigFieldType_CONFIG_FIELD_TYPE_BYTES       ConfigFieldType = 5
+	ConfigFieldType_CONFIG_FIELD_TYPE_DURATION    ConfigFieldType = 6
+	ConfigFieldType_CONFIG_FIELD_TYPE_ENUM        ConfigFieldType = 7
+	ConfigFieldType_CONFIG_FIELD_TYPE_LIST        ConfigFieldType = 8
+	ConfigFieldType_CONFIG_FIELD_TYPE_OBJECT      ConfigFieldType = 9
+)
+
+// Enum value maps for ConfigFieldType.
+var (
+	ConfigFieldType_name = map[int32]string{
+		0: "CONFIG_FIELD_TYPE_UNSPECIFIED",
+		1: "CONFIG_FIELD_TYPE_BOOL",
+		2: "CONFIG_FIELD_TYPE_INT64",
+		3: "CONFIG_FIELD_TYPE_DOUBLE",
+		4: "CONFIG_FIELD_TYPE_STRING",
+		5: "CONFIG_FIELD_TYPE_BYTES",
+		6: "CONFIG_FIELD_TYPE_DURATION",
+		7: "CONFIG_FIELD_TYPE_ENUM",
+		8: "CONFIG_FIELD_TYPE_LIST",
+		9: "CONFIG_FIELD_TYPE_OBJECT",
+	}
+	ConfigFieldType_value = map[string]int32{
+		"CONFIG_FIELD_TYPE_UNSPECIFIED": 0,
+		"CONFIG_FIELD_TYPE_BOOL":        1,
+		"CONFIG_FIELD_TYPE_INT64":       2,
+		"CONFIG_FIELD_TYPE_DOUBLE":      3,
+		"CONFIG_FIELD_TYPE_STRING":      4,
+		"CONFIG_FIELD_TYPE_BYTES":       5,
+		"CONFIG_FIELD_TYPE_DURATION":    6,
+		"CONFIG_FIELD_TYPE_ENUM":        7,
+		"CONFIG_FIELD_TYPE_LIST":        8,
+		"CONFIG_FIELD_TYPE_OBJECT":      9,
+	}
+)
+
+func (x ConfigFieldType) Enum() *ConfigFieldType {
+	p := new(ConfigFieldType)
+	*p = x
+	return p
+}
+
+func (x ConfigFieldType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ConfigFieldType) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[3].Descriptor()
+}
+
+func (ConfigFieldType) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[3]
+}
+
+func (x ConfigFieldType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ConfigFieldType.Descriptor instead.
+func (ConfigFieldType) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{3}
+}
+
+type ConfigWidget int32
+
+const (
+	ConfigWidget_CONFIG_WIDGET_UNSPECIFIED  ConfigWidget = 0
+	ConfigWidget_CONFIG_WIDGET_TOGGLE       ConfigWidget = 1
+	ConfigWidget_CONFIG_WIDGET_TEXT         ConfigWidget = 2
+	ConfigWidget_CONFIG_WIDGET_TEXTAREA     ConfigWidget = 3
+	ConfigWidget_CONFIG_WIDGET_NUMBER       ConfigWidget = 4
+	ConfigWidget_CONFIG_WIDGET_SELECT       ConfigWidget = 5
+	ConfigWidget_CONFIG_WIDGET_MULTI_SELECT ConfigWidget = 6
+	ConfigWidget_CONFIG_WIDGET_DURATION     ConfigWidget = 7
+	ConfigWidget_CONFIG_WIDGET_PASSWORD     ConfigWidget = 8
+)
+
+// Enum value maps for ConfigWidget.
+var (
+	ConfigWidget_name = map[int32]string{
+		0: "CONFIG_WIDGET_UNSPECIFIED",
+		1: "CONFIG_WIDGET_TOGGLE",
+		2: "CONFIG_WIDGET_TEXT",
+		3: "CONFIG_WIDGET_TEXTAREA",
+		4: "CONFIG_WIDGET_NUMBER",
+		5: "CONFIG_WIDGET_SELECT",
+		6: "CONFIG_WIDGET_MULTI_SELECT",
+		7: "CONFIG_WIDGET_DURATION",
+		8: "CONFIG_WIDGET_PASSWORD",
+	}
+	ConfigWidget_value = map[string]int32{
+		"CONFIG_WIDGET_UNSPECIFIED":  0,
+		"CONFIG_WIDGET_TOGGLE":       1,
+		"CONFIG_WIDGET_TEXT":         2,
+		"CONFIG_WIDGET_TEXTAREA":     3,
+		"CONFIG_WIDGET_NUMBER":       4,
+		"CONFIG_WIDGET_SELECT":       5,
+		"CONFIG_WIDGET_MULTI_SELECT": 6,
+		"CONFIG_WIDGET_DURATION":     7,
+		"CONFIG_WIDGET_PASSWORD":     8,
+	}
+)
+
+func (x ConfigWidget) Enum() *ConfigWidget {
+	p := new(ConfigWidget)
+	*p = x
+	return p
+}
+
+func (x ConfigWidget) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ConfigWidget) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[4].Descriptor()
+}
+
+func (ConfigWidget) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[4]
+}
+
+func (x ConfigWidget) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ConfigWidget.Descriptor instead.
+func (ConfigWidget) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{4}
+}
+
+type ValidationRuleType int32
+
+const (
+	ValidationRuleType_VALIDATION_RULE_TYPE_UNSPECIFIED ValidationRuleType = 0
+	ValidationRuleType_VALIDATION_RULE_TYPE_REGEX       ValidationRuleType = 1
+	ValidationRuleType_VALIDATION_RULE_TYPE_MIN_LENGTH  ValidationRuleType = 2
+	ValidationRuleType_VALIDATION_RULE_TYPE_MAX_LENGTH  ValidationRuleType = 3
+	ValidationRuleType_VALIDATION_RULE_TYPE_MIN_ITEMS   ValidationRuleType = 4
+	ValidationRuleType_VALIDATION_RULE_TYPE_MAX_ITEMS   ValidationRuleType = 5
+	ValidationRuleType_VALIDATION_RULE_TYPE_CUSTOM      ValidationRuleType = 6
+)
+
+// Enum value maps for ValidationRuleType.
+var (
+	ValidationRuleType_name = map[int32]string{
+		0: "VALIDATION_RULE_TYPE_UNSPECIFIED",
+		1: "VALIDATION_RULE_TYPE_REGEX",
+		2: "VALIDATION_RULE_TYPE_MIN_LENGTH",
+		3: "VALIDATION_RULE_TYPE_MAX_LENGTH",
+		4: "VALIDATION_RULE_TYPE_MIN_ITEMS",
+		5: "VALIDATION_RULE_TYPE_MAX_ITEMS",
+		6: "VALIDATION_RULE_TYPE_CUSTOM",
+	}
+	ValidationRuleType_value = map[string]int32{
+		"VALIDATION_RULE_TYPE_UNSPECIFIED": 0,
+		"VALIDATION_RULE_TYPE_REGEX":       1,
+		"VALIDATION_RULE_TYPE_MIN_LENGTH":  2,
+		"VALIDATION_RULE_TYPE_MAX_LENGTH":  3,
+		"VALIDATION_RULE_TYPE_MIN_ITEMS":   4,
+		"VALIDATION_RULE_TYPE_MAX_ITEMS":   5,
+		"VALIDATION_RULE_TYPE_CUSTOM":      6,
+	}
+)
+
+func (x ValidationRuleType) Enum() *ValidationRuleType {
+	p := new(ValidationRuleType)
+	*p = x
+	return p
+}
+
+func (x ValidationRuleType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ValidationRuleType) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[5].Descriptor()
+}
+
+func (ValidationRuleType) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[5]
+}
+
+func (x ValidationRuleType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ValidationRuleType.Descriptor instead.
+func (ValidationRuleType) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{5}
+}
+
+type ActivitySource int32
+
+const (
+	ActivitySource_ACTIVITY_SOURCE_UNSPECIFIED ActivitySource = 0
+	ActivitySource_ACTIVITY_SOURCE_ADMIN       ActivitySource = 1
+	ActivitySource_ACTIVITY_SOURCE_DETECTOR    ActivitySource = 2
+	ActivitySource_ACTIVITY_SOURCE_EXECUTOR    ActivitySource = 3
+)
+
+// Enum value maps for ActivitySource.
+var (
+	ActivitySource_name = map[int32]string{
+		0: "ACTIVITY_SOURCE_UNSPECIFIED",
+		1: "ACTIVITY_SOURCE_ADMIN",
+		2: "ACTIVITY_SOURCE_DETECTOR",
+		3: "ACTIVITY_SOURCE_EXECUTOR",
+	}
+	ActivitySource_value = map[string]int32{
+		"ACTIVITY_SOURCE_UNSPECIFIED": 0,
+		"ACTIVITY_SOURCE_ADMIN":       1,
+		"ACTIVITY_SOURCE_DETECTOR":    2,
+		"ACTIVITY_SOURCE_EXECUTOR":    3,
+	}
+)
+
+func (x ActivitySource) Enum() *ActivitySource {
+	p := new(ActivitySource)
+	*p = x
+	return p
+}
+
+func (x ActivitySource) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ActivitySource) Descriptor() protoreflect.EnumDescriptor {
+	return file_plugin_proto_enumTypes[6].Descriptor()
+}
+
+func (ActivitySource) Type() protoreflect.EnumType {
+	return &file_plugin_proto_enumTypes[6]
+}
+
+func (x ActivitySource) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ActivitySource.Descriptor instead.
+func (ActivitySource) EnumDescriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{6}
+}
+
+// WorkerToAdminMessage carries worker-originated events and responses.
+type WorkerToAdminMessage struct {
+	state    protoimpl.MessageState `protogen:"open.v1"`
+	WorkerId string                 `protobuf:"bytes,1,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty"`
+	SentAt   *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=sent_at,json=sentAt,proto3" json:"sent_at,omitempty"`
+	// Types that are valid to be assigned to Body:
+	//
+	//	*WorkerToAdminMessage_Hello
+	//	*WorkerToAdminMessage_Heartbeat
+	//	*WorkerToAdminMessage_Acknowledge
+	//	*WorkerToAdminMessage_ConfigSchemaResponse
+	//	*WorkerToAdminMessage_DetectionProposals
+	//	*WorkerToAdminMessage_DetectionComplete
+	//	*WorkerToAdminMessage_JobProgressUpdate
+	//	*WorkerToAdminMessage_JobCompleted
+	Body          isWorkerToAdminMessage_Body `protobuf_oneof:"body"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *WorkerToAdminMessage) Reset() {
+	*x = WorkerToAdminMessage{}
+	mi := &file_plugin_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *WorkerToAdminMessage) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkerToAdminMessage) ProtoMessage() {}
+
+func (x *WorkerToAdminMessage) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkerToAdminMessage.ProtoReflect.Descriptor instead.
+func (*WorkerToAdminMessage) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *WorkerToAdminMessage) GetWorkerId() string {
+	if x != nil {
+		return x.WorkerId
+	}
+	return ""
+}
+
+func (x *WorkerToAdminMessage) GetSentAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.SentAt
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetBody() isWorkerToAdminMessage_Body {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetHello() *WorkerHello {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_Hello); ok {
+			return x.Hello
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetHeartbeat() *WorkerHeartbeat {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_Heartbeat); ok {
+			return x.Heartbeat
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetAcknowledge() *WorkerAcknowledge {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_Acknowledge); ok {
+			return x.Acknowledge
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetConfigSchemaResponse() *ConfigSchemaResponse {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_ConfigSchemaResponse); ok {
+			return x.ConfigSchemaResponse
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetDetectionProposals() *DetectionProposals {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_DetectionProposals); ok {
+			return x.DetectionProposals
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetDetectionComplete() *DetectionComplete {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_DetectionComplete); ok {
+			return x.DetectionComplete
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetJobProgressUpdate() *JobProgressUpdate {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_JobProgressUpdate); ok {
+			return x.JobProgressUpdate
+		}
+	}
+	return nil
+}
+
+func (x *WorkerToAdminMessage) GetJobCompleted() *JobCompleted {
+	if x != nil {
+		if x, ok := x.Body.(*WorkerToAdminMessage_JobCompleted); ok {
+			return x.JobCompleted
+		}
+	}
+	return nil
+}
+
+type isWorkerToAdminMessage_Body interface {
+	isWorkerToAdminMessage_Body()
+}
+
+type WorkerToAdminMessage_Hello struct {
+	Hello *WorkerHello `protobuf:"bytes,10,opt,name=hello,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_Heartbeat struct {
+	Heartbeat *WorkerHeartbeat `protobuf:"bytes,11,opt,name=heartbeat,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_Acknowledge struct {
+	Acknowledge *WorkerAcknowledge `protobuf:"bytes,12,opt,name=acknowledge,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_ConfigSchemaResponse struct {
+	ConfigSchemaResponse *ConfigSchemaResponse `protobuf:"bytes,13,opt,name=config_schema_response,json=configSchemaResponse,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_DetectionProposals struct {
+	DetectionProposals *DetectionProposals `protobuf:"bytes,14,opt,name=detection_proposals,json=detectionProposals,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_DetectionComplete struct {
+	DetectionComplete *DetectionComplete `protobuf:"bytes,15,opt,name=detection_complete,json=detectionComplete,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_JobProgressUpdate struct {
+	JobProgressUpdate *JobProgressUpdate `protobuf:"bytes,16,opt,name=job_progress_update,json=jobProgressUpdate,proto3,oneof"`
+}
+
+type WorkerToAdminMessage_JobCompleted struct {
+	JobCompleted *JobCompleted `protobuf:"bytes,17,opt,name=job_completed,json=jobCompleted,proto3,oneof"`
+}
+
+func (*WorkerToAdminMessage_Hello) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_Heartbeat) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_Acknowledge) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_ConfigSchemaResponse) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_DetectionProposals) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_DetectionComplete) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_JobProgressUpdate) isWorkerToAdminMessage_Body() {}
+
+func (*WorkerToAdminMessage_JobCompleted) isWorkerToAdminMessage_Body() {}
+
+// AdminToWorkerMessage carries commands and lifecycle notifications from admin.
+type AdminToWorkerMessage struct {
+	state     protoimpl.MessageState `protogen:"open.v1"`
+	RequestId string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	SentAt    *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=sent_at,json=sentAt,proto3" json:"sent_at,omitempty"`
+	// Types that are valid to be assigned to Body:
+	//
+	//	*AdminToWorkerMessage_Hello
+	//	*AdminToWorkerMessage_RequestConfigSchema
+	//	*AdminToWorkerMessage_RunDetectionRequest
+	//	*AdminToWorkerMessage_ExecuteJobRequest
+	//	*AdminToWorkerMessage_CancelRequest
+	//	*AdminToWorkerMessage_Shutdown
+	Body          isAdminToWorkerMessage_Body `protobuf_oneof:"body"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *AdminToWorkerMessage) Reset() {
+	*x = AdminToWorkerMessage{}
+	mi := &file_plugin_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AdminToWorkerMessage) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AdminToWorkerMessage) ProtoMessage() {}
+
+func (x *AdminToWorkerMessage) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AdminToWorkerMessage.ProtoReflect.Descriptor instead.
+func (*AdminToWorkerMessage) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *AdminToWorkerMessage) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *AdminToWorkerMessage) GetSentAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.SentAt
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetBody() isAdminToWorkerMessage_Body {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetHello() *AdminHello {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_Hello); ok {
+			return x.Hello
+		}
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetRequestConfigSchema() *RequestConfigSchema {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_RequestConfigSchema); ok {
+			return x.RequestConfigSchema
+		}
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetRunDetectionRequest() *RunDetectionRequest {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_RunDetectionRequest); ok {
+			return x.RunDetectionRequest
+		}
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetExecuteJobRequest() *ExecuteJobRequest {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_ExecuteJobRequest); ok {
+			return x.ExecuteJobRequest
+		}
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetCancelRequest() *CancelRequest {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_CancelRequest); ok {
+			return x.CancelRequest
+		}
+	}
+	return nil
+}
+
+func (x *AdminToWorkerMessage) GetShutdown() *AdminShutdown {
+	if x != nil {
+		if x, ok := x.Body.(*AdminToWorkerMessage_Shutdown); ok {
+			return x.Shutdown
+		}
+	}
+	return nil
+}
+
+type isAdminToWorkerMessage_Body interface {
+	isAdminToWorkerMessage_Body()
+}
+
+type AdminToWorkerMessage_Hello struct {
+	Hello *AdminHello `protobuf:"bytes,10,opt,name=hello,proto3,oneof"`
+}
+
+type AdminToWorkerMessage_RequestConfigSchema struct {
+	RequestConfigSchema *RequestConfigSchema `protobuf:"bytes,11,opt,name=request_config_schema,json=requestConfigSchema,proto3,oneof"`
+}
+
+type AdminToWorkerMessage_RunDetectionRequest struct {
+	RunDetectionRequest *RunDetectionRequest `protobuf:"bytes,12,opt,name=run_detection_request,json=runDetectionRequest,proto3,oneof"`
+}
+
+type AdminToWorkerMessage_ExecuteJobRequest struct {
+	ExecuteJobRequest *ExecuteJobRequest `protobuf:"bytes,13,opt,name=execute_job_request,json=executeJobRequest,proto3,oneof"`
+}
+
+type AdminToWorkerMessage_CancelRequest struct {
+	CancelRequest *CancelRequest `protobuf:"bytes,14,opt,name=cancel_request,json=cancelRequest,proto3,oneof"`
+}
+
+type AdminToWorkerMessage_Shutdown struct {
+	Shutdown *AdminShutdown `protobuf:"bytes,15,opt,name=shutdown,proto3,oneof"`
+}
+
+func (*AdminToWorkerMessage_Hello) isAdminToWorkerMessage_Body() {}
+
+func (*AdminToWorkerMessage_RequestConfigSchema) isAdminToWorkerMessage_Body() {}
+
+func (*AdminToWorkerMessage_RunDetectionRequest) isAdminToWorkerMessage_Body() {}
+
+func (*AdminToWorkerMessage_ExecuteJobRequest) isAdminToWorkerMessage_Body() {}
+
+func (*AdminToWorkerMessage_CancelRequest) isAdminToWorkerMessage_Body() {}
+
+func (*AdminToWorkerMessage_Shutdown) isAdminToWorkerMessage_Body() {}
+
+type WorkerHello struct {
+	state            protoimpl.MessageState `protogen:"open.v1"`
+	WorkerId         string                 `protobuf:"bytes,1,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty"`
+	WorkerInstanceId string                 `protobuf:"bytes,2,opt,name=worker_instance_id,json=workerInstanceId,proto3" json:"worker_instance_id,omitempty"`
+	Address          string                 `protobuf:"bytes,3,opt,name=address,proto3" json:"address,omitempty"`
+	WorkerVersion    string                 `protobuf:"bytes,4,opt,name=worker_version,json=workerVersion,proto3" json:"worker_version,omitempty"`
+	ProtocolVersion  string                 `protobuf:"bytes,5,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"`
+	Capabilities     []*JobTypeCapability   `protobuf:"bytes,6,rep,name=capabilities,proto3" json:"capabilities,omitempty"`
+	Metadata         map[string]string      `protobuf:"bytes,7,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields    protoimpl.UnknownFields
+	sizeCache        protoimpl.SizeCache
+}
+
+func (x *WorkerHello) Reset() {
+	*x = WorkerHello{}
+	mi := &file_plugin_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *WorkerHello) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkerHello) ProtoMessage() {}
+
+func (x *WorkerHello) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkerHello.ProtoReflect.Descriptor instead.
+func (*WorkerHello) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *WorkerHello) GetWorkerId() string {
+	if x != nil {
+		return x.WorkerId
+	}
+	return ""
+}
+
+func (x *WorkerHello) GetWorkerInstanceId() string {
+	if x != nil {
+		return x.WorkerInstanceId
+	}
+	return ""
+}
+
+func (x *WorkerHello) GetAddress() string {
+	if x != nil {
+		return x.Address
+	}
+	return ""
+}
+
+func (x *WorkerHello) GetWorkerVersion() string {
+	if x != nil {
+		return x.WorkerVersion
+	}
+	return ""
+}
+
+func (x *WorkerHello) GetProtocolVersion() string {
+	if x != nil {
+		return x.ProtocolVersion
+	}
+	return ""
+}
+
+func (x *WorkerHello) GetCapabilities() []*JobTypeCapability {
+	if x != nil {
+		return x.Capabilities
+	}
+	return nil
+}
+
+func (x *WorkerHello) GetMetadata() map[string]string {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+type AdminHello struct {
+	state                    protoimpl.MessageState `protogen:"open.v1"`
+	Accepted                 bool                   `protobuf:"varint,1,opt,name=accepted,proto3" json:"accepted,omitempty"`
+	Message                  string                 `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	HeartbeatIntervalSeconds int32                  `protobuf:"varint,3,opt,name=heartbeat_interval_seconds,json=heartbeatIntervalSeconds,proto3" json:"heartbeat_interval_seconds,omitempty"`
+	ReconnectDelaySeconds    int32                  `protobuf:"varint,4,opt,name=reconnect_delay_seconds,json=reconnectDelaySeconds,proto3" json:"reconnect_delay_seconds,omitempty"`
+	unknownFields            protoimpl.UnknownFields
+	sizeCache                protoimpl.SizeCache
+}
+
+func (x *AdminHello) Reset() {
+	*x = AdminHello{}
+	mi := &file_plugin_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AdminHello) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AdminHello) ProtoMessage() {}
+
+func (x *AdminHello) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AdminHello.ProtoReflect.Descriptor instead.
+func (*AdminHello) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *AdminHello) GetAccepted() bool {
+	if x != nil {
+		return x.Accepted
+	}
+	return false
+}
+
+func (x *AdminHello) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *AdminHello) GetHeartbeatIntervalSeconds() int32 {
+	if x != nil {
+		return x.HeartbeatIntervalSeconds
+	}
+	return 0
+}
+
+func (x *AdminHello) GetReconnectDelaySeconds() int32 {
+	if x != nil {
+		return x.ReconnectDelaySeconds
+	}
+	return 0
+}
+
+type WorkerHeartbeat struct {
+	state               protoimpl.MessageState `protogen:"open.v1"`
+	WorkerId            string                 `protobuf:"bytes,1,opt,name=worker_id,json=workerId,proto3" json:"worker_id,omitempty"`
+	RunningWork         []*RunningWork         `protobuf:"bytes,2,rep,name=running_work,json=runningWork,proto3" json:"running_work,omitempty"`
+	DetectionSlotsUsed  int32                  `protobuf:"varint,3,opt,name=detection_slots_used,json=detectionSlotsUsed,proto3" json:"detection_slots_used,omitempty"`
+	DetectionSlotsTotal int32                  `protobuf:"varint,4,opt,name=detection_slots_total,json=detectionSlotsTotal,proto3" json:"detection_slots_total,omitempty"`
+	ExecutionSlotsUsed  int32                  `protobuf:"varint,5,opt,name=execution_slots_used,json=executionSlotsUsed,proto3" json:"execution_slots_used,omitempty"`
+	ExecutionSlotsTotal int32                  `protobuf:"varint,6,opt,name=execution_slots_total,json=executionSlotsTotal,proto3" json:"execution_slots_total,omitempty"`
+	QueuedJobsByType    map[string]int32       `protobuf:"bytes,7,rep,name=queued_jobs_by_type,json=queuedJobsByType,proto3" json:"queued_jobs_by_type,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
+	Metadata            map[string]string      `protobuf:"bytes,8,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
+}
+
+func (x *WorkerHeartbeat) Reset() {
+	*x = WorkerHeartbeat{}
+	mi := &file_plugin_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *WorkerHeartbeat) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkerHeartbeat) ProtoMessage() {}
+
+func (x *WorkerHeartbeat) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkerHeartbeat.ProtoReflect.Descriptor instead.
+func (*WorkerHeartbeat) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *WorkerHeartbeat) GetWorkerId() string {
+	if x != nil {
+		return x.WorkerId
+	}
+	return ""
+}
+
+func (x *WorkerHeartbeat) GetRunningWork() []*RunningWork {
+	if x != nil {
+		return x.RunningWork
+	}
+	return nil
+}
+
+func (x *WorkerHeartbeat) GetDetectionSlotsUsed() int32 {
+	if x != nil {
+		return x.DetectionSlotsUsed
+	}
+	return 0
+}
+
+func (x *WorkerHeartbeat) GetDetectionSlotsTotal() int32 {
+	if x != nil {
+		return x.DetectionSlotsTotal
+	}
+	return 0
+}
+
+func (x *WorkerHeartbeat) GetExecutionSlotsUsed() int32 {
+	if x != nil {
+		return x.ExecutionSlotsUsed
+	}
+	return 0
+}
+
+func (x *WorkerHeartbeat) GetExecutionSlotsTotal() int32 {
+	if x != nil {
+		return x.ExecutionSlotsTotal
+	}
+	return 0
+}
+
+func (x *WorkerHeartbeat) GetQueuedJobsByType() map[string]int32 {
+	if x != nil {
+		return x.QueuedJobsByType
+	}
+	return nil
+}
+
+func (x *WorkerHeartbeat) GetMetadata() map[string]string {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+type WorkerAcknowledge struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	RequestId     string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	Accepted      bool                   `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"`
+	Message       string                 `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *WorkerAcknowledge) Reset() {
+	*x = WorkerAcknowledge{}
+	mi := &file_plugin_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *WorkerAcknowledge) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*WorkerAcknowledge) ProtoMessage() {}
+
+func (x *WorkerAcknowledge) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use WorkerAcknowledge.ProtoReflect.Descriptor instead.
+func (*WorkerAcknowledge) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *WorkerAcknowledge) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *WorkerAcknowledge) GetAccepted() bool {
+	if x != nil {
+		return x.Accepted
+	}
+	return false
+}
+
+func (x *WorkerAcknowledge) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+type RunningWork struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	WorkId          string                 `protobuf:"bytes,1,opt,name=work_id,json=workId,proto3" json:"work_id,omitempty"`
+	Kind            WorkKind               `protobuf:"varint,2,opt,name=kind,proto3,enum=plugin.WorkKind" json:"kind,omitempty"`
+	JobType         string                 `protobuf:"bytes,3,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	State           JobState               `protobuf:"varint,4,opt,name=state,proto3,enum=plugin.JobState" json:"state,omitempty"`
+	ProgressPercent float64                `protobuf:"fixed64,5,opt,name=progress_percent,json=progressPercent,proto3" json:"progress_percent,omitempty"`
+	Stage           string                 `protobuf:"bytes,6,opt,name=stage,proto3" json:"stage,omitempty"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *RunningWork) Reset() {
+	*x = RunningWork{}
+	mi := &file_plugin_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RunningWork) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RunningWork) ProtoMessage() {}
+
+func (x *RunningWork) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RunningWork.ProtoReflect.Descriptor instead.
+func (*RunningWork) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *RunningWork) GetWorkId() string {
+	if x != nil {
+		return x.WorkId
+	}
+	return ""
+}
+
+func (x *RunningWork) GetKind() WorkKind {
+	if x != nil {
+		return x.Kind
+	}
+	return WorkKind_WORK_KIND_UNSPECIFIED
+}
+
+func (x *RunningWork) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *RunningWork) GetState() JobState {
+	if x != nil {
+		return x.State
+	}
+	return JobState_JOB_STATE_UNSPECIFIED
+}
+
+func (x *RunningWork) GetProgressPercent() float64 {
+	if x != nil {
+		return x.ProgressPercent
+	}
+	return 0
+}
+
+func (x *RunningWork) GetStage() string {
+	if x != nil {
+		return x.Stage
+	}
+	return ""
+}
+
+type JobTypeCapability struct {
+	state                   protoimpl.MessageState `protogen:"open.v1"`
+	JobType                 string                 `protobuf:"bytes,1,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	CanDetect               bool                   `protobuf:"varint,2,opt,name=can_detect,json=canDetect,proto3" json:"can_detect,omitempty"`
+	CanExecute              bool                   `protobuf:"varint,3,opt,name=can_execute,json=canExecute,proto3" json:"can_execute,omitempty"`
+	MaxDetectionConcurrency int32                  `protobuf:"varint,4,opt,name=max_detection_concurrency,json=maxDetectionConcurrency,proto3" json:"max_detection_concurrency,omitempty"`
+	MaxExecutionConcurrency int32                  `protobuf:"varint,5,opt,name=max_execution_concurrency,json=maxExecutionConcurrency,proto3" json:"max_execution_concurrency,omitempty"`
+	DisplayName             string                 `protobuf:"bytes,6,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"`
+	Description             string                 `protobuf:"bytes,7,opt,name=description,proto3" json:"description,omitempty"`
+	unknownFields           protoimpl.UnknownFields
+	sizeCache               protoimpl.SizeCache
+}
+
+func (x *JobTypeCapability) Reset() {
+	*x = JobTypeCapability{}
+	mi := &file_plugin_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobTypeCapability) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobTypeCapability) ProtoMessage() {}
+
+func (x *JobTypeCapability) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobTypeCapability.ProtoReflect.Descriptor instead.
+func (*JobTypeCapability) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *JobTypeCapability) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobTypeCapability) GetCanDetect() bool {
+	if x != nil {
+		return x.CanDetect
+	}
+	return false
+}
+
+func (x *JobTypeCapability) GetCanExecute() bool {
+	if x != nil {
+		return x.CanExecute
+	}
+	return false
+}
+
+func (x *JobTypeCapability) GetMaxDetectionConcurrency() int32 {
+	if x != nil {
+		return x.MaxDetectionConcurrency
+	}
+	return 0
+}
+
+func (x *JobTypeCapability) GetMaxExecutionConcurrency() int32 {
+	if x != nil {
+		return x.MaxExecutionConcurrency
+	}
+	return 0
+}
+
+func (x *JobTypeCapability) GetDisplayName() string {
+	if x != nil {
+		return x.DisplayName
+	}
+	return ""
+}
+
+func (x *JobTypeCapability) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+type RequestConfigSchema struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	JobType       string                 `protobuf:"bytes,1,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	ForceRefresh  bool                   `protobuf:"varint,2,opt,name=force_refresh,json=forceRefresh,proto3" json:"force_refresh,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *RequestConfigSchema) Reset() {
+	*x = RequestConfigSchema{}
+	mi := &file_plugin_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RequestConfigSchema) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RequestConfigSchema) ProtoMessage() {}
+
+func (x *RequestConfigSchema) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[8]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RequestConfigSchema.ProtoReflect.Descriptor instead.
+func (*RequestConfigSchema) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *RequestConfigSchema) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *RequestConfigSchema) GetForceRefresh() bool {
+	if x != nil {
+		return x.ForceRefresh
+	}
+	return false
+}
+
+type ConfigSchemaResponse struct {
+	state             protoimpl.MessageState `protogen:"open.v1"`
+	RequestId         string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobType           string                 `protobuf:"bytes,2,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	Success           bool                   `protobuf:"varint,3,opt,name=success,proto3" json:"success,omitempty"`
+	ErrorMessage      string                 `protobuf:"bytes,4,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
+	JobTypeDescriptor *JobTypeDescriptor     `protobuf:"bytes,5,opt,name=job_type_descriptor,json=jobTypeDescriptor,proto3" json:"job_type_descriptor,omitempty"`
+	unknownFields     protoimpl.UnknownFields
+	sizeCache         protoimpl.SizeCache
+}
+
+func (x *ConfigSchemaResponse) Reset() {
+	*x = ConfigSchemaResponse{}
+	mi := &file_plugin_proto_msgTypes[9]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigSchemaResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigSchemaResponse) ProtoMessage() {}
+
+func (x *ConfigSchemaResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[9]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigSchemaResponse.ProtoReflect.Descriptor instead.
+func (*ConfigSchemaResponse) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *ConfigSchemaResponse) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *ConfigSchemaResponse) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *ConfigSchemaResponse) GetSuccess() bool {
+	if x != nil {
+		return x.Success
+	}
+	return false
+}
+
+func (x *ConfigSchemaResponse) GetErrorMessage() string {
+	if x != nil {
+		return x.ErrorMessage
+	}
+	return ""
+}
+
+func (x *ConfigSchemaResponse) GetJobTypeDescriptor() *JobTypeDescriptor {
+	if x != nil {
+		return x.JobTypeDescriptor
+	}
+	return nil
+}
+
+// JobTypeDescriptor defines one job type contract, including UI schema and defaults.
+type JobTypeDescriptor struct {
+	state             protoimpl.MessageState `protogen:"open.v1"`
+	JobType           string                 `protobuf:"bytes,1,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	DisplayName       string                 `protobuf:"bytes,2,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"`
+	Description       string                 `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"`
+	Icon              string                 `protobuf:"bytes,4,opt,name=icon,proto3" json:"icon,omitempty"`
+	DescriptorVersion uint32                 `protobuf:"varint,5,opt,name=descriptor_version,json=descriptorVersion,proto3" json:"descriptor_version,omitempty"`
+	// Admin-owned options such as detection frequency and dispatch concurrency.
+	AdminConfigForm *ConfigForm `protobuf:"bytes,6,opt,name=admin_config_form,json=adminConfigForm,proto3" json:"admin_config_form,omitempty"`
+	// Worker-owned options used during detection and execution.
+	WorkerConfigForm     *ConfigForm             `protobuf:"bytes,7,opt,name=worker_config_form,json=workerConfigForm,proto3" json:"worker_config_form,omitempty"`
+	AdminRuntimeDefaults *AdminRuntimeDefaults   `protobuf:"bytes,8,opt,name=admin_runtime_defaults,json=adminRuntimeDefaults,proto3" json:"admin_runtime_defaults,omitempty"`
+	WorkerDefaultValues  map[string]*ConfigValue `protobuf:"bytes,9,rep,name=worker_default_values,json=workerDefaultValues,proto3" json:"worker_default_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields        protoimpl.UnknownFields
+	sizeCache            protoimpl.SizeCache
+}
+
+func (x *JobTypeDescriptor) Reset() {
+	*x = JobTypeDescriptor{}
+	mi := &file_plugin_proto_msgTypes[10]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobTypeDescriptor) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobTypeDescriptor) ProtoMessage() {}
+
+func (x *JobTypeDescriptor) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[10]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobTypeDescriptor.ProtoReflect.Descriptor instead.
+func (*JobTypeDescriptor) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *JobTypeDescriptor) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobTypeDescriptor) GetDisplayName() string {
+	if x != nil {
+		return x.DisplayName
+	}
+	return ""
+}
+
+func (x *JobTypeDescriptor) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *JobTypeDescriptor) GetIcon() string {
+	if x != nil {
+		return x.Icon
+	}
+	return ""
+}
+
+func (x *JobTypeDescriptor) GetDescriptorVersion() uint32 {
+	if x != nil {
+		return x.DescriptorVersion
+	}
+	return 0
+}
+
+func (x *JobTypeDescriptor) GetAdminConfigForm() *ConfigForm {
+	if x != nil {
+		return x.AdminConfigForm
+	}
+	return nil
+}
+
+func (x *JobTypeDescriptor) GetWorkerConfigForm() *ConfigForm {
+	if x != nil {
+		return x.WorkerConfigForm
+	}
+	return nil
+}
+
+func (x *JobTypeDescriptor) GetAdminRuntimeDefaults() *AdminRuntimeDefaults {
+	if x != nil {
+		return x.AdminRuntimeDefaults
+	}
+	return nil
+}
+
+func (x *JobTypeDescriptor) GetWorkerDefaultValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.WorkerDefaultValues
+	}
+	return nil
+}
+
+type ConfigForm struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	FormId        string                  `protobuf:"bytes,1,opt,name=form_id,json=formId,proto3" json:"form_id,omitempty"`
+	Title         string                  `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"`
+	Description   string                  `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"`
+	Sections      []*ConfigSection        `protobuf:"bytes,4,rep,name=sections,proto3" json:"sections,omitempty"`
+	DefaultValues map[string]*ConfigValue `protobuf:"bytes,5,rep,name=default_values,json=defaultValues,proto3" json:"default_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ConfigForm) Reset() {
+	*x = ConfigForm{}
+	mi := &file_plugin_proto_msgTypes[11]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigForm) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigForm) ProtoMessage() {}
+
+func (x *ConfigForm) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[11]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigForm.ProtoReflect.Descriptor instead.
+func (*ConfigForm) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{11}
+}
+
+func (x *ConfigForm) GetFormId() string {
+	if x != nil {
+		return x.FormId
+	}
+	return ""
+}
+
+func (x *ConfigForm) GetTitle() string {
+	if x != nil {
+		return x.Title
+	}
+	return ""
+}
+
+func (x *ConfigForm) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *ConfigForm) GetSections() []*ConfigSection {
+	if x != nil {
+		return x.Sections
+	}
+	return nil
+}
+
+func (x *ConfigForm) GetDefaultValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.DefaultValues
+	}
+	return nil
+}
+
+type ConfigSection struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	SectionId     string                 `protobuf:"bytes,1,opt,name=section_id,json=sectionId,proto3" json:"section_id,omitempty"`
+	Title         string                 `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"`
+	Description   string                 `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"`
+	Fields        []*ConfigField         `protobuf:"bytes,4,rep,name=fields,proto3" json:"fields,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ConfigSection) Reset() {
+	*x = ConfigSection{}
+	mi := &file_plugin_proto_msgTypes[12]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigSection) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigSection) ProtoMessage() {}
+
+func (x *ConfigSection) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[12]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigSection.ProtoReflect.Descriptor instead.
+func (*ConfigSection) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{12}
+}
+
+func (x *ConfigSection) GetSectionId() string {
+	if x != nil {
+		return x.SectionId
+	}
+	return ""
+}
+
+func (x *ConfigSection) GetTitle() string {
+	if x != nil {
+		return x.Title
+	}
+	return ""
+}
+
+func (x *ConfigSection) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *ConfigSection) GetFields() []*ConfigField {
+	if x != nil {
+		return x.Fields
+	}
+	return nil
+}
+
+type ConfigField struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	Name            string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Label           string                 `protobuf:"bytes,2,opt,name=label,proto3" json:"label,omitempty"`
+	Description     string                 `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"`
+	HelpText        string                 `protobuf:"bytes,4,opt,name=help_text,json=helpText,proto3" json:"help_text,omitempty"`
+	Placeholder     string                 `protobuf:"bytes,5,opt,name=placeholder,proto3" json:"placeholder,omitempty"`
+	FieldType       ConfigFieldType        `protobuf:"varint,6,opt,name=field_type,json=fieldType,proto3,enum=plugin.ConfigFieldType" json:"field_type,omitempty"`
+	Widget          ConfigWidget           `protobuf:"varint,7,opt,name=widget,proto3,enum=plugin.ConfigWidget" json:"widget,omitempty"`
+	Required        bool                   `protobuf:"varint,8,opt,name=required,proto3" json:"required,omitempty"`
+	ReadOnly        bool                   `protobuf:"varint,9,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"`
+	Sensitive       bool                   `protobuf:"varint,10,opt,name=sensitive,proto3" json:"sensitive,omitempty"`
+	MinValue        *ConfigValue           `protobuf:"bytes,11,opt,name=min_value,json=minValue,proto3" json:"min_value,omitempty"`
+	MaxValue        *ConfigValue           `protobuf:"bytes,12,opt,name=max_value,json=maxValue,proto3" json:"max_value,omitempty"`
+	Options         []*ConfigOption        `protobuf:"bytes,13,rep,name=options,proto3" json:"options,omitempty"`
+	ValidationRules []*ValidationRule      `protobuf:"bytes,14,rep,name=validation_rules,json=validationRules,proto3" json:"validation_rules,omitempty"`
+	// Simple visibility dependency: show this field when the referenced field equals value.
+	VisibleWhenField  string       `protobuf:"bytes,15,opt,name=visible_when_field,json=visibleWhenField,proto3" json:"visible_when_field,omitempty"`
+	VisibleWhenEquals *ConfigValue `protobuf:"bytes,16,opt,name=visible_when_equals,json=visibleWhenEquals,proto3" json:"visible_when_equals,omitempty"`
+	unknownFields     protoimpl.UnknownFields
+	sizeCache         protoimpl.SizeCache
+}
+
+func (x *ConfigField) Reset() {
+	*x = ConfigField{}
+	mi := &file_plugin_proto_msgTypes[13]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigField) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigField) ProtoMessage() {}
+
+func (x *ConfigField) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[13]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigField.ProtoReflect.Descriptor instead.
+func (*ConfigField) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{13}
+}
+
+func (x *ConfigField) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *ConfigField) GetLabel() string {
+	if x != nil {
+		return x.Label
+	}
+	return ""
+}
+
+func (x *ConfigField) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *ConfigField) GetHelpText() string {
+	if x != nil {
+		return x.HelpText
+	}
+	return ""
+}
+
+func (x *ConfigField) GetPlaceholder() string {
+	if x != nil {
+		return x.Placeholder
+	}
+	return ""
+}
+
+func (x *ConfigField) GetFieldType() ConfigFieldType {
+	if x != nil {
+		return x.FieldType
+	}
+	return ConfigFieldType_CONFIG_FIELD_TYPE_UNSPECIFIED
+}
+
+func (x *ConfigField) GetWidget() ConfigWidget {
+	if x != nil {
+		return x.Widget
+	}
+	return ConfigWidget_CONFIG_WIDGET_UNSPECIFIED
+}
+
+func (x *ConfigField) GetRequired() bool {
+	if x != nil {
+		return x.Required
+	}
+	return false
+}
+
+func (x *ConfigField) GetReadOnly() bool {
+	if x != nil {
+		return x.ReadOnly
+	}
+	return false
+}
+
+func (x *ConfigField) GetSensitive() bool {
+	if x != nil {
+		return x.Sensitive
+	}
+	return false
+}
+
+func (x *ConfigField) GetMinValue() *ConfigValue {
+	if x != nil {
+		return x.MinValue
+	}
+	return nil
+}
+
+func (x *ConfigField) GetMaxValue() *ConfigValue {
+	if x != nil {
+		return x.MaxValue
+	}
+	return nil
+}
+
+func (x *ConfigField) GetOptions() []*ConfigOption {
+	if x != nil {
+		return x.Options
+	}
+	return nil
+}
+
+func (x *ConfigField) GetValidationRules() []*ValidationRule {
+	if x != nil {
+		return x.ValidationRules
+	}
+	return nil
+}
+
+func (x *ConfigField) GetVisibleWhenField() string {
+	if x != nil {
+		return x.VisibleWhenField
+	}
+	return ""
+}
+
+func (x *ConfigField) GetVisibleWhenEquals() *ConfigValue {
+	if x != nil {
+		return x.VisibleWhenEquals
+	}
+	return nil
+}
+
+type ConfigOption struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Value         string                 `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"`
+	Label         string                 `protobuf:"bytes,2,opt,name=label,proto3" json:"label,omitempty"`
+	Description   string                 `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"`
+	Disabled      bool                   `protobuf:"varint,4,opt,name=disabled,proto3" json:"disabled,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ConfigOption) Reset() {
+	*x = ConfigOption{}
+	mi := &file_plugin_proto_msgTypes[14]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigOption) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigOption) ProtoMessage() {}
+
+func (x *ConfigOption) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[14]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigOption.ProtoReflect.Descriptor instead.
+func (*ConfigOption) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{14}
+}
+
+func (x *ConfigOption) GetValue() string {
+	if x != nil {
+		return x.Value
+	}
+	return ""
+}
+
+func (x *ConfigOption) GetLabel() string {
+	if x != nil {
+		return x.Label
+	}
+	return ""
+}
+
+func (x *ConfigOption) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *ConfigOption) GetDisabled() bool {
+	if x != nil {
+		return x.Disabled
+	}
+	return false
+}
+
+type ValidationRule struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Type          ValidationRuleType     `protobuf:"varint,1,opt,name=type,proto3,enum=plugin.ValidationRuleType" json:"type,omitempty"`
+	Expression    string                 `protobuf:"bytes,2,opt,name=expression,proto3" json:"expression,omitempty"`
+	ErrorMessage  string                 `protobuf:"bytes,3,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ValidationRule) Reset() {
+	*x = ValidationRule{}
+	mi := &file_plugin_proto_msgTypes[15]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ValidationRule) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValidationRule) ProtoMessage() {}
+
+func (x *ValidationRule) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[15]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValidationRule.ProtoReflect.Descriptor instead.
+func (*ValidationRule) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{15}
+}
+
+func (x *ValidationRule) GetType() ValidationRuleType {
+	if x != nil {
+		return x.Type
+	}
+	return ValidationRuleType_VALIDATION_RULE_TYPE_UNSPECIFIED
+}
+
+func (x *ValidationRule) GetExpression() string {
+	if x != nil {
+		return x.Expression
+	}
+	return ""
+}
+
+func (x *ValidationRule) GetErrorMessage() string {
+	if x != nil {
+		return x.ErrorMessage
+	}
+	return ""
+}
+
+type ConfigValue struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to Kind:
+	//
+	//	*ConfigValue_BoolValue
+	//	*ConfigValue_Int64Value
+	//	*ConfigValue_DoubleValue
+	//	*ConfigValue_StringValue
+	//	*ConfigValue_BytesValue
+	//	*ConfigValue_DurationValue
+	//	*ConfigValue_StringList
+	//	*ConfigValue_Int64List
+	//	*ConfigValue_DoubleList
+	//	*ConfigValue_BoolList
+	//	*ConfigValue_ListValue
+	//	*ConfigValue_MapValue
+	Kind          isConfigValue_Kind `protobuf_oneof:"kind"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ConfigValue) Reset() {
+	*x = ConfigValue{}
+	mi := &file_plugin_proto_msgTypes[16]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ConfigValue) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ConfigValue) ProtoMessage() {}
+
+func (x *ConfigValue) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[16]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ConfigValue.ProtoReflect.Descriptor instead.
+func (*ConfigValue) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{16}
+}
+
+func (x *ConfigValue) GetKind() isConfigValue_Kind {
+	if x != nil {
+		return x.Kind
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetBoolValue() bool {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_BoolValue); ok {
+			return x.BoolValue
+		}
+	}
+	return false
+}
+
+func (x *ConfigValue) GetInt64Value() int64 {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_Int64Value); ok {
+			return x.Int64Value
+		}
+	}
+	return 0
+}
+
+func (x *ConfigValue) GetDoubleValue() float64 {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_DoubleValue); ok {
+			return x.DoubleValue
+		}
+	}
+	return 0
+}
+
+func (x *ConfigValue) GetStringValue() string {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_StringValue); ok {
+			return x.StringValue
+		}
+	}
+	return ""
+}
+
+func (x *ConfigValue) GetBytesValue() []byte {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_BytesValue); ok {
+			return x.BytesValue
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetDurationValue() *durationpb.Duration {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_DurationValue); ok {
+			return x.DurationValue
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetStringList() *StringList {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_StringList); ok {
+			return x.StringList
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetInt64List() *Int64List {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_Int64List); ok {
+			return x.Int64List
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetDoubleList() *DoubleList {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_DoubleList); ok {
+			return x.DoubleList
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetBoolList() *BoolList {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_BoolList); ok {
+			return x.BoolList
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetListValue() *ValueList {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_ListValue); ok {
+			return x.ListValue
+		}
+	}
+	return nil
+}
+
+func (x *ConfigValue) GetMapValue() *ValueMap {
+	if x != nil {
+		if x, ok := x.Kind.(*ConfigValue_MapValue); ok {
+			return x.MapValue
+		}
+	}
+	return nil
+}
+
+type isConfigValue_Kind interface {
+	isConfigValue_Kind()
+}
+
+type ConfigValue_BoolValue struct {
+	BoolValue bool `protobuf:"varint,1,opt,name=bool_value,json=boolValue,proto3,oneof"`
+}
+
+type ConfigValue_Int64Value struct {
+	Int64Value int64 `protobuf:"varint,2,opt,name=int64_value,json=int64Value,proto3,oneof"`
+}
+
+type ConfigValue_DoubleValue struct {
+	DoubleValue float64 `protobuf:"fixed64,3,opt,name=double_value,json=doubleValue,proto3,oneof"`
+}
+
+type ConfigValue_StringValue struct {
+	StringValue string `protobuf:"bytes,4,opt,name=string_value,json=stringValue,proto3,oneof"`
+}
+
+type ConfigValue_BytesValue struct {
+	BytesValue []byte `protobuf:"bytes,5,opt,name=bytes_value,json=bytesValue,proto3,oneof"`
+}
+
+type ConfigValue_DurationValue struct {
+	DurationValue *durationpb.Duration `protobuf:"bytes,6,opt,name=duration_value,json=durationValue,proto3,oneof"`
+}
+
+type ConfigValue_StringList struct {
+	StringList *StringList `protobuf:"bytes,7,opt,name=string_list,json=stringList,proto3,oneof"`
+}
+
+type ConfigValue_Int64List struct {
+	Int64List *Int64List `protobuf:"bytes,8,opt,name=int64_list,json=int64List,proto3,oneof"`
+}
+
+type ConfigValue_DoubleList struct {
+	DoubleList *DoubleList `protobuf:"bytes,9,opt,name=double_list,json=doubleList,proto3,oneof"`
+}
+
+type ConfigValue_BoolList struct {
+	BoolList *BoolList `protobuf:"bytes,10,opt,name=bool_list,json=boolList,proto3,oneof"`
+}
+
+type ConfigValue_ListValue struct {
+	ListValue *ValueList `protobuf:"bytes,11,opt,name=list_value,json=listValue,proto3,oneof"`
+}
+
+type ConfigValue_MapValue struct {
+	MapValue *ValueMap `protobuf:"bytes,12,opt,name=map_value,json=mapValue,proto3,oneof"`
+}
+
+func (*ConfigValue_BoolValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_Int64Value) isConfigValue_Kind() {}
+
+func (*ConfigValue_DoubleValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_StringValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_BytesValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_DurationValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_StringList) isConfigValue_Kind() {}
+
+func (*ConfigValue_Int64List) isConfigValue_Kind() {}
+
+func (*ConfigValue_DoubleList) isConfigValue_Kind() {}
+
+func (*ConfigValue_BoolList) isConfigValue_Kind() {}
+
+func (*ConfigValue_ListValue) isConfigValue_Kind() {}
+
+func (*ConfigValue_MapValue) isConfigValue_Kind() {}
+
+type StringList struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []string               `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *StringList) Reset() {
+	*x = StringList{}
+	mi := &file_plugin_proto_msgTypes[17]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *StringList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*StringList) ProtoMessage() {}
+
+func (x *StringList) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[17]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use StringList.ProtoReflect.Descriptor instead.
+func (*StringList) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{17}
+}
+
+func (x *StringList) GetValues() []string {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
+type Int64List struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []int64                `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Int64List) Reset() {
+	*x = Int64List{}
+	mi := &file_plugin_proto_msgTypes[18]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Int64List) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Int64List) ProtoMessage() {}
+
+func (x *Int64List) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[18]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Int64List.ProtoReflect.Descriptor instead.
+func (*Int64List) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{18}
+}
+
+func (x *Int64List) GetValues() []int64 {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
+type DoubleList struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []float64              `protobuf:"fixed64,1,rep,packed,name=values,proto3" json:"values,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *DoubleList) Reset() {
+	*x = DoubleList{}
+	mi := &file_plugin_proto_msgTypes[19]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *DoubleList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DoubleList) ProtoMessage() {}
+
+func (x *DoubleList) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[19]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DoubleList.ProtoReflect.Descriptor instead.
+func (*DoubleList) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{19}
+}
+
+func (x *DoubleList) GetValues() []float64 {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
+type BoolList struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []bool                 `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BoolList) Reset() {
+	*x = BoolList{}
+	mi := &file_plugin_proto_msgTypes[20]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BoolList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BoolList) ProtoMessage() {}
+
+func (x *BoolList) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[20]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BoolList.ProtoReflect.Descriptor instead.
+func (*BoolList) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{20}
+}
+
+func (x *BoolList) GetValues() []bool {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
+type ValueList struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Values        []*ConfigValue         `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ValueList) Reset() {
+	*x = ValueList{}
+	mi := &file_plugin_proto_msgTypes[21]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ValueList) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValueList) ProtoMessage() {}
+
+func (x *ValueList) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[21]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValueList.ProtoReflect.Descriptor instead.
+func (*ValueList) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{21}
+}
+
+func (x *ValueList) GetValues() []*ConfigValue {
+	if x != nil {
+		return x.Values
+	}
+	return nil
+}
+
+type ValueMap struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	Fields        map[string]*ConfigValue `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ValueMap) Reset() {
+	*x = ValueMap{}
+	mi := &file_plugin_proto_msgTypes[22]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ValueMap) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ValueMap) ProtoMessage() {}
+
+func (x *ValueMap) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[22]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ValueMap.ProtoReflect.Descriptor instead.
+func (*ValueMap) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{22}
+}
+
+func (x *ValueMap) GetFields() map[string]*ConfigValue {
+	if x != nil {
+		return x.Fields
+	}
+	return nil
+}
+
+type AdminRuntimeDefaults struct {
+	state                         protoimpl.MessageState `protogen:"open.v1"`
+	Enabled                       bool                   `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+	DetectionIntervalSeconds      int32                  `protobuf:"varint,2,opt,name=detection_interval_seconds,json=detectionIntervalSeconds,proto3" json:"detection_interval_seconds,omitempty"`
+	DetectionTimeoutSeconds       int32                  `protobuf:"varint,3,opt,name=detection_timeout_seconds,json=detectionTimeoutSeconds,proto3" json:"detection_timeout_seconds,omitempty"`
+	MaxJobsPerDetection           int32                  `protobuf:"varint,4,opt,name=max_jobs_per_detection,json=maxJobsPerDetection,proto3" json:"max_jobs_per_detection,omitempty"`
+	GlobalExecutionConcurrency    int32                  `protobuf:"varint,5,opt,name=global_execution_concurrency,json=globalExecutionConcurrency,proto3" json:"global_execution_concurrency,omitempty"`
+	PerWorkerExecutionConcurrency int32                  `protobuf:"varint,6,opt,name=per_worker_execution_concurrency,json=perWorkerExecutionConcurrency,proto3" json:"per_worker_execution_concurrency,omitempty"`
+	RetryLimit                    int32                  `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
+	RetryBackoffSeconds           int32                  `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
+	unknownFields                 protoimpl.UnknownFields
+	sizeCache                     protoimpl.SizeCache
+}
+
+func (x *AdminRuntimeDefaults) Reset() {
+	*x = AdminRuntimeDefaults{}
+	mi := &file_plugin_proto_msgTypes[23]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AdminRuntimeDefaults) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AdminRuntimeDefaults) ProtoMessage() {}
+
+func (x *AdminRuntimeDefaults) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[23]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AdminRuntimeDefaults.ProtoReflect.Descriptor instead.
+func (*AdminRuntimeDefaults) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{23}
+}
+
+func (x *AdminRuntimeDefaults) GetEnabled() bool {
+	if x != nil {
+		return x.Enabled
+	}
+	return false
+}
+
+func (x *AdminRuntimeDefaults) GetDetectionIntervalSeconds() int32 {
+	if x != nil {
+		return x.DetectionIntervalSeconds
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetDetectionTimeoutSeconds() int32 {
+	if x != nil {
+		return x.DetectionTimeoutSeconds
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetMaxJobsPerDetection() int32 {
+	if x != nil {
+		return x.MaxJobsPerDetection
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetGlobalExecutionConcurrency() int32 {
+	if x != nil {
+		return x.GlobalExecutionConcurrency
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetPerWorkerExecutionConcurrency() int32 {
+	if x != nil {
+		return x.PerWorkerExecutionConcurrency
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetRetryLimit() int32 {
+	if x != nil {
+		return x.RetryLimit
+	}
+	return 0
+}
+
+func (x *AdminRuntimeDefaults) GetRetryBackoffSeconds() int32 {
+	if x != nil {
+		return x.RetryBackoffSeconds
+	}
+	return 0
+}
+
+type AdminRuntimeConfig struct {
+	state                         protoimpl.MessageState `protogen:"open.v1"`
+	Enabled                       bool                   `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"`
+	DetectionIntervalSeconds      int32                  `protobuf:"varint,2,opt,name=detection_interval_seconds,json=detectionIntervalSeconds,proto3" json:"detection_interval_seconds,omitempty"`
+	DetectionTimeoutSeconds       int32                  `protobuf:"varint,3,opt,name=detection_timeout_seconds,json=detectionTimeoutSeconds,proto3" json:"detection_timeout_seconds,omitempty"`
+	MaxJobsPerDetection           int32                  `protobuf:"varint,4,opt,name=max_jobs_per_detection,json=maxJobsPerDetection,proto3" json:"max_jobs_per_detection,omitempty"`
+	GlobalExecutionConcurrency    int32                  `protobuf:"varint,5,opt,name=global_execution_concurrency,json=globalExecutionConcurrency,proto3" json:"global_execution_concurrency,omitempty"`
+	PerWorkerExecutionConcurrency int32                  `protobuf:"varint,6,opt,name=per_worker_execution_concurrency,json=perWorkerExecutionConcurrency,proto3" json:"per_worker_execution_concurrency,omitempty"`
+	RetryLimit                    int32                  `protobuf:"varint,7,opt,name=retry_limit,json=retryLimit,proto3" json:"retry_limit,omitempty"`
+	RetryBackoffSeconds           int32                  `protobuf:"varint,8,opt,name=retry_backoff_seconds,json=retryBackoffSeconds,proto3" json:"retry_backoff_seconds,omitempty"`
+	unknownFields                 protoimpl.UnknownFields
+	sizeCache                     protoimpl.SizeCache
+}
+
+func (x *AdminRuntimeConfig) Reset() {
+	*x = AdminRuntimeConfig{}
+	mi := &file_plugin_proto_msgTypes[24]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AdminRuntimeConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AdminRuntimeConfig) ProtoMessage() {}
+
+func (x *AdminRuntimeConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[24]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AdminRuntimeConfig.ProtoReflect.Descriptor instead.
+func (*AdminRuntimeConfig) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{24}
+}
+
+func (x *AdminRuntimeConfig) GetEnabled() bool {
+	if x != nil {
+		return x.Enabled
+	}
+	return false
+}
+
+func (x *AdminRuntimeConfig) GetDetectionIntervalSeconds() int32 {
+	if x != nil {
+		return x.DetectionIntervalSeconds
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetDetectionTimeoutSeconds() int32 {
+	if x != nil {
+		return x.DetectionTimeoutSeconds
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetMaxJobsPerDetection() int32 {
+	if x != nil {
+		return x.MaxJobsPerDetection
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetGlobalExecutionConcurrency() int32 {
+	if x != nil {
+		return x.GlobalExecutionConcurrency
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetPerWorkerExecutionConcurrency() int32 {
+	if x != nil {
+		return x.PerWorkerExecutionConcurrency
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetRetryLimit() int32 {
+	if x != nil {
+		return x.RetryLimit
+	}
+	return 0
+}
+
+func (x *AdminRuntimeConfig) GetRetryBackoffSeconds() int32 {
+	if x != nil {
+		return x.RetryBackoffSeconds
+	}
+	return 0
+}
+
+type RunDetectionRequest struct {
+	state              protoimpl.MessageState  `protogen:"open.v1"`
+	RequestId          string                  `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobType            string                  `protobuf:"bytes,2,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	DetectionSequence  int64                   `protobuf:"varint,3,opt,name=detection_sequence,json=detectionSequence,proto3" json:"detection_sequence,omitempty"`
+	AdminRuntime       *AdminRuntimeConfig     `protobuf:"bytes,4,opt,name=admin_runtime,json=adminRuntime,proto3" json:"admin_runtime,omitempty"`
+	AdminConfigValues  map[string]*ConfigValue `protobuf:"bytes,5,rep,name=admin_config_values,json=adminConfigValues,proto3" json:"admin_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	WorkerConfigValues map[string]*ConfigValue `protobuf:"bytes,6,rep,name=worker_config_values,json=workerConfigValues,proto3" json:"worker_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	ClusterContext     *ClusterContext         `protobuf:"bytes,7,opt,name=cluster_context,json=clusterContext,proto3" json:"cluster_context,omitempty"`
+	LastSuccessfulRun  *timestamppb.Timestamp  `protobuf:"bytes,8,opt,name=last_successful_run,json=lastSuccessfulRun,proto3" json:"last_successful_run,omitempty"`
+	MaxResults         int32                   `protobuf:"varint,9,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
+}
+
+func (x *RunDetectionRequest) Reset() {
+	*x = RunDetectionRequest{}
+	mi := &file_plugin_proto_msgTypes[25]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *RunDetectionRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RunDetectionRequest) ProtoMessage() {}
+
+func (x *RunDetectionRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[25]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RunDetectionRequest.ProtoReflect.Descriptor instead.
+func (*RunDetectionRequest) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{25}
+}
+
+func (x *RunDetectionRequest) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *RunDetectionRequest) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *RunDetectionRequest) GetDetectionSequence() int64 {
+	if x != nil {
+		return x.DetectionSequence
+	}
+	return 0
+}
+
+func (x *RunDetectionRequest) GetAdminRuntime() *AdminRuntimeConfig {
+	if x != nil {
+		return x.AdminRuntime
+	}
+	return nil
+}
+
+func (x *RunDetectionRequest) GetAdminConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.AdminConfigValues
+	}
+	return nil
+}
+
+func (x *RunDetectionRequest) GetWorkerConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.WorkerConfigValues
+	}
+	return nil
+}
+
+func (x *RunDetectionRequest) GetClusterContext() *ClusterContext {
+	if x != nil {
+		return x.ClusterContext
+	}
+	return nil
+}
+
+func (x *RunDetectionRequest) GetLastSuccessfulRun() *timestamppb.Timestamp {
+	if x != nil {
+		return x.LastSuccessfulRun
+	}
+	return nil
+}
+
+func (x *RunDetectionRequest) GetMaxResults() int32 {
+	if x != nil {
+		return x.MaxResults
+	}
+	return 0
+}
+
+type DetectionProposals struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	RequestId     string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobType       string                 `protobuf:"bytes,2,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	Proposals     []*JobProposal         `protobuf:"bytes,3,rep,name=proposals,proto3" json:"proposals,omitempty"`
+	HasMore       bool                   `protobuf:"varint,4,opt,name=has_more,json=hasMore,proto3" json:"has_more,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *DetectionProposals) Reset() {
+	*x = DetectionProposals{}
+	mi := &file_plugin_proto_msgTypes[26]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *DetectionProposals) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DetectionProposals) ProtoMessage() {}
+
+func (x *DetectionProposals) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[26]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DetectionProposals.ProtoReflect.Descriptor instead.
+func (*DetectionProposals) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{26}
+}
+
+func (x *DetectionProposals) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *DetectionProposals) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *DetectionProposals) GetProposals() []*JobProposal {
+	if x != nil {
+		return x.Proposals
+	}
+	return nil
+}
+
+func (x *DetectionProposals) GetHasMore() bool {
+	if x != nil {
+		return x.HasMore
+	}
+	return false
+}
+
+type DetectionComplete struct {
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	RequestId      string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobType        string                 `protobuf:"bytes,2,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	Success        bool                   `protobuf:"varint,3,opt,name=success,proto3" json:"success,omitempty"`
+	ErrorMessage   string                 `protobuf:"bytes,4,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
+	TotalProposals int32                  `protobuf:"varint,5,opt,name=total_proposals,json=totalProposals,proto3" json:"total_proposals,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
+}
+
+func (x *DetectionComplete) Reset() {
+	*x = DetectionComplete{}
+	mi := &file_plugin_proto_msgTypes[27]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *DetectionComplete) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DetectionComplete) ProtoMessage() {}
+
+func (x *DetectionComplete) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[27]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DetectionComplete.ProtoReflect.Descriptor instead.
+func (*DetectionComplete) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{27}
+}
+
+func (x *DetectionComplete) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *DetectionComplete) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *DetectionComplete) GetSuccess() bool {
+	if x != nil {
+		return x.Success
+	}
+	return false
+}
+
+func (x *DetectionComplete) GetErrorMessage() string {
+	if x != nil {
+		return x.ErrorMessage
+	}
+	return ""
+}
+
+func (x *DetectionComplete) GetTotalProposals() int32 {
+	if x != nil {
+		return x.TotalProposals
+	}
+	return 0
+}
+
+type JobProposal struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	ProposalId    string                  `protobuf:"bytes,1,opt,name=proposal_id,json=proposalId,proto3" json:"proposal_id,omitempty"`
+	DedupeKey     string                  `protobuf:"bytes,2,opt,name=dedupe_key,json=dedupeKey,proto3" json:"dedupe_key,omitempty"`
+	JobType       string                  `protobuf:"bytes,3,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	Priority      JobPriority             `protobuf:"varint,4,opt,name=priority,proto3,enum=plugin.JobPriority" json:"priority,omitempty"`
+	Summary       string                  `protobuf:"bytes,5,opt,name=summary,proto3" json:"summary,omitempty"`
+	Detail        string                  `protobuf:"bytes,6,opt,name=detail,proto3" json:"detail,omitempty"`
+	Parameters    map[string]*ConfigValue `protobuf:"bytes,7,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Labels        map[string]string       `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	NotBefore     *timestamppb.Timestamp  `protobuf:"bytes,9,opt,name=not_before,json=notBefore,proto3" json:"not_before,omitempty"`
+	ExpiresAt     *timestamppb.Timestamp  `protobuf:"bytes,10,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *JobProposal) Reset() {
+	*x = JobProposal{}
+	mi := &file_plugin_proto_msgTypes[28]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobProposal) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobProposal) ProtoMessage() {}
+
+func (x *JobProposal) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[28]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobProposal.ProtoReflect.Descriptor instead.
+func (*JobProposal) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{28}
+}
+
+func (x *JobProposal) GetProposalId() string {
+	if x != nil {
+		return x.ProposalId
+	}
+	return ""
+}
+
+func (x *JobProposal) GetDedupeKey() string {
+	if x != nil {
+		return x.DedupeKey
+	}
+	return ""
+}
+
+func (x *JobProposal) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobProposal) GetPriority() JobPriority {
+	if x != nil {
+		return x.Priority
+	}
+	return JobPriority_JOB_PRIORITY_UNSPECIFIED
+}
+
+func (x *JobProposal) GetSummary() string {
+	if x != nil {
+		return x.Summary
+	}
+	return ""
+}
+
+func (x *JobProposal) GetDetail() string {
+	if x != nil {
+		return x.Detail
+	}
+	return ""
+}
+
+func (x *JobProposal) GetParameters() map[string]*ConfigValue {
+	if x != nil {
+		return x.Parameters
+	}
+	return nil
+}
+
+func (x *JobProposal) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+func (x *JobProposal) GetNotBefore() *timestamppb.Timestamp {
+	if x != nil {
+		return x.NotBefore
+	}
+	return nil
+}
+
+func (x *JobProposal) GetExpiresAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.ExpiresAt
+	}
+	return nil
+}
+
+type ExecuteJobRequest struct {
+	state              protoimpl.MessageState  `protogen:"open.v1"`
+	RequestId          string                  `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	Job                *JobSpec                `protobuf:"bytes,2,opt,name=job,proto3" json:"job,omitempty"`
+	AdminRuntime       *AdminRuntimeConfig     `protobuf:"bytes,3,opt,name=admin_runtime,json=adminRuntime,proto3" json:"admin_runtime,omitempty"`
+	AdminConfigValues  map[string]*ConfigValue `protobuf:"bytes,4,rep,name=admin_config_values,json=adminConfigValues,proto3" json:"admin_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	WorkerConfigValues map[string]*ConfigValue `protobuf:"bytes,5,rep,name=worker_config_values,json=workerConfigValues,proto3" json:"worker_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	ClusterContext     *ClusterContext         `protobuf:"bytes,6,opt,name=cluster_context,json=clusterContext,proto3" json:"cluster_context,omitempty"`
+	Attempt            int32                   `protobuf:"varint,7,opt,name=attempt,proto3" json:"attempt,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
+}
+
+func (x *ExecuteJobRequest) Reset() {
+	*x = ExecuteJobRequest{}
+	mi := &file_plugin_proto_msgTypes[29]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ExecuteJobRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ExecuteJobRequest) ProtoMessage() {}
+
+func (x *ExecuteJobRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[29]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ExecuteJobRequest.ProtoReflect.Descriptor instead.
+func (*ExecuteJobRequest) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{29}
+}
+
+func (x *ExecuteJobRequest) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *ExecuteJobRequest) GetJob() *JobSpec {
+	if x != nil {
+		return x.Job
+	}
+	return nil
+}
+
+func (x *ExecuteJobRequest) GetAdminRuntime() *AdminRuntimeConfig {
+	if x != nil {
+		return x.AdminRuntime
+	}
+	return nil
+}
+
+func (x *ExecuteJobRequest) GetAdminConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.AdminConfigValues
+	}
+	return nil
+}
+
+func (x *ExecuteJobRequest) GetWorkerConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.WorkerConfigValues
+	}
+	return nil
+}
+
+func (x *ExecuteJobRequest) GetClusterContext() *ClusterContext {
+	if x != nil {
+		return x.ClusterContext
+	}
+	return nil
+}
+
+func (x *ExecuteJobRequest) GetAttempt() int32 {
+	if x != nil {
+		return x.Attempt
+	}
+	return 0
+}
+
+type JobSpec struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	JobId         string                  `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
+	JobType       string                  `protobuf:"bytes,2,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	DedupeKey     string                  `protobuf:"bytes,3,opt,name=dedupe_key,json=dedupeKey,proto3" json:"dedupe_key,omitempty"`
+	Priority      JobPriority             `protobuf:"varint,4,opt,name=priority,proto3,enum=plugin.JobPriority" json:"priority,omitempty"`
+	Summary       string                  `protobuf:"bytes,5,opt,name=summary,proto3" json:"summary,omitempty"`
+	Detail        string                  `protobuf:"bytes,6,opt,name=detail,proto3" json:"detail,omitempty"`
+	Parameters    map[string]*ConfigValue `protobuf:"bytes,7,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Labels        map[string]string       `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	CreatedAt     *timestamppb.Timestamp  `protobuf:"bytes,9,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"`
+	ScheduledAt   *timestamppb.Timestamp  `protobuf:"bytes,10,opt,name=scheduled_at,json=scheduledAt,proto3" json:"scheduled_at,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *JobSpec) Reset() {
+	*x = JobSpec{}
+	mi := &file_plugin_proto_msgTypes[30]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobSpec) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobSpec) ProtoMessage() {}
+
+func (x *JobSpec) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[30]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobSpec.ProtoReflect.Descriptor instead.
+func (*JobSpec) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{30}
+}
+
+func (x *JobSpec) GetJobId() string {
+	if x != nil {
+		return x.JobId
+	}
+	return ""
+}
+
+func (x *JobSpec) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobSpec) GetDedupeKey() string {
+	if x != nil {
+		return x.DedupeKey
+	}
+	return ""
+}
+
+func (x *JobSpec) GetPriority() JobPriority {
+	if x != nil {
+		return x.Priority
+	}
+	return JobPriority_JOB_PRIORITY_UNSPECIFIED
+}
+
+func (x *JobSpec) GetSummary() string {
+	if x != nil {
+		return x.Summary
+	}
+	return ""
+}
+
+func (x *JobSpec) GetDetail() string {
+	if x != nil {
+		return x.Detail
+	}
+	return ""
+}
+
+func (x *JobSpec) GetParameters() map[string]*ConfigValue {
+	if x != nil {
+		return x.Parameters
+	}
+	return nil
+}
+
+func (x *JobSpec) GetLabels() map[string]string {
+	if x != nil {
+		return x.Labels
+	}
+	return nil
+}
+
+func (x *JobSpec) GetCreatedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.CreatedAt
+	}
+	return nil
+}
+
+func (x *JobSpec) GetScheduledAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.ScheduledAt
+	}
+	return nil
+}
+
+type JobProgressUpdate struct {
+	state           protoimpl.MessageState  `protogen:"open.v1"`
+	RequestId       string                  `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobId           string                  `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
+	JobType         string                  `protobuf:"bytes,3,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	State           JobState                `protobuf:"varint,4,opt,name=state,proto3,enum=plugin.JobState" json:"state,omitempty"`
+	ProgressPercent float64                 `protobuf:"fixed64,5,opt,name=progress_percent,json=progressPercent,proto3" json:"progress_percent,omitempty"`
+	Stage           string                  `protobuf:"bytes,6,opt,name=stage,proto3" json:"stage,omitempty"`
+	Message         string                  `protobuf:"bytes,7,opt,name=message,proto3" json:"message,omitempty"`
+	Metrics         map[string]*ConfigValue `protobuf:"bytes,8,rep,name=metrics,proto3" json:"metrics,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Activities      []*ActivityEvent        `protobuf:"bytes,9,rep,name=activities,proto3" json:"activities,omitempty"`
+	UpdatedAt       *timestamppb.Timestamp  `protobuf:"bytes,10,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *JobProgressUpdate) Reset() {
+	*x = JobProgressUpdate{}
+	mi := &file_plugin_proto_msgTypes[31]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobProgressUpdate) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobProgressUpdate) ProtoMessage() {}
+
+func (x *JobProgressUpdate) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[31]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobProgressUpdate.ProtoReflect.Descriptor instead.
+func (*JobProgressUpdate) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{31}
+}
+
+func (x *JobProgressUpdate) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *JobProgressUpdate) GetJobId() string {
+	if x != nil {
+		return x.JobId
+	}
+	return ""
+}
+
+func (x *JobProgressUpdate) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobProgressUpdate) GetState() JobState {
+	if x != nil {
+		return x.State
+	}
+	return JobState_JOB_STATE_UNSPECIFIED
+}
+
+func (x *JobProgressUpdate) GetProgressPercent() float64 {
+	if x != nil {
+		return x.ProgressPercent
+	}
+	return 0
+}
+
+func (x *JobProgressUpdate) GetStage() string {
+	if x != nil {
+		return x.Stage
+	}
+	return ""
+}
+
+func (x *JobProgressUpdate) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *JobProgressUpdate) GetMetrics() map[string]*ConfigValue {
+	if x != nil {
+		return x.Metrics
+	}
+	return nil
+}
+
+func (x *JobProgressUpdate) GetActivities() []*ActivityEvent {
+	if x != nil {
+		return x.Activities
+	}
+	return nil
+}
+
+func (x *JobProgressUpdate) GetUpdatedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.UpdatedAt
+	}
+	return nil
+}
+
+type JobCompleted struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	RequestId     string                 `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	JobId         string                 `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
+	JobType       string                 `protobuf:"bytes,3,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	Success       bool                   `protobuf:"varint,4,opt,name=success,proto3" json:"success,omitempty"`
+	ErrorMessage  string                 `protobuf:"bytes,5,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"`
+	Result        *JobResult             `protobuf:"bytes,6,opt,name=result,proto3" json:"result,omitempty"`
+	Activities    []*ActivityEvent       `protobuf:"bytes,7,rep,name=activities,proto3" json:"activities,omitempty"`
+	CompletedAt   *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=completed_at,json=completedAt,proto3" json:"completed_at,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *JobCompleted) Reset() {
+	*x = JobCompleted{}
+	mi := &file_plugin_proto_msgTypes[32]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobCompleted) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobCompleted) ProtoMessage() {}
+
+func (x *JobCompleted) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[32]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobCompleted.ProtoReflect.Descriptor instead.
+func (*JobCompleted) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{32}
+}
+
+func (x *JobCompleted) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+func (x *JobCompleted) GetJobId() string {
+	if x != nil {
+		return x.JobId
+	}
+	return ""
+}
+
+func (x *JobCompleted) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *JobCompleted) GetSuccess() bool {
+	if x != nil {
+		return x.Success
+	}
+	return false
+}
+
+func (x *JobCompleted) GetErrorMessage() string {
+	if x != nil {
+		return x.ErrorMessage
+	}
+	return ""
+}
+
+func (x *JobCompleted) GetResult() *JobResult {
+	if x != nil {
+		return x.Result
+	}
+	return nil
+}
+
+func (x *JobCompleted) GetActivities() []*ActivityEvent {
+	if x != nil {
+		return x.Activities
+	}
+	return nil
+}
+
+func (x *JobCompleted) GetCompletedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.CompletedAt
+	}
+	return nil
+}
+
+type JobResult struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	OutputValues  map[string]*ConfigValue `protobuf:"bytes,1,rep,name=output_values,json=outputValues,proto3" json:"output_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Summary       string                  `protobuf:"bytes,2,opt,name=summary,proto3" json:"summary,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *JobResult) Reset() {
+	*x = JobResult{}
+	mi := &file_plugin_proto_msgTypes[33]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *JobResult) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JobResult) ProtoMessage() {}
+
+func (x *JobResult) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[33]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use JobResult.ProtoReflect.Descriptor instead.
+func (*JobResult) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{33}
+}
+
+func (x *JobResult) GetOutputValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.OutputValues
+	}
+	return nil
+}
+
+func (x *JobResult) GetSummary() string {
+	if x != nil {
+		return x.Summary
+	}
+	return ""
+}
+
+type ClusterContext struct {
+	state               protoimpl.MessageState `protogen:"open.v1"`
+	MasterGrpcAddresses []string               `protobuf:"bytes,1,rep,name=master_grpc_addresses,json=masterGrpcAddresses,proto3" json:"master_grpc_addresses,omitempty"`
+	FilerGrpcAddresses  []string               `protobuf:"bytes,2,rep,name=filer_grpc_addresses,json=filerGrpcAddresses,proto3" json:"filer_grpc_addresses,omitempty"`
+	VolumeGrpcAddresses []string               `protobuf:"bytes,3,rep,name=volume_grpc_addresses,json=volumeGrpcAddresses,proto3" json:"volume_grpc_addresses,omitempty"`
+	Metadata            map[string]string      `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
+}
+
+func (x *ClusterContext) Reset() {
+	*x = ClusterContext{}
+	mi := &file_plugin_proto_msgTypes[34]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ClusterContext) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ClusterContext) ProtoMessage() {}
+
+func (x *ClusterContext) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[34]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ClusterContext.ProtoReflect.Descriptor instead.
+func (*ClusterContext) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{34}
+}
+
+func (x *ClusterContext) GetMasterGrpcAddresses() []string {
+	if x != nil {
+		return x.MasterGrpcAddresses
+	}
+	return nil
+}
+
+func (x *ClusterContext) GetFilerGrpcAddresses() []string {
+	if x != nil {
+		return x.FilerGrpcAddresses
+	}
+	return nil
+}
+
+func (x *ClusterContext) GetVolumeGrpcAddresses() []string {
+	if x != nil {
+		return x.VolumeGrpcAddresses
+	}
+	return nil
+}
+
+func (x *ClusterContext) GetMetadata() map[string]string {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+type ActivityEvent struct {
+	state         protoimpl.MessageState  `protogen:"open.v1"`
+	Source        ActivitySource          `protobuf:"varint,1,opt,name=source,proto3,enum=plugin.ActivitySource" json:"source,omitempty"`
+	Message       string                  `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	Stage         string                  `protobuf:"bytes,3,opt,name=stage,proto3" json:"stage,omitempty"`
+	Details       map[string]*ConfigValue `protobuf:"bytes,4,rep,name=details,proto3" json:"details,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	CreatedAt     *timestamppb.Timestamp  `protobuf:"bytes,5,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ActivityEvent) Reset() {
+	*x = ActivityEvent{}
+	mi := &file_plugin_proto_msgTypes[35]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ActivityEvent) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ActivityEvent) ProtoMessage() {}
+
+func (x *ActivityEvent) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[35]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ActivityEvent.ProtoReflect.Descriptor instead.
+func (*ActivityEvent) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{35}
+}
+
+func (x *ActivityEvent) GetSource() ActivitySource {
+	if x != nil {
+		return x.Source
+	}
+	return ActivitySource_ACTIVITY_SOURCE_UNSPECIFIED
+}
+
+func (x *ActivityEvent) GetMessage() string {
+	if x != nil {
+		return x.Message
+	}
+	return ""
+}
+
+func (x *ActivityEvent) GetStage() string {
+	if x != nil {
+		return x.Stage
+	}
+	return ""
+}
+
+func (x *ActivityEvent) GetDetails() map[string]*ConfigValue {
+	if x != nil {
+		return x.Details
+	}
+	return nil
+}
+
+func (x *ActivityEvent) GetCreatedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.CreatedAt
+	}
+	return nil
+}
+
+type CancelRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	TargetId      string                 `protobuf:"bytes,1,opt,name=target_id,json=targetId,proto3" json:"target_id,omitempty"`
+	TargetKind    WorkKind               `protobuf:"varint,2,opt,name=target_kind,json=targetKind,proto3,enum=plugin.WorkKind" json:"target_kind,omitempty"`
+	Reason        string                 `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"`
+	Force         bool                   `protobuf:"varint,4,opt,name=force,proto3" json:"force,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *CancelRequest) Reset() {
+	*x = CancelRequest{}
+	mi := &file_plugin_proto_msgTypes[36]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *CancelRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CancelRequest) ProtoMessage() {}
+
+func (x *CancelRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[36]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CancelRequest.ProtoReflect.Descriptor instead.
+func (*CancelRequest) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{36}
+}
+
+func (x *CancelRequest) GetTargetId() string {
+	if x != nil {
+		return x.TargetId
+	}
+	return ""
+}
+
+func (x *CancelRequest) GetTargetKind() WorkKind {
+	if x != nil {
+		return x.TargetKind
+	}
+	return WorkKind_WORK_KIND_UNSPECIFIED
+}
+
+func (x *CancelRequest) GetReason() string {
+	if x != nil {
+		return x.Reason
+	}
+	return ""
+}
+
+func (x *CancelRequest) GetForce() bool {
+	if x != nil {
+		return x.Force
+	}
+	return false
+}
+
+type AdminShutdown struct {
+	state              protoimpl.MessageState `protogen:"open.v1"`
+	Reason             string                 `protobuf:"bytes,1,opt,name=reason,proto3" json:"reason,omitempty"`
+	GracePeriodSeconds int32                  `protobuf:"varint,2,opt,name=grace_period_seconds,json=gracePeriodSeconds,proto3" json:"grace_period_seconds,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
+}
+
+func (x *AdminShutdown) Reset() {
+	*x = AdminShutdown{}
+	mi := &file_plugin_proto_msgTypes[37]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AdminShutdown) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AdminShutdown) ProtoMessage() {}
+
+func (x *AdminShutdown) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[37]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AdminShutdown.ProtoReflect.Descriptor instead.
+func (*AdminShutdown) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{37}
+}
+
+func (x *AdminShutdown) GetReason() string {
+	if x != nil {
+		return x.Reason
+	}
+	return ""
+}
+
+func (x *AdminShutdown) GetGracePeriodSeconds() int32 {
+	if x != nil {
+		return x.GracePeriodSeconds
+	}
+	return 0
+}
+
+// PersistedJobTypeConfig is the admin-side on-disk model per job type.
+type PersistedJobTypeConfig struct {
+	state              protoimpl.MessageState  `protogen:"open.v1"`
+	JobType            string                  `protobuf:"bytes,1,opt,name=job_type,json=jobType,proto3" json:"job_type,omitempty"`
+	DescriptorVersion  uint32                  `protobuf:"varint,2,opt,name=descriptor_version,json=descriptorVersion,proto3" json:"descriptor_version,omitempty"`
+	AdminConfigValues  map[string]*ConfigValue `protobuf:"bytes,3,rep,name=admin_config_values,json=adminConfigValues,proto3" json:"admin_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	WorkerConfigValues map[string]*ConfigValue `protobuf:"bytes,4,rep,name=worker_config_values,json=workerConfigValues,proto3" json:"worker_config_values,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	AdminRuntime       *AdminRuntimeConfig     `protobuf:"bytes,5,opt,name=admin_runtime,json=adminRuntime,proto3" json:"admin_runtime,omitempty"`
+	UpdatedAt          *timestamppb.Timestamp  `protobuf:"bytes,6,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"`
+	UpdatedBy          string                  `protobuf:"bytes,7,opt,name=updated_by,json=updatedBy,proto3" json:"updated_by,omitempty"`
+	unknownFields      protoimpl.UnknownFields
+	sizeCache          protoimpl.SizeCache
+}
+
+func (x *PersistedJobTypeConfig) Reset() {
+	*x = PersistedJobTypeConfig{}
+	mi := &file_plugin_proto_msgTypes[38]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *PersistedJobTypeConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*PersistedJobTypeConfig) ProtoMessage() {}
+
+func (x *PersistedJobTypeConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_plugin_proto_msgTypes[38]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use PersistedJobTypeConfig.ProtoReflect.Descriptor instead.
+func (*PersistedJobTypeConfig) Descriptor() ([]byte, []int) {
+	return file_plugin_proto_rawDescGZIP(), []int{38}
+}
+
+func (x *PersistedJobTypeConfig) GetJobType() string {
+	if x != nil {
+		return x.JobType
+	}
+	return ""
+}
+
+func (x *PersistedJobTypeConfig) GetDescriptorVersion() uint32 {
+	if x != nil {
+		return x.DescriptorVersion
+	}
+	return 0
+}
+
+func (x *PersistedJobTypeConfig) GetAdminConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.AdminConfigValues
+	}
+	return nil
+}
+
+func (x *PersistedJobTypeConfig) GetWorkerConfigValues() map[string]*ConfigValue {
+	if x != nil {
+		return x.WorkerConfigValues
+	}
+	return nil
+}
+
+func (x *PersistedJobTypeConfig) GetAdminRuntime() *AdminRuntimeConfig {
+	if x != nil {
+		return x.AdminRuntime
+	}
+	return nil
+}
+
+func (x *PersistedJobTypeConfig) GetUpdatedAt() *timestamppb.Timestamp {
+	if x != nil {
+		return x.UpdatedAt
+	}
+	return nil
+}
+
+func (x *PersistedJobTypeConfig) GetUpdatedBy() string {
+	if x != nil {
+		return x.UpdatedBy
+	}
+	return ""
+}
+
+var File_plugin_proto protoreflect.FileDescriptor
+
+const file_plugin_proto_rawDesc = "" +
+	"\n" +
+	"\fplugin.proto\x12\x06plugin\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x90\x05\n" +
+	"\x14WorkerToAdminMessage\x12\x1b\n" +
+	"\tworker_id\x18\x01 \x01(\tR\bworkerId\x123\n" +
+	"\asent_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x06sentAt\x12+\n" +
+	"\x05hello\x18\n" +
+	" \x01(\v2\x13.plugin.WorkerHelloH\x00R\x05hello\x127\n" +
+	"\theartbeat\x18\v \x01(\v2\x17.plugin.WorkerHeartbeatH\x00R\theartbeat\x12=\n" +
+	"\vacknowledge\x18\f \x01(\v2\x19.plugin.WorkerAcknowledgeH\x00R\vacknowledge\x12T\n" +
+	"\x16config_schema_response\x18\r \x01(\v2\x1c.plugin.ConfigSchemaResponseH\x00R\x14configSchemaResponse\x12M\n" +
+	"\x13detection_proposals\x18\x0e \x01(\v2\x1a.plugin.DetectionProposalsH\x00R\x12detectionProposals\x12J\n" +
+	"\x12detection_complete\x18\x0f \x01(\v2\x19.plugin.DetectionCompleteH\x00R\x11detectionComplete\x12K\n" +
+	"\x13job_progress_update\x18\x10 \x01(\v2\x19.plugin.JobProgressUpdateH\x00R\x11jobProgressUpdate\x12;\n" +
+	"\rjob_completed\x18\x11 \x01(\v2\x14.plugin.JobCompletedH\x00R\fjobCompletedB\x06\n" +
+	"\x04body\"\x86\x04\n" +
+	"\x14AdminToWorkerMessage\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x123\n" +
+	"\asent_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x06sentAt\x12*\n" +
+	"\x05hello\x18\n" +
+	" \x01(\v2\x12.plugin.AdminHelloH\x00R\x05hello\x12Q\n" +
+	"\x15request_config_schema\x18\v \x01(\v2\x1b.plugin.RequestConfigSchemaH\x00R\x13requestConfigSchema\x12Q\n" +
+	"\x15run_detection_request\x18\f \x01(\v2\x1b.plugin.RunDetectionRequestH\x00R\x13runDetectionRequest\x12K\n" +
+	"\x13execute_job_request\x18\r \x01(\v2\x19.plugin.ExecuteJobRequestH\x00R\x11executeJobRequest\x12>\n" +
+	"\x0ecancel_request\x18\x0e \x01(\v2\x15.plugin.CancelRequestH\x00R\rcancelRequest\x123\n" +
+	"\bshutdown\x18\x0f \x01(\v2\x15.plugin.AdminShutdownH\x00R\bshutdownB\x06\n" +
+	"\x04body\"\xff\x02\n" +
+	"\vWorkerHello\x12\x1b\n" +
+	"\tworker_id\x18\x01 \x01(\tR\bworkerId\x12,\n" +
+	"\x12worker_instance_id\x18\x02 \x01(\tR\x10workerInstanceId\x12\x18\n" +
+	"\aaddress\x18\x03 \x01(\tR\aaddress\x12%\n" +
+	"\x0eworker_version\x18\x04 \x01(\tR\rworkerVersion\x12)\n" +
+	"\x10protocol_version\x18\x05 \x01(\tR\x0fprotocolVersion\x12=\n" +
+	"\fcapabilities\x18\x06 \x03(\v2\x19.plugin.JobTypeCapabilityR\fcapabilities\x12=\n" +
+	"\bmetadata\x18\a \x03(\v2!.plugin.WorkerHello.MetadataEntryR\bmetadata\x1a;\n" +
+	"\rMetadataEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb8\x01\n" +
+	"\n" +
+	"AdminHello\x12\x1a\n" +
+	"\baccepted\x18\x01 \x01(\bR\baccepted\x12\x18\n" +
+	"\amessage\x18\x02 \x01(\tR\amessage\x12<\n" +
+	"\x1aheartbeat_interval_seconds\x18\x03 \x01(\x05R\x18heartbeatIntervalSeconds\x126\n" +
+	"\x17reconnect_delay_seconds\x18\x04 \x01(\x05R\x15reconnectDelaySeconds\"\xd5\x04\n" +
+	"\x0fWorkerHeartbeat\x12\x1b\n" +
+	"\tworker_id\x18\x01 \x01(\tR\bworkerId\x126\n" +
+	"\frunning_work\x18\x02 \x03(\v2\x13.plugin.RunningWorkR\vrunningWork\x120\n" +
+	"\x14detection_slots_used\x18\x03 \x01(\x05R\x12detectionSlotsUsed\x122\n" +
+	"\x15detection_slots_total\x18\x04 \x01(\x05R\x13detectionSlotsTotal\x120\n" +
+	"\x14execution_slots_used\x18\x05 \x01(\x05R\x12executionSlotsUsed\x122\n" +
+	"\x15execution_slots_total\x18\x06 \x01(\x05R\x13executionSlotsTotal\x12\\\n" +
+	"\x13queued_jobs_by_type\x18\a \x03(\v2-.plugin.WorkerHeartbeat.QueuedJobsByTypeEntryR\x10queuedJobsByType\x12A\n" +
+	"\bmetadata\x18\b \x03(\v2%.plugin.WorkerHeartbeat.MetadataEntryR\bmetadata\x1aC\n" +
+	"\x15QueuedJobsByTypeEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\x05R\x05value:\x028\x01\x1a;\n" +
+	"\rMetadataEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"h\n" +
+	"\x11WorkerAcknowledge\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x1a\n" +
+	"\baccepted\x18\x02 \x01(\bR\baccepted\x12\x18\n" +
+	"\amessage\x18\x03 \x01(\tR\amessage\"\xd0\x01\n" +
+	"\vRunningWork\x12\x17\n" +
+	"\awork_id\x18\x01 \x01(\tR\x06workId\x12$\n" +
+	"\x04kind\x18\x02 \x01(\x0e2\x10.plugin.WorkKindR\x04kind\x12\x19\n" +
+	"\bjob_type\x18\x03 \x01(\tR\ajobType\x12&\n" +
+	"\x05state\x18\x04 \x01(\x0e2\x10.plugin.JobStateR\x05state\x12)\n" +
+	"\x10progress_percent\x18\x05 \x01(\x01R\x0fprogressPercent\x12\x14\n" +
+	"\x05stage\x18\x06 \x01(\tR\x05stage\"\xab\x02\n" +
+	"\x11JobTypeCapability\x12\x19\n" +
+	"\bjob_type\x18\x01 \x01(\tR\ajobType\x12\x1d\n" +
+	"\n" +
+	"can_detect\x18\x02 \x01(\bR\tcanDetect\x12\x1f\n" +
+	"\vcan_execute\x18\x03 \x01(\bR\n" +
+	"canExecute\x12:\n" +
+	"\x19max_detection_concurrency\x18\x04 \x01(\x05R\x17maxDetectionConcurrency\x12:\n" +
+	"\x19max_execution_concurrency\x18\x05 \x01(\x05R\x17maxExecutionConcurrency\x12!\n" +
+	"\fdisplay_name\x18\x06 \x01(\tR\vdisplayName\x12 \n" +
+	"\vdescription\x18\a \x01(\tR\vdescription\"U\n" +
+	"\x13RequestConfigSchema\x12\x19\n" +
+	"\bjob_type\x18\x01 \x01(\tR\ajobType\x12#\n" +
+	"\rforce_refresh\x18\x02 \x01(\bR\fforceRefresh\"\xda\x01\n" +
+	"\x14ConfigSchemaResponse\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +
+	"\bjob_type\x18\x02 \x01(\tR\ajobType\x12\x18\n" +
+	"\asuccess\x18\x03 \x01(\bR\asuccess\x12#\n" +
+	"\rerror_message\x18\x04 \x01(\tR\ferrorMessage\x12I\n" +
+	"\x13job_type_descriptor\x18\x05 \x01(\v2\x19.plugin.JobTypeDescriptorR\x11jobTypeDescriptor\"\xd1\x04\n" +
+	"\x11JobTypeDescriptor\x12\x19\n" +
+	"\bjob_type\x18\x01 \x01(\tR\ajobType\x12!\n" +
+	"\fdisplay_name\x18\x02 \x01(\tR\vdisplayName\x12 \n" +
+	"\vdescription\x18\x03 \x01(\tR\vdescription\x12\x12\n" +
+	"\x04icon\x18\x04 \x01(\tR\x04icon\x12-\n" +
+	"\x12descriptor_version\x18\x05 \x01(\rR\x11descriptorVersion\x12>\n" +
+	"\x11admin_config_form\x18\x06 \x01(\v2\x12.plugin.ConfigFormR\x0fadminConfigForm\x12@\n" +
+	"\x12worker_config_form\x18\a \x01(\v2\x12.plugin.ConfigFormR\x10workerConfigForm\x12R\n" +
+	"\x16admin_runtime_defaults\x18\b \x01(\v2\x1c.plugin.AdminRuntimeDefaultsR\x14adminRuntimeDefaults\x12f\n" +
+	"\x15worker_default_values\x18\t \x03(\v22.plugin.JobTypeDescriptor.WorkerDefaultValuesEntryR\x13workerDefaultValues\x1a[\n" +
+	"\x18WorkerDefaultValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xb5\x02\n" +
+	"\n" +
+	"ConfigForm\x12\x17\n" +
+	"\aform_id\x18\x01 \x01(\tR\x06formId\x12\x14\n" +
+	"\x05title\x18\x02 \x01(\tR\x05title\x12 \n" +
+	"\vdescription\x18\x03 \x01(\tR\vdescription\x121\n" +
+	"\bsections\x18\x04 \x03(\v2\x15.plugin.ConfigSectionR\bsections\x12L\n" +
+	"\x0edefault_values\x18\x05 \x03(\v2%.plugin.ConfigForm.DefaultValuesEntryR\rdefaultValues\x1aU\n" +
+	"\x12DefaultValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\x93\x01\n" +
+	"\rConfigSection\x12\x1d\n" +
+	"\n" +
+	"section_id\x18\x01 \x01(\tR\tsectionId\x12\x14\n" +
+	"\x05title\x18\x02 \x01(\tR\x05title\x12 \n" +
+	"\vdescription\x18\x03 \x01(\tR\vdescription\x12+\n" +
+	"\x06fields\x18\x04 \x03(\v2\x13.plugin.ConfigFieldR\x06fields\"\x9f\x05\n" +
+	"\vConfigField\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" +
+	"\x05label\x18\x02 \x01(\tR\x05label\x12 \n" +
+	"\vdescription\x18\x03 \x01(\tR\vdescription\x12\x1b\n" +
+	"\thelp_text\x18\x04 \x01(\tR\bhelpText\x12 \n" +
+	"\vplaceholder\x18\x05 \x01(\tR\vplaceholder\x126\n" +
+	"\n" +
+	"field_type\x18\x06 \x01(\x0e2\x17.plugin.ConfigFieldTypeR\tfieldType\x12,\n" +
+	"\x06widget\x18\a \x01(\x0e2\x14.plugin.ConfigWidgetR\x06widget\x12\x1a\n" +
+	"\brequired\x18\b \x01(\bR\brequired\x12\x1b\n" +
+	"\tread_only\x18\t \x01(\bR\breadOnly\x12\x1c\n" +
+	"\tsensitive\x18\n" +
+	" \x01(\bR\tsensitive\x120\n" +
+	"\tmin_value\x18\v \x01(\v2\x13.plugin.ConfigValueR\bminValue\x120\n" +
+	"\tmax_value\x18\f \x01(\v2\x13.plugin.ConfigValueR\bmaxValue\x12.\n" +
+	"\aoptions\x18\r \x03(\v2\x14.plugin.ConfigOptionR\aoptions\x12A\n" +
+	"\x10validation_rules\x18\x0e \x03(\v2\x16.plugin.ValidationRuleR\x0fvalidationRules\x12,\n" +
+	"\x12visible_when_field\x18\x0f \x01(\tR\x10visibleWhenField\x12C\n" +
+	"\x13visible_when_equals\x18\x10 \x01(\v2\x13.plugin.ConfigValueR\x11visibleWhenEquals\"x\n" +
+	"\fConfigOption\x12\x14\n" +
+	"\x05value\x18\x01 \x01(\tR\x05value\x12\x14\n" +
+	"\x05label\x18\x02 \x01(\tR\x05label\x12 \n" +
+	"\vdescription\x18\x03 \x01(\tR\vdescription\x12\x1a\n" +
+	"\bdisabled\x18\x04 \x01(\bR\bdisabled\"\x85\x01\n" +
+	"\x0eValidationRule\x12.\n" +
+	"\x04type\x18\x01 \x01(\x0e2\x1a.plugin.ValidationRuleTypeR\x04type\x12\x1e\n" +
+	"\n" +
+	"expression\x18\x02 \x01(\tR\n" +
+	"expression\x12#\n" +
+	"\rerror_message\x18\x03 \x01(\tR\ferrorMessage\"\xc2\x04\n" +
+	"\vConfigValue\x12\x1f\n" +
+	"\n" +
+	"bool_value\x18\x01 \x01(\bH\x00R\tboolValue\x12!\n" +
+	"\vint64_value\x18\x02 \x01(\x03H\x00R\n" +
+	"int64Value\x12#\n" +
+	"\fdouble_value\x18\x03 \x01(\x01H\x00R\vdoubleValue\x12#\n" +
+	"\fstring_value\x18\x04 \x01(\tH\x00R\vstringValue\x12!\n" +
+	"\vbytes_value\x18\x05 \x01(\fH\x00R\n" +
+	"bytesValue\x12B\n" +
+	"\x0eduration_value\x18\x06 \x01(\v2\x19.google.protobuf.DurationH\x00R\rdurationValue\x125\n" +
+	"\vstring_list\x18\a \x01(\v2\x12.plugin.StringListH\x00R\n" +
+	"stringList\x122\n" +
+	"\n" +
+	"int64_list\x18\b \x01(\v2\x11.plugin.Int64ListH\x00R\tint64List\x125\n" +
+	"\vdouble_list\x18\t \x01(\v2\x12.plugin.DoubleListH\x00R\n" +
+	"doubleList\x12/\n" +
+	"\tbool_list\x18\n" +
+	" \x01(\v2\x10.plugin.BoolListH\x00R\bboolList\x122\n" +
+	"\n" +
+	"list_value\x18\v \x01(\v2\x11.plugin.ValueListH\x00R\tlistValue\x12/\n" +
+	"\tmap_value\x18\f \x01(\v2\x10.plugin.ValueMapH\x00R\bmapValueB\x06\n" +
+	"\x04kind\"$\n" +
+	"\n" +
+	"StringList\x12\x16\n" +
+	"\x06values\x18\x01 \x03(\tR\x06values\"#\n" +
+	"\tInt64List\x12\x16\n" +
+	"\x06values\x18\x01 \x03(\x03R\x06values\"$\n" +
+	"\n" +
+	"DoubleList\x12\x16\n" +
+	"\x06values\x18\x01 \x03(\x01R\x06values\"\"\n" +
+	"\bBoolList\x12\x16\n" +
+	"\x06values\x18\x01 \x03(\bR\x06values\"8\n" +
+	"\tValueList\x12+\n" +
+	"\x06values\x18\x01 \x03(\v2\x13.plugin.ConfigValueR\x06values\"\x90\x01\n" +
+	"\bValueMap\x124\n" +
+	"\x06fields\x18\x01 \x03(\v2\x1c.plugin.ValueMap.FieldsEntryR\x06fields\x1aN\n" +
+	"\vFieldsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xbf\x03\n" +
+	"\x14AdminRuntimeDefaults\x12\x18\n" +
+	"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
+	"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
+	"\x19detection_timeout_seconds\x18\x03 \x01(\x05R\x17detectionTimeoutSeconds\x123\n" +
+	"\x16max_jobs_per_detection\x18\x04 \x01(\x05R\x13maxJobsPerDetection\x12@\n" +
+	"\x1cglobal_execution_concurrency\x18\x05 \x01(\x05R\x1aglobalExecutionConcurrency\x12G\n" +
+	" per_worker_execution_concurrency\x18\x06 \x01(\x05R\x1dperWorkerExecutionConcurrency\x12\x1f\n" +
+	"\vretry_limit\x18\a \x01(\x05R\n" +
+	"retryLimit\x122\n" +
+	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\"\xbd\x03\n" +
+	"\x12AdminRuntimeConfig\x12\x18\n" +
+	"\aenabled\x18\x01 \x01(\bR\aenabled\x12<\n" +
+	"\x1adetection_interval_seconds\x18\x02 \x01(\x05R\x18detectionIntervalSeconds\x12:\n" +
+	"\x19detection_timeout_seconds\x18\x03 \x01(\x05R\x17detectionTimeoutSeconds\x123\n" +
+	"\x16max_jobs_per_detection\x18\x04 \x01(\x05R\x13maxJobsPerDetection\x12@\n" +
+	"\x1cglobal_execution_concurrency\x18\x05 \x01(\x05R\x1aglobalExecutionConcurrency\x12G\n" +
+	" per_worker_execution_concurrency\x18\x06 \x01(\x05R\x1dperWorkerExecutionConcurrency\x12\x1f\n" +
+	"\vretry_limit\x18\a \x01(\x05R\n" +
+	"retryLimit\x122\n" +
+	"\x15retry_backoff_seconds\x18\b \x01(\x05R\x13retryBackoffSeconds\"\xef\x05\n" +
+	"\x13RunDetectionRequest\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +
+	"\bjob_type\x18\x02 \x01(\tR\ajobType\x12-\n" +
+	"\x12detection_sequence\x18\x03 \x01(\x03R\x11detectionSequence\x12?\n" +
+	"\radmin_runtime\x18\x04 \x01(\v2\x1a.plugin.AdminRuntimeConfigR\fadminRuntime\x12b\n" +
+	"\x13admin_config_values\x18\x05 \x03(\v22.plugin.RunDetectionRequest.AdminConfigValuesEntryR\x11adminConfigValues\x12e\n" +
+	"\x14worker_config_values\x18\x06 \x03(\v23.plugin.RunDetectionRequest.WorkerConfigValuesEntryR\x12workerConfigValues\x12?\n" +
+	"\x0fcluster_context\x18\a \x01(\v2\x16.plugin.ClusterContextR\x0eclusterContext\x12J\n" +
+	"\x13last_successful_run\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\x11lastSuccessfulRun\x12\x1f\n" +
+	"\vmax_results\x18\t \x01(\x05R\n" +
+	"maxResults\x1aY\n" +
+	"\x16AdminConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\x1aZ\n" +
+	"\x17WorkerConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\x9c\x01\n" +
+	"\x12DetectionProposals\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +
+	"\bjob_type\x18\x02 \x01(\tR\ajobType\x121\n" +
+	"\tproposals\x18\x03 \x03(\v2\x13.plugin.JobProposalR\tproposals\x12\x19\n" +
+	"\bhas_more\x18\x04 \x01(\bR\ahasMore\"\xb5\x01\n" +
+	"\x11DetectionComplete\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x19\n" +
+	"\bjob_type\x18\x02 \x01(\tR\ajobType\x12\x18\n" +
+	"\asuccess\x18\x03 \x01(\bR\asuccess\x12#\n" +
+	"\rerror_message\x18\x04 \x01(\tR\ferrorMessage\x12'\n" +
+	"\x0ftotal_proposals\x18\x05 \x01(\x05R\x0etotalProposals\"\xce\x04\n" +
+	"\vJobProposal\x12\x1f\n" +
+	"\vproposal_id\x18\x01 \x01(\tR\n" +
+	"proposalId\x12\x1d\n" +
+	"\n" +
+	"dedupe_key\x18\x02 \x01(\tR\tdedupeKey\x12\x19\n" +
+	"\bjob_type\x18\x03 \x01(\tR\ajobType\x12/\n" +
+	"\bpriority\x18\x04 \x01(\x0e2\x13.plugin.JobPriorityR\bpriority\x12\x18\n" +
+	"\asummary\x18\x05 \x01(\tR\asummary\x12\x16\n" +
+	"\x06detail\x18\x06 \x01(\tR\x06detail\x12C\n" +
+	"\n" +
+	"parameters\x18\a \x03(\v2#.plugin.JobProposal.ParametersEntryR\n" +
+	"parameters\x127\n" +
+	"\x06labels\x18\b \x03(\v2\x1f.plugin.JobProposal.LabelsEntryR\x06labels\x129\n" +
+	"\n" +
+	"not_before\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\tnotBefore\x129\n" +
+	"\n" +
+	"expires_at\x18\n" +
+	" \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x1aR\n" +
+	"\x0fParametersEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\x1a9\n" +
+	"\vLabelsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xef\x04\n" +
+	"\x11ExecuteJobRequest\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12!\n" +
+	"\x03job\x18\x02 \x01(\v2\x0f.plugin.JobSpecR\x03job\x12?\n" +
+	"\radmin_runtime\x18\x03 \x01(\v2\x1a.plugin.AdminRuntimeConfigR\fadminRuntime\x12`\n" +
+	"\x13admin_config_values\x18\x04 \x03(\v20.plugin.ExecuteJobRequest.AdminConfigValuesEntryR\x11adminConfigValues\x12c\n" +
+	"\x14worker_config_values\x18\x05 \x03(\v21.plugin.ExecuteJobRequest.WorkerConfigValuesEntryR\x12workerConfigValues\x12?\n" +
+	"\x0fcluster_context\x18\x06 \x01(\v2\x16.plugin.ClusterContextR\x0eclusterContext\x12\x18\n" +
+	"\aattempt\x18\a \x01(\x05R\aattempt\x1aY\n" +
+	"\x16AdminConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\x1aZ\n" +
+	"\x17WorkerConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xbc\x04\n" +
+	"\aJobSpec\x12\x15\n" +
+	"\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x19\n" +
+	"\bjob_type\x18\x02 \x01(\tR\ajobType\x12\x1d\n" +
+	"\n" +
+	"dedupe_key\x18\x03 \x01(\tR\tdedupeKey\x12/\n" +
+	"\bpriority\x18\x04 \x01(\x0e2\x13.plugin.JobPriorityR\bpriority\x12\x18\n" +
+	"\asummary\x18\x05 \x01(\tR\asummary\x12\x16\n" +
+	"\x06detail\x18\x06 \x01(\tR\x06detail\x12?\n" +
+	"\n" +
+	"parameters\x18\a \x03(\v2\x1f.plugin.JobSpec.ParametersEntryR\n" +
+	"parameters\x123\n" +
+	"\x06labels\x18\b \x03(\v2\x1b.plugin.JobSpec.LabelsEntryR\x06labels\x129\n" +
+	"\n" +
+	"created_at\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12=\n" +
+	"\fscheduled_at\x18\n" +
+	" \x01(\v2\x1a.google.protobuf.TimestampR\vscheduledAt\x1aR\n" +
+	"\x0fParametersEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\x1a9\n" +
+	"\vLabelsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xec\x03\n" +
+	"\x11JobProgressUpdate\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x15\n" +
+	"\x06job_id\x18\x02 \x01(\tR\x05jobId\x12\x19\n" +
+	"\bjob_type\x18\x03 \x01(\tR\ajobType\x12&\n" +
+	"\x05state\x18\x04 \x01(\x0e2\x10.plugin.JobStateR\x05state\x12)\n" +
+	"\x10progress_percent\x18\x05 \x01(\x01R\x0fprogressPercent\x12\x14\n" +
+	"\x05stage\x18\x06 \x01(\tR\x05stage\x12\x18\n" +
+	"\amessage\x18\a \x01(\tR\amessage\x12@\n" +
+	"\ametrics\x18\b \x03(\v2&.plugin.JobProgressUpdate.MetricsEntryR\ametrics\x125\n" +
+	"\n" +
+	"activities\x18\t \x03(\v2\x15.plugin.ActivityEventR\n" +
+	"activities\x129\n" +
+	"\n" +
+	"updated_at\x18\n" +
+	" \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\x1aO\n" +
+	"\fMetricsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xbf\x02\n" +
+	"\fJobCompleted\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\x01 \x01(\tR\trequestId\x12\x15\n" +
+	"\x06job_id\x18\x02 \x01(\tR\x05jobId\x12\x19\n" +
+	"\bjob_type\x18\x03 \x01(\tR\ajobType\x12\x18\n" +
+	"\asuccess\x18\x04 \x01(\bR\asuccess\x12#\n" +
+	"\rerror_message\x18\x05 \x01(\tR\ferrorMessage\x12)\n" +
+	"\x06result\x18\x06 \x01(\v2\x11.plugin.JobResultR\x06result\x125\n" +
+	"\n" +
+	"activities\x18\a \x03(\v2\x15.plugin.ActivityEventR\n" +
+	"activities\x12=\n" +
+	"\fcompleted_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\vcompletedAt\"\xc5\x01\n" +
+	"\tJobResult\x12H\n" +
+	"\routput_values\x18\x01 \x03(\v2#.plugin.JobResult.OutputValuesEntryR\foutputValues\x12\x18\n" +
+	"\asummary\x18\x02 \x01(\tR\asummary\x1aT\n" +
+	"\x11OutputValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\xa9\x02\n" +
+	"\x0eClusterContext\x122\n" +
+	"\x15master_grpc_addresses\x18\x01 \x03(\tR\x13masterGrpcAddresses\x120\n" +
+	"\x14filer_grpc_addresses\x18\x02 \x03(\tR\x12filerGrpcAddresses\x122\n" +
+	"\x15volume_grpc_addresses\x18\x03 \x03(\tR\x13volumeGrpcAddresses\x12@\n" +
+	"\bmetadata\x18\x04 \x03(\v2$.plugin.ClusterContext.MetadataEntryR\bmetadata\x1a;\n" +
+	"\rMetadataEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb9\x02\n" +
+	"\rActivityEvent\x12.\n" +
+	"\x06source\x18\x01 \x01(\x0e2\x16.plugin.ActivitySourceR\x06source\x12\x18\n" +
+	"\amessage\x18\x02 \x01(\tR\amessage\x12\x14\n" +
+	"\x05stage\x18\x03 \x01(\tR\x05stage\x12<\n" +
+	"\adetails\x18\x04 \x03(\v2\".plugin.ActivityEvent.DetailsEntryR\adetails\x129\n" +
+	"\n" +
+	"created_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x1aO\n" +
+	"\fDetailsEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\"\x8d\x01\n" +
+	"\rCancelRequest\x12\x1b\n" +
+	"\ttarget_id\x18\x01 \x01(\tR\btargetId\x121\n" +
+	"\vtarget_kind\x18\x02 \x01(\x0e2\x10.plugin.WorkKindR\n" +
+	"targetKind\x12\x16\n" +
+	"\x06reason\x18\x03 \x01(\tR\x06reason\x12\x14\n" +
+	"\x05force\x18\x04 \x01(\bR\x05force\"Y\n" +
+	"\rAdminShutdown\x12\x16\n" +
+	"\x06reason\x18\x01 \x01(\tR\x06reason\x120\n" +
+	"\x14grace_period_seconds\x18\x02 \x01(\x05R\x12gracePeriodSeconds\"\x85\x05\n" +
+	"\x16PersistedJobTypeConfig\x12\x19\n" +
+	"\bjob_type\x18\x01 \x01(\tR\ajobType\x12-\n" +
+	"\x12descriptor_version\x18\x02 \x01(\rR\x11descriptorVersion\x12e\n" +
+	"\x13admin_config_values\x18\x03 \x03(\v25.plugin.PersistedJobTypeConfig.AdminConfigValuesEntryR\x11adminConfigValues\x12h\n" +
+	"\x14worker_config_values\x18\x04 \x03(\v26.plugin.PersistedJobTypeConfig.WorkerConfigValuesEntryR\x12workerConfigValues\x12?\n" +
+	"\radmin_runtime\x18\x05 \x01(\v2\x1a.plugin.AdminRuntimeConfigR\fadminRuntime\x129\n" +
+	"\n" +
+	"updated_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\x12\x1d\n" +
+	"\n" +
+	"updated_by\x18\a \x01(\tR\tupdatedBy\x1aY\n" +
+	"\x16AdminConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01\x1aZ\n" +
+	"\x17WorkerConfigValuesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12)\n" +
+	"\x05value\x18\x02 \x01(\v2\x13.plugin.ConfigValueR\x05value:\x028\x01*W\n" +
+	"\bWorkKind\x12\x19\n" +
+	"\x15WORK_KIND_UNSPECIFIED\x10\x00\x12\x17\n" +
+	"\x13WORK_KIND_DETECTION\x10\x01\x12\x17\n" +
+	"\x13WORK_KIND_EXECUTION\x10\x02*\x8c\x01\n" +
+	"\vJobPriority\x12\x1c\n" +
+	"\x18JOB_PRIORITY_UNSPECIFIED\x10\x00\x12\x14\n" +
+	"\x10JOB_PRIORITY_LOW\x10\x01\x12\x17\n" +
+	"\x13JOB_PRIORITY_NORMAL\x10\x02\x12\x15\n" +
+	"\x11JOB_PRIORITY_HIGH\x10\x03\x12\x19\n" +
+	"\x15JOB_PRIORITY_CRITICAL\x10\x04*\xb2\x01\n" +
+	"\bJobState\x12\x19\n" +
+	"\x15JOB_STATE_UNSPECIFIED\x10\x00\x12\x15\n" +
+	"\x11JOB_STATE_PENDING\x10\x01\x12\x16\n" +
+	"\x12JOB_STATE_ASSIGNED\x10\x02\x12\x15\n" +
+	"\x11JOB_STATE_RUNNING\x10\x03\x12\x17\n" +
+	"\x13JOB_STATE_SUCCEEDED\x10\x04\x12\x14\n" +
+	"\x10JOB_STATE_FAILED\x10\x05\x12\x16\n" +
+	"\x12JOB_STATE_CANCELED\x10\x06*\xbc\x02\n" +
+	"\x0fConfigFieldType\x12!\n" +
+	"\x1dCONFIG_FIELD_TYPE_UNSPECIFIED\x10\x00\x12\x1a\n" +
+	"\x16CONFIG_FIELD_TYPE_BOOL\x10\x01\x12\x1b\n" +
+	"\x17CONFIG_FIELD_TYPE_INT64\x10\x02\x12\x1c\n" +
+	"\x18CONFIG_FIELD_TYPE_DOUBLE\x10\x03\x12\x1c\n" +
+	"\x18CONFIG_FIELD_TYPE_STRING\x10\x04\x12\x1b\n" +
+	"\x17CONFIG_FIELD_TYPE_BYTES\x10\x05\x12\x1e\n" +
+	"\x1aCONFIG_FIELD_TYPE_DURATION\x10\x06\x12\x1a\n" +
+	"\x16CONFIG_FIELD_TYPE_ENUM\x10\a\x12\x1a\n" +
+	"\x16CONFIG_FIELD_TYPE_LIST\x10\b\x12\x1c\n" +
+	"\x18CONFIG_FIELD_TYPE_OBJECT\x10\t*\x87\x02\n" +
+	"\fConfigWidget\x12\x1d\n" +
+	"\x19CONFIG_WIDGET_UNSPECIFIED\x10\x00\x12\x18\n" +
+	"\x14CONFIG_WIDGET_TOGGLE\x10\x01\x12\x16\n" +
+	"\x12CONFIG_WIDGET_TEXT\x10\x02\x12\x1a\n" +
+	"\x16CONFIG_WIDGET_TEXTAREA\x10\x03\x12\x18\n" +
+	"\x14CONFIG_WIDGET_NUMBER\x10\x04\x12\x18\n" +
+	"\x14CONFIG_WIDGET_SELECT\x10\x05\x12\x1e\n" +
+	"\x1aCONFIG_WIDGET_MULTI_SELECT\x10\x06\x12\x1a\n" +
+	"\x16CONFIG_WIDGET_DURATION\x10\a\x12\x1a\n" +
+	"\x16CONFIG_WIDGET_PASSWORD\x10\b*\x8d\x02\n" +
+	"\x12ValidationRuleType\x12$\n" +
+	" VALIDATION_RULE_TYPE_UNSPECIFIED\x10\x00\x12\x1e\n" +
+	"\x1aVALIDATION_RULE_TYPE_REGEX\x10\x01\x12#\n" +
+	"\x1fVALIDATION_RULE_TYPE_MIN_LENGTH\x10\x02\x12#\n" +
+	"\x1fVALIDATION_RULE_TYPE_MAX_LENGTH\x10\x03\x12\"\n" +
+	"\x1eVALIDATION_RULE_TYPE_MIN_ITEMS\x10\x04\x12\"\n" +
+	"\x1eVALIDATION_RULE_TYPE_MAX_ITEMS\x10\x05\x12\x1f\n" +
+	"\x1bVALIDATION_RULE_TYPE_CUSTOM\x10\x06*\x88\x01\n" +
+	"\x0eActivitySource\x12\x1f\n" +
+	"\x1bACTIVITY_SOURCE_UNSPECIFIED\x10\x00\x12\x19\n" +
+	"\x15ACTIVITY_SOURCE_ADMIN\x10\x01\x12\x1c\n" +
+	"\x18ACTIVITY_SOURCE_DETECTOR\x10\x02\x12\x1c\n" +
+	"\x18ACTIVITY_SOURCE_EXECUTOR\x10\x032f\n" +
+	"\x14PluginControlService\x12N\n" +
+	"\fWorkerStream\x12\x1c.plugin.WorkerToAdminMessage\x1a\x1c.plugin.AdminToWorkerMessage(\x010\x01B2Z0github.com/seaweedfs/seaweedfs/weed/pb/plugin_pbb\x06proto3"
+
+var (
+	file_plugin_proto_rawDescOnce sync.Once
+	file_plugin_proto_rawDescData []byte
+)
+
+func file_plugin_proto_rawDescGZIP() []byte {
+	file_plugin_proto_rawDescOnce.Do(func() {
+		file_plugin_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_plugin_proto_rawDesc), len(file_plugin_proto_rawDesc)))
+	})
+	return file_plugin_proto_rawDescData
+}
+
+var file_plugin_proto_enumTypes = make([]protoimpl.EnumInfo, 7)
+var file_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 59)
+var file_plugin_proto_goTypes = []any{
+	(WorkKind)(0),                  // 0: plugin.WorkKind
+	(JobPriority)(0),               // 1: plugin.JobPriority
+	(JobState)(0),                  // 2: plugin.JobState
+	(ConfigFieldType)(0),           // 3: plugin.ConfigFieldType
+	(ConfigWidget)(0),              // 4: plugin.ConfigWidget
+	(ValidationRuleType)(0),        // 5: plugin.ValidationRuleType
+	(ActivitySource)(0),            // 6: plugin.ActivitySource
+	(*WorkerToAdminMessage)(nil),   // 7: plugin.WorkerToAdminMessage
+	(*AdminToWorkerMessage)(nil),   // 8: plugin.AdminToWorkerMessage
+	(*WorkerHello)(nil),            // 9: plugin.WorkerHello
+	(*AdminHello)(nil),             // 10: plugin.AdminHello
+	(*WorkerHeartbeat)(nil),        // 11: plugin.WorkerHeartbeat
+	(*WorkerAcknowledge)(nil),      // 12: plugin.WorkerAcknowledge
+	(*RunningWork)(nil),            // 13: plugin.RunningWork
+	(*JobTypeCapability)(nil),      // 14: plugin.JobTypeCapability
+	(*RequestConfigSchema)(nil),    // 15: plugin.RequestConfigSchema
+	(*ConfigSchemaResponse)(nil),   // 16: plugin.ConfigSchemaResponse
+	(*JobTypeDescriptor)(nil),      // 17: plugin.JobTypeDescriptor
+	(*ConfigForm)(nil),             // 18: plugin.ConfigForm
+	(*ConfigSection)(nil),          // 19: plugin.ConfigSection
+	(*ConfigField)(nil),            // 20: plugin.ConfigField
+	(*ConfigOption)(nil),           // 21: plugin.ConfigOption
+	(*ValidationRule)(nil),         // 22: plugin.ValidationRule
+	(*ConfigValue)(nil),            // 23: plugin.ConfigValue
+	(*StringList)(nil),             // 24: plugin.StringList
+	(*Int64List)(nil),              // 25: plugin.Int64List
+	(*DoubleList)(nil),             // 26: plugin.DoubleList
+	(*BoolList)(nil),               // 27: plugin.BoolList
+	(*ValueList)(nil),              // 28: plugin.ValueList
+	(*ValueMap)(nil),               // 29: plugin.ValueMap
+	(*AdminRuntimeDefaults)(nil),   // 30: plugin.AdminRuntimeDefaults
+	(*AdminRuntimeConfig)(nil),     // 31: plugin.AdminRuntimeConfig
+	(*RunDetectionRequest)(nil),    // 32: plugin.RunDetectionRequest
+	(*DetectionProposals)(nil),     // 33: plugin.DetectionProposals
+	(*DetectionComplete)(nil),      // 34: plugin.DetectionComplete
+	(*JobProposal)(nil),            // 35: plugin.JobProposal
+	(*ExecuteJobRequest)(nil),      // 36: plugin.ExecuteJobRequest
+	(*JobSpec)(nil),                // 37: plugin.JobSpec
+	(*JobProgressUpdate)(nil),      // 38: plugin.JobProgressUpdate
+	(*JobCompleted)(nil),           // 39: plugin.JobCompleted
+	(*JobResult)(nil),              // 40: plugin.JobResult
+	(*ClusterContext)(nil),         // 41: plugin.ClusterContext
+	(*ActivityEvent)(nil),          // 42: plugin.ActivityEvent
+	(*CancelRequest)(nil),          // 43: plugin.CancelRequest
+	(*AdminShutdown)(nil),          // 44: plugin.AdminShutdown
+	(*PersistedJobTypeConfig)(nil), // 45: plugin.PersistedJobTypeConfig
+	nil,                            // 46: plugin.WorkerHello.MetadataEntry
+	nil,                            // 47: plugin.WorkerHeartbeat.QueuedJobsByTypeEntry
+	nil,                            // 48: plugin.WorkerHeartbeat.MetadataEntry
+	nil,                            // 49: plugin.JobTypeDescriptor.WorkerDefaultValuesEntry
+	nil,                            // 50: plugin.ConfigForm.DefaultValuesEntry
+	nil,                            // 51: plugin.ValueMap.FieldsEntry
+	nil,                            // 52: plugin.RunDetectionRequest.AdminConfigValuesEntry
+	nil,                            // 53: plugin.RunDetectionRequest.WorkerConfigValuesEntry
+	nil,                            // 54: plugin.JobProposal.ParametersEntry
+	nil,                            // 55: plugin.JobProposal.LabelsEntry
+	nil,                            // 56: plugin.ExecuteJobRequest.AdminConfigValuesEntry
+	nil,                            // 57: plugin.ExecuteJobRequest.WorkerConfigValuesEntry
+	nil,                            // 58: plugin.JobSpec.ParametersEntry
+	nil,                            // 59: plugin.JobSpec.LabelsEntry
+	nil,                            // 60: plugin.JobProgressUpdate.MetricsEntry
+	nil,                            // 61: plugin.JobResult.OutputValuesEntry
+	nil,                            // 62: plugin.ClusterContext.MetadataEntry
+	nil,                            // 63: plugin.ActivityEvent.DetailsEntry
+	nil,                            // 64: plugin.PersistedJobTypeConfig.AdminConfigValuesEntry
+	nil,                            // 65: plugin.PersistedJobTypeConfig.WorkerConfigValuesEntry
+	(*timestamppb.Timestamp)(nil),  // 66: google.protobuf.Timestamp
+	(*durationpb.Duration)(nil),    // 67: google.protobuf.Duration
+}
+var file_plugin_proto_depIdxs = []int32{
+	66,  // 0: plugin.WorkerToAdminMessage.sent_at:type_name -> google.protobuf.Timestamp
+	9,   // 1: plugin.WorkerToAdminMessage.hello:type_name -> plugin.WorkerHello
+	11,  // 2: plugin.WorkerToAdminMessage.heartbeat:type_name -> plugin.WorkerHeartbeat
+	12,  // 3: plugin.WorkerToAdminMessage.acknowledge:type_name -> plugin.WorkerAcknowledge
+	16,  // 4: plugin.WorkerToAdminMessage.config_schema_response:type_name -> plugin.ConfigSchemaResponse
+	33,  // 5: plugin.WorkerToAdminMessage.detection_proposals:type_name -> plugin.DetectionProposals
+	34,  // 6: plugin.WorkerToAdminMessage.detection_complete:type_name -> plugin.DetectionComplete
+	38,  // 7: plugin.WorkerToAdminMessage.job_progress_update:type_name -> plugin.JobProgressUpdate
+	39,  // 8: plugin.WorkerToAdminMessage.job_completed:type_name -> plugin.JobCompleted
+	66,  // 9: plugin.AdminToWorkerMessage.sent_at:type_name -> google.protobuf.Timestamp
+	10,  // 10: plugin.AdminToWorkerMessage.hello:type_name -> plugin.AdminHello
+	15,  // 11: plugin.AdminToWorkerMessage.request_config_schema:type_name -> plugin.RequestConfigSchema
+	32,  // 12: plugin.AdminToWorkerMessage.run_detection_request:type_name -> plugin.RunDetectionRequest
+	36,  // 13: plugin.AdminToWorkerMessage.execute_job_request:type_name -> plugin.ExecuteJobRequest
+	43,  // 14: plugin.AdminToWorkerMessage.cancel_request:type_name -> plugin.CancelRequest
+	44,  // 15: plugin.AdminToWorkerMessage.shutdown:type_name -> plugin.AdminShutdown
+	14,  // 16: plugin.WorkerHello.capabilities:type_name -> plugin.JobTypeCapability
+	46,  // 17: plugin.WorkerHello.metadata:type_name -> plugin.WorkerHello.MetadataEntry
+	13,  // 18: plugin.WorkerHeartbeat.running_work:type_name -> plugin.RunningWork
+	47,  // 19: plugin.WorkerHeartbeat.queued_jobs_by_type:type_name -> plugin.WorkerHeartbeat.QueuedJobsByTypeEntry
+	48,  // 20: plugin.WorkerHeartbeat.metadata:type_name -> plugin.WorkerHeartbeat.MetadataEntry
+	0,   // 21: plugin.RunningWork.kind:type_name -> plugin.WorkKind
+	2,   // 22: plugin.RunningWork.state:type_name -> plugin.JobState
+	17,  // 23: plugin.ConfigSchemaResponse.job_type_descriptor:type_name -> plugin.JobTypeDescriptor
+	18,  // 24: plugin.JobTypeDescriptor.admin_config_form:type_name -> plugin.ConfigForm
+	18,  // 25: plugin.JobTypeDescriptor.worker_config_form:type_name -> plugin.ConfigForm
+	30,  // 26: plugin.JobTypeDescriptor.admin_runtime_defaults:type_name -> plugin.AdminRuntimeDefaults
+	49,  // 27: plugin.JobTypeDescriptor.worker_default_values:type_name -> plugin.JobTypeDescriptor.WorkerDefaultValuesEntry
+	19,  // 28: plugin.ConfigForm.sections:type_name -> plugin.ConfigSection
+	50,  // 29: plugin.ConfigForm.default_values:type_name -> plugin.ConfigForm.DefaultValuesEntry
+	20,  // 30: plugin.ConfigSection.fields:type_name -> plugin.ConfigField
+	3,   // 31: plugin.ConfigField.field_type:type_name -> plugin.ConfigFieldType
+	4,   // 32: plugin.ConfigField.widget:type_name -> plugin.ConfigWidget
+	23,  // 33: plugin.ConfigField.min_value:type_name -> plugin.ConfigValue
+	23,  // 34: plugin.ConfigField.max_value:type_name -> plugin.ConfigValue
+	21,  // 35: plugin.ConfigField.options:type_name -> plugin.ConfigOption
+	22,  // 36: plugin.ConfigField.validation_rules:type_name -> plugin.ValidationRule
+	23,  // 37: plugin.ConfigField.visible_when_equals:type_name -> plugin.ConfigValue
+	5,   // 38: plugin.ValidationRule.type:type_name -> plugin.ValidationRuleType
+	67,  // 39: plugin.ConfigValue.duration_value:type_name -> google.protobuf.Duration
+	24,  // 40: plugin.ConfigValue.string_list:type_name -> plugin.StringList
+	25,  // 41: plugin.ConfigValue.int64_list:type_name -> plugin.Int64List
+	26,  // 42: plugin.ConfigValue.double_list:type_name -> plugin.DoubleList
+	27,  // 43: plugin.ConfigValue.bool_list:type_name -> plugin.BoolList
+	28,  // 44: plugin.ConfigValue.list_value:type_name -> plugin.ValueList
+	29,  // 45: plugin.ConfigValue.map_value:type_name -> plugin.ValueMap
+	23,  // 46: plugin.ValueList.values:type_name -> plugin.ConfigValue
+	51,  // 47: plugin.ValueMap.fields:type_name -> plugin.ValueMap.FieldsEntry
+	31,  // 48: plugin.RunDetectionRequest.admin_runtime:type_name -> plugin.AdminRuntimeConfig
+	52,  // 49: plugin.RunDetectionRequest.admin_config_values:type_name -> plugin.RunDetectionRequest.AdminConfigValuesEntry
+	53,  // 50: plugin.RunDetectionRequest.worker_config_values:type_name -> plugin.RunDetectionRequest.WorkerConfigValuesEntry
+	41,  // 51: plugin.RunDetectionRequest.cluster_context:type_name -> plugin.ClusterContext
+	66,  // 52: plugin.RunDetectionRequest.last_successful_run:type_name -> google.protobuf.Timestamp
+	35,  // 53: plugin.DetectionProposals.proposals:type_name -> plugin.JobProposal
+	1,   // 54: plugin.JobProposal.priority:type_name -> plugin.JobPriority
+	54,  // 55: plugin.JobProposal.parameters:type_name -> plugin.JobProposal.ParametersEntry
+	55,  // 56: plugin.JobProposal.labels:type_name -> plugin.JobProposal.LabelsEntry
+	66,  // 57: plugin.JobProposal.not_before:type_name -> google.protobuf.Timestamp
+	66,  // 58: plugin.JobProposal.expires_at:type_name -> google.protobuf.Timestamp
+	37,  // 59: plugin.ExecuteJobRequest.job:type_name -> plugin.JobSpec
+	31,  // 60: plugin.ExecuteJobRequest.admin_runtime:type_name -> plugin.AdminRuntimeConfig
+	56,  // 61: plugin.ExecuteJobRequest.admin_config_values:type_name -> plugin.ExecuteJobRequest.AdminConfigValuesEntry
+	57,  // 62: plugin.ExecuteJobRequest.worker_config_values:type_name -> plugin.ExecuteJobRequest.WorkerConfigValuesEntry
+	41,  // 63: plugin.ExecuteJobRequest.cluster_context:type_name -> plugin.ClusterContext
+	1,   // 64: plugin.JobSpec.priority:type_name -> plugin.JobPriority
+	58,  // 65: plugin.JobSpec.parameters:type_name -> plugin.JobSpec.ParametersEntry
+	59,  // 66: plugin.JobSpec.labels:type_name -> plugin.JobSpec.LabelsEntry
+	66,  // 67: plugin.JobSpec.created_at:type_name -> google.protobuf.Timestamp
+	66,  // 68: plugin.JobSpec.scheduled_at:type_name -> google.protobuf.Timestamp
+	2,   // 69: plugin.JobProgressUpdate.state:type_name -> plugin.JobState
+	60,  // 70: plugin.JobProgressUpdate.metrics:type_name -> plugin.JobProgressUpdate.MetricsEntry
+	42,  // 71: plugin.JobProgressUpdate.activities:type_name -> plugin.ActivityEvent
+	66,  // 72: plugin.JobProgressUpdate.updated_at:type_name -> google.protobuf.Timestamp
+	40,  // 73: plugin.JobCompleted.result:type_name -> plugin.JobResult
+	42,  // 74: plugin.JobCompleted.activities:type_name -> plugin.ActivityEvent
+	66,  // 75: plugin.JobCompleted.completed_at:type_name -> google.protobuf.Timestamp
+	61,  // 76: plugin.JobResult.output_values:type_name -> plugin.JobResult.OutputValuesEntry
+	62,  // 77: plugin.ClusterContext.metadata:type_name -> plugin.ClusterContext.MetadataEntry
+	6,   // 78: plugin.ActivityEvent.source:type_name -> plugin.ActivitySource
+	63,  // 79: plugin.ActivityEvent.details:type_name -> plugin.ActivityEvent.DetailsEntry
+	66,  // 80: plugin.ActivityEvent.created_at:type_name -> google.protobuf.Timestamp
+	0,   // 81: plugin.CancelRequest.target_kind:type_name -> plugin.WorkKind
+	64,  // 82: plugin.PersistedJobTypeConfig.admin_config_values:type_name -> plugin.PersistedJobTypeConfig.AdminConfigValuesEntry
+	65,  // 83: plugin.PersistedJobTypeConfig.worker_config_values:type_name -> plugin.PersistedJobTypeConfig.WorkerConfigValuesEntry
+	31,  // 84: plugin.PersistedJobTypeConfig.admin_runtime:type_name -> plugin.AdminRuntimeConfig
+	66,  // 85: plugin.PersistedJobTypeConfig.updated_at:type_name -> google.protobuf.Timestamp
+	23,  // 86: plugin.JobTypeDescriptor.WorkerDefaultValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 87: plugin.ConfigForm.DefaultValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 88: plugin.ValueMap.FieldsEntry.value:type_name -> plugin.ConfigValue
+	23,  // 89: plugin.RunDetectionRequest.AdminConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 90: plugin.RunDetectionRequest.WorkerConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 91: plugin.JobProposal.ParametersEntry.value:type_name -> plugin.ConfigValue
+	23,  // 92: plugin.ExecuteJobRequest.AdminConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 93: plugin.ExecuteJobRequest.WorkerConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 94: plugin.JobSpec.ParametersEntry.value:type_name -> plugin.ConfigValue
+	23,  // 95: plugin.JobProgressUpdate.MetricsEntry.value:type_name -> plugin.ConfigValue
+	23,  // 96: plugin.JobResult.OutputValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 97: plugin.ActivityEvent.DetailsEntry.value:type_name -> plugin.ConfigValue
+	23,  // 98: plugin.PersistedJobTypeConfig.AdminConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	23,  // 99: plugin.PersistedJobTypeConfig.WorkerConfigValuesEntry.value:type_name -> plugin.ConfigValue
+	7,   // 100: plugin.PluginControlService.WorkerStream:input_type -> plugin.WorkerToAdminMessage
+	8,   // 101: plugin.PluginControlService.WorkerStream:output_type -> plugin.AdminToWorkerMessage
+	101, // [101:102] is the sub-list for method output_type
+	100, // [100:101] is the sub-list for method input_type
+	100, // [100:100] is the sub-list for extension type_name
+	100, // [100:100] is the sub-list for extension extendee
+	0,   // [0:100] is the sub-list for field type_name
+}
+
+func init() { file_plugin_proto_init() }
+func file_plugin_proto_init() {
+	if File_plugin_proto != nil {
+		return
+	}
+	file_plugin_proto_msgTypes[0].OneofWrappers = []any{
+		(*WorkerToAdminMessage_Hello)(nil),
+		(*WorkerToAdminMessage_Heartbeat)(nil),
+		(*WorkerToAdminMessage_Acknowledge)(nil),
+		(*WorkerToAdminMessage_ConfigSchemaResponse)(nil),
+		(*WorkerToAdminMessage_DetectionProposals)(nil),
+		(*WorkerToAdminMessage_DetectionComplete)(nil),
+		(*WorkerToAdminMessage_JobProgressUpdate)(nil),
+		(*WorkerToAdminMessage_JobCompleted)(nil),
+	}
+	file_plugin_proto_msgTypes[1].OneofWrappers = []any{
+		(*AdminToWorkerMessage_Hello)(nil),
+		(*AdminToWorkerMessage_RequestConfigSchema)(nil),
+		(*AdminToWorkerMessage_RunDetectionRequest)(nil),
+		(*AdminToWorkerMessage_ExecuteJobRequest)(nil),
+		(*AdminToWorkerMessage_CancelRequest)(nil),
+		(*AdminToWorkerMessage_Shutdown)(nil),
+	}
+	file_plugin_proto_msgTypes[16].OneofWrappers = []any{
+		(*ConfigValue_BoolValue)(nil),
+		(*ConfigValue_Int64Value)(nil),
+		(*ConfigValue_DoubleValue)(nil),
+		(*ConfigValue_StringValue)(nil),
+		(*ConfigValue_BytesValue)(nil),
+		(*ConfigValue_DurationValue)(nil),
+		(*ConfigValue_StringList)(nil),
+		(*ConfigValue_Int64List)(nil),
+		(*ConfigValue_DoubleList)(nil),
+		(*ConfigValue_BoolList)(nil),
+		(*ConfigValue_ListValue)(nil),
+		(*ConfigValue_MapValue)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_plugin_proto_rawDesc), len(file_plugin_proto_rawDesc)),
+			NumEnums:      7,
+			NumMessages:   59,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_plugin_proto_goTypes,
+		DependencyIndexes: file_plugin_proto_depIdxs,
+		EnumInfos:         file_plugin_proto_enumTypes,
+		MessageInfos:      file_plugin_proto_msgTypes,
+	}.Build()
+	File_plugin_proto = out.File
+	file_plugin_proto_goTypes = nil
+	file_plugin_proto_depIdxs = nil
+}
diff --git a/weed/pb/plugin_pb/plugin_grpc.pb.go b/weed/pb/plugin_pb/plugin_grpc.pb.go
new file mode 100644
index 000000000..72281016a
--- /dev/null
+++ b/weed/pb/plugin_pb/plugin_grpc.pb.go
@@ -0,0 +1,121 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v6.33.4
+// source: plugin.proto
+
+package plugin_pb
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	PluginControlService_WorkerStream_FullMethodName = "/plugin.PluginControlService/WorkerStream"
+)
+
+// PluginControlServiceClient is the client API for PluginControlService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+//
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+type PluginControlServiceClient interface {
+	WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage], error)
+}
+
+type pluginControlServiceClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewPluginControlServiceClient(cc grpc.ClientConnInterface) PluginControlServiceClient {
+	return &pluginControlServiceClient{cc}
+}
+
+func (c *pluginControlServiceClient) WorkerStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &PluginControlService_ServiceDesc.Streams[0], PluginControlService_WorkerStream_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[WorkerToAdminMessage, AdminToWorkerMessage]{ClientStream: stream}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type PluginControlService_WorkerStreamClient = grpc.BidiStreamingClient[WorkerToAdminMessage, AdminToWorkerMessage]
+
+// PluginControlServiceServer is the server API for PluginControlService service.
+// All implementations must embed UnimplementedPluginControlServiceServer
+// for forward compatibility.
+//
+// PluginControlService is the admin-facing stream API for external workers.
+// Workers initiate and keep this stream alive; all control plane traffic flows through it.
+type PluginControlServiceServer interface {
+	WorkerStream(grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]) error
+	mustEmbedUnimplementedPluginControlServiceServer()
+}
+
+// UnimplementedPluginControlServiceServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedPluginControlServiceServer struct{}
+
+func (UnimplementedPluginControlServiceServer) WorkerStream(grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]) error {
+	return status.Errorf(codes.Unimplemented, "method WorkerStream not implemented")
+}
+func (UnimplementedPluginControlServiceServer) mustEmbedUnimplementedPluginControlServiceServer() {}
+func (UnimplementedPluginControlServiceServer) testEmbeddedByValue()                              {}
+
+// UnsafePluginControlServiceServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to PluginControlServiceServer will
+// result in compilation errors.
+type UnsafePluginControlServiceServer interface {
+	mustEmbedUnimplementedPluginControlServiceServer()
+}
+
+func RegisterPluginControlServiceServer(s grpc.ServiceRegistrar, srv PluginControlServiceServer) {
+	// If the following call pancis, it indicates UnimplementedPluginControlServiceServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&PluginControlService_ServiceDesc, srv)
+}
+
+func _PluginControlService_WorkerStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(PluginControlServiceServer).WorkerStream(&grpc.GenericServerStream[WorkerToAdminMessage, AdminToWorkerMessage]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type PluginControlService_WorkerStreamServer = grpc.BidiStreamingServer[WorkerToAdminMessage, AdminToWorkerMessage]
+
+// PluginControlService_ServiceDesc is the grpc.ServiceDesc for PluginControlService service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var PluginControlService_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "plugin.PluginControlService",
+	HandlerType: (*PluginControlServiceServer)(nil),
+	Methods:     []grpc.MethodDesc{},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "WorkerStream",
+			Handler:       _PluginControlService_WorkerStream_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "plugin.proto",
+}
diff --git a/weed/plugin/worker/erasure_coding_handler.go b/weed/plugin/worker/erasure_coding_handler.go
new file mode 100644
index 000000000..0c4e8e148
--- /dev/null
+++ b/weed/plugin/worker/erasure_coding_handler.go
@@ -0,0 +1,899 @@
+package pluginworker
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	erasurecodingtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+)
+
+type erasureCodingWorkerConfig struct {
+	TaskConfig         *erasurecodingtask.Config
+	MinIntervalSeconds int
+}
+
+// ErasureCodingHandler is the plugin job handler for erasure coding.
+type ErasureCodingHandler struct {
+	grpcDialOption grpc.DialOption
+}
+
+func NewErasureCodingHandler(grpcDialOption grpc.DialOption) *ErasureCodingHandler {
+	return &ErasureCodingHandler{grpcDialOption: grpcDialOption}
+}
+
+func (h *ErasureCodingHandler) Capability() *plugin_pb.JobTypeCapability {
+	return &plugin_pb.JobTypeCapability{
+		JobType:                 "erasure_coding",
+		CanDetect:               true,
+		CanExecute:              true,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 1,
+		DisplayName:             "Erasure Coding",
+		Description:             "Converts full and quiet volumes into EC shards",
+	}
+}
+
+func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return &plugin_pb.JobTypeDescriptor{
+		JobType:           "erasure_coding",
+		DisplayName:       "Erasure Coding",
+		Description:       "Detect and execute erasure coding for suitable volumes",
+		Icon:              "fas fa-shield-alt",
+		DescriptorVersion: 1,
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "erasure-coding-admin",
+			Title:       "Erasure Coding Admin Config",
+			Description: "Admin-side controls for erasure coding detection scope.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "scope",
+					Title:       "Scope",
+					Description: "Optional filters applied before erasure coding detection.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "collection_filter",
+							Label:       "Collection Filter",
+							Description: "Only detect erasure coding opportunities in this collection when set.",
+							Placeholder: "all collections",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"collection_filter": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
+				},
+			},
+		},
+		WorkerConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "erasure-coding-worker",
+			Title:       "Erasure Coding Worker Config",
+			Description: "Worker-side detection thresholds.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "thresholds",
+					Title:       "Detection Thresholds",
+					Description: "Controls for when erasure coding jobs should be proposed.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "quiet_for_seconds",
+							Label:       "Quiet Period (s)",
+							Description: "Volume must remain unmodified for at least this duration.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+						{
+							Name:        "fullness_ratio",
+							Label:       "Fullness Ratio",
+							Description: "Minimum volume fullness ratio to trigger erasure coding.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
+							MaxValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
+						},
+						{
+							Name:        "min_size_mb",
+							Label:       "Minimum Volume Size (MB)",
+							Description: "Only volumes larger than this size are considered.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
+						},
+						{
+							Name:        "min_interval_seconds",
+							Label:       "Minimum Detection Interval (s)",
+							Description: "Skip detection if the last successful run is more recent than this interval.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"quiet_for_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 300},
+				},
+				"fullness_ratio": {
+					Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.8},
+				},
+				"min_size_mb": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
+				},
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			Enabled:                       true,
+			DetectionIntervalSeconds:      60 * 5,
+			DetectionTimeoutSeconds:       300,
+			MaxJobsPerDetection:           500,
+			GlobalExecutionConcurrency:    16,
+			PerWorkerExecutionConcurrency: 4,
+			RetryLimit:                    1,
+			RetryBackoffSeconds:           30,
+		},
+		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
+			"quiet_for_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 300},
+			},
+			"fullness_ratio": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.8},
+			},
+			"min_size_mb": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
+			},
+			"min_interval_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
+			},
+		},
+	}
+}
+
+func (h *ErasureCodingHandler) Detect(
+	ctx context.Context,
+	request *plugin_pb.RunDetectionRequest,
+	sender DetectionSender,
+) error {
+	if request == nil {
+		return fmt.Errorf("run detection request is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("detection sender is nil")
+	}
+	if request.JobType != "" && request.JobType != "erasure_coding" {
+		return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.JobType)
+	}
+
+	workerConfig := deriveErasureCodingWorkerConfig(request.GetWorkerConfigValues())
+	if shouldSkipDetectionByInterval(request.GetLastSuccessfulRun(), workerConfig.MinIntervalSeconds) {
+		minInterval := time.Duration(workerConfig.MinIntervalSeconds) * time.Second
+		_ = sender.SendActivity(buildDetectorActivity(
+			"skipped_by_interval",
+			fmt.Sprintf("ERASURE CODING: Detection skipped due to min interval (%s)", minInterval),
+			map[string]*plugin_pb.ConfigValue{
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(workerConfig.MinIntervalSeconds)},
+				},
+			},
+		))
+		if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+			JobType:   "erasure_coding",
+			Proposals: []*plugin_pb.JobProposal{},
+			HasMore:   false,
+		}); err != nil {
+			return err
+		}
+		return sender.SendComplete(&plugin_pb.DetectionComplete{
+			JobType:        "erasure_coding",
+			Success:        true,
+			TotalProposals: 0,
+		})
+	}
+
+	collectionFilter := strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
+	if collectionFilter != "" {
+		workerConfig.TaskConfig.CollectionFilter = collectionFilter
+	}
+
+	masters := make([]string, 0)
+	if request.ClusterContext != nil {
+		masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
+	}
+
+	metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
+	if err != nil {
+		return err
+	}
+
+	clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
+	results, err := erasurecodingtask.Detection(metrics, clusterInfo, workerConfig.TaskConfig)
+	if err != nil {
+		return err
+	}
+	if traceErr := emitErasureCodingDetectionDecisionTrace(sender, metrics, workerConfig.TaskConfig, results); traceErr != nil {
+		glog.Warningf("Plugin worker failed to emit erasure_coding detection trace: %v", traceErr)
+	}
+
+	maxResults := int(request.MaxResults)
+	hasMore := false
+	if maxResults > 0 && len(results) > maxResults {
+		hasMore = true
+		results = results[:maxResults]
+	}
+
+	proposals := make([]*plugin_pb.JobProposal, 0, len(results))
+	for _, result := range results {
+		proposal, proposalErr := buildErasureCodingProposal(result)
+		if proposalErr != nil {
+			glog.Warningf("Plugin worker skip invalid erasure_coding proposal: %v", proposalErr)
+			continue
+		}
+		proposals = append(proposals, proposal)
+	}
+
+	if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+		JobType:   "erasure_coding",
+		Proposals: proposals,
+		HasMore:   hasMore,
+	}); err != nil {
+		return err
+	}
+
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		JobType:        "erasure_coding",
+		Success:        true,
+		TotalProposals: int32(len(proposals)),
+	})
+}
+
+func emitErasureCodingDetectionDecisionTrace(
+	sender DetectionSender,
+	metrics []*workertypes.VolumeHealthMetrics,
+	taskConfig *erasurecodingtask.Config,
+	results []*workertypes.TaskDetectionResult,
+) error {
+	if sender == nil || taskConfig == nil {
+		return nil
+	}
+
+	quietThreshold := time.Duration(taskConfig.QuietForSeconds) * time.Second
+	minSizeBytes := uint64(taskConfig.MinSizeMB) * 1024 * 1024
+	allowedCollections := make(map[string]bool)
+	if strings.TrimSpace(taskConfig.CollectionFilter) != "" {
+		for _, collection := range strings.Split(taskConfig.CollectionFilter, ",") {
+			trimmed := strings.TrimSpace(collection)
+			if trimmed != "" {
+				allowedCollections[trimmed] = true
+			}
+		}
+	}
+
+	volumeGroups := make(map[uint32][]*workertypes.VolumeHealthMetrics)
+	for _, metric := range metrics {
+		if metric == nil {
+			continue
+		}
+		volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
+	}
+
+	skippedAlreadyEC := 0
+	skippedTooSmall := 0
+	skippedCollectionFilter := 0
+	skippedQuietTime := 0
+	skippedFullness := 0
+
+	for _, groupMetrics := range volumeGroups {
+		if len(groupMetrics) == 0 {
+			continue
+		}
+		metric := groupMetrics[0]
+		for _, candidate := range groupMetrics {
+			if candidate != nil && candidate.Server < metric.Server {
+				metric = candidate
+			}
+		}
+		if metric == nil {
+			continue
+		}
+
+		if metric.IsECVolume {
+			skippedAlreadyEC++
+			continue
+		}
+		if metric.Size < minSizeBytes {
+			skippedTooSmall++
+			continue
+		}
+		if len(allowedCollections) > 0 && !allowedCollections[metric.Collection] {
+			skippedCollectionFilter++
+			continue
+		}
+		if metric.Age < quietThreshold {
+			skippedQuietTime++
+		}
+		if metric.FullnessRatio < taskConfig.FullnessRatio {
+			skippedFullness++
+		}
+	}
+
+	totalVolumes := len(metrics)
+	summaryMessage := ""
+	if len(results) == 0 {
+		summaryMessage = fmt.Sprintf(
+			"EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
+			totalVolumes,
+			skippedAlreadyEC,
+			skippedTooSmall,
+			skippedCollectionFilter,
+			skippedQuietTime,
+			skippedFullness,
+		)
+	} else {
+		summaryMessage = fmt.Sprintf(
+			"EC detection: Created %d task(s) from %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
+			len(results),
+			totalVolumes,
+			skippedAlreadyEC,
+			skippedTooSmall,
+			skippedCollectionFilter,
+			skippedQuietTime,
+			skippedFullness,
+		)
+	}
+
+	if err := sender.SendActivity(buildDetectorActivity("decision_summary", summaryMessage, map[string]*plugin_pb.ConfigValue{
+		"total_volumes": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
+		},
+		"selected_tasks": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
+		},
+		"skipped_already_ec": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedAlreadyEC)},
+		},
+		"skipped_too_small": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedTooSmall)},
+		},
+		"skipped_filtered": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedCollectionFilter)},
+		},
+		"skipped_not_quiet": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedQuietTime)},
+		},
+		"skipped_not_full": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedFullness)},
+		},
+		"quiet_for_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
+		},
+		"min_size_mb": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
+		},
+		"fullness_threshold_percent": {
+			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
+		},
+	})); err != nil {
+		return err
+	}
+
+	detailsEmitted := 0
+	for _, metric := range metrics {
+		if metric == nil || metric.IsECVolume {
+			continue
+		}
+		sizeMB := float64(metric.Size) / (1024 * 1024)
+		message := fmt.Sprintf(
+			"ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
+			metric.VolumeID,
+			sizeMB,
+			taskConfig.MinSizeMB,
+			metric.Age.Truncate(time.Minute),
+			quietThreshold.Truncate(time.Minute),
+			metric.FullnessRatio*100,
+			taskConfig.FullnessRatio*100,
+		)
+		if err := sender.SendActivity(buildDetectorActivity("decision_volume", message, map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.VolumeID)},
+			},
+			"size_mb": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: sizeMB},
+			},
+			"required_min_size_mb": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
+			},
+			"age_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.Age.Seconds())},
+			},
+			"required_quiet_for_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
+			},
+			"fullness_percent": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: metric.FullnessRatio * 100},
+			},
+			"required_fullness_percent": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
+			},
+		})); err != nil {
+			return err
+		}
+		detailsEmitted++
+		if detailsEmitted >= 3 {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (h *ErasureCodingHandler) Execute(
+	ctx context.Context,
+	request *plugin_pb.ExecuteJobRequest,
+	sender ExecutionSender,
+) error {
+	if request == nil || request.Job == nil {
+		return fmt.Errorf("execute request/job is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("execution sender is nil")
+	}
+	if request.Job.JobType != "" && request.Job.JobType != "erasure_coding" {
+		return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.Job.JobType)
+	}
+
+	params, err := decodeErasureCodingTaskParams(request.Job)
+	if err != nil {
+		return err
+	}
+
+	applyErasureCodingExecutionDefaults(params, request.GetClusterContext())
+
+	if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
+		return fmt.Errorf("erasure coding source node is required")
+	}
+	if len(params.Targets) == 0 {
+		return fmt.Errorf("erasure coding targets are required")
+	}
+
+	task := erasurecodingtask.NewErasureCodingTask(
+		request.Job.JobId,
+		params.Sources[0].Node,
+		params.VolumeId,
+		params.Collection,
+	)
+	task.SetProgressCallback(func(progress float64, stage string) {
+		message := fmt.Sprintf("erasure coding progress %.0f%%", progress)
+		if strings.TrimSpace(stage) != "" {
+			message = stage
+		}
+		_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+			JobId:           request.Job.JobId,
+			JobType:         request.Job.JobType,
+			State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+			ProgressPercent: progress,
+			Stage:           stage,
+			Message:         message,
+			Activities: []*plugin_pb.ActivityEvent{
+				buildExecutorActivity(stage, message),
+			},
+		})
+	})
+
+	if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
+		JobId:           request.Job.JobId,
+		JobType:         request.Job.JobType,
+		State:           plugin_pb.JobState_JOB_STATE_ASSIGNED,
+		ProgressPercent: 0,
+		Stage:           "assigned",
+		Message:         "erasure coding job accepted",
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("assigned", "erasure coding job accepted"),
+		},
+	}); err != nil {
+		return err
+	}
+
+	if err := task.Execute(ctx, params); err != nil {
+		_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+			JobId:           request.Job.JobId,
+			JobType:         request.Job.JobType,
+			State:           plugin_pb.JobState_JOB_STATE_FAILED,
+			ProgressPercent: 100,
+			Stage:           "failed",
+			Message:         err.Error(),
+			Activities: []*plugin_pb.ActivityEvent{
+				buildExecutorActivity("failed", err.Error()),
+			},
+		})
+		return err
+	}
+
+	sourceNode := params.Sources[0].Node
+	resultSummary := fmt.Sprintf("erasure coding completed for volume %d across %d targets", params.VolumeId, len(params.Targets))
+
+	return sender.SendCompleted(&plugin_pb.JobCompleted{
+		JobId:   request.Job.JobId,
+		JobType: request.Job.JobType,
+		Success: true,
+		Result: &plugin_pb.JobResult{
+			Summary: resultSummary,
+			OutputValues: map[string]*plugin_pb.ConfigValue{
+				"volume_id": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
+				},
+				"source_server": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
+				},
+				"target_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
+				},
+			},
+		},
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("completed", resultSummary),
+		},
+	})
+}
+
+func (h *ErasureCodingHandler) collectVolumeMetrics(
+	ctx context.Context,
+	masterAddresses []string,
+	collectionFilter string,
+) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
+	// Reuse the same master topology fetch/build flow used by the vacuum handler.
+	helper := &VacuumHandler{grpcDialOption: h.grpcDialOption}
+	return helper.collectVolumeMetrics(ctx, masterAddresses, collectionFilter)
+}
+
+func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *erasureCodingWorkerConfig {
+	taskConfig := erasurecodingtask.NewDefaultConfig()
+
+	quietForSeconds := int(readInt64Config(values, "quiet_for_seconds", int64(taskConfig.QuietForSeconds)))
+	if quietForSeconds < 0 {
+		quietForSeconds = 0
+	}
+	taskConfig.QuietForSeconds = quietForSeconds
+
+	fullnessRatio := readDoubleConfig(values, "fullness_ratio", taskConfig.FullnessRatio)
+	if fullnessRatio < 0 {
+		fullnessRatio = 0
+	}
+	if fullnessRatio > 1 {
+		fullnessRatio = 1
+	}
+	taskConfig.FullnessRatio = fullnessRatio
+
+	minSizeMB := int(readInt64Config(values, "min_size_mb", int64(taskConfig.MinSizeMB)))
+	if minSizeMB < 1 {
+		minSizeMB = 1
+	}
+	taskConfig.MinSizeMB = minSizeMB
+
+	minIntervalSeconds := int(readInt64Config(values, "min_interval_seconds", 60*60))
+	if minIntervalSeconds < 0 {
+		minIntervalSeconds = 0
+	}
+
+	return &erasureCodingWorkerConfig{
+		TaskConfig:         taskConfig,
+		MinIntervalSeconds: minIntervalSeconds,
+	}
+}
+
+func buildErasureCodingProposal(
+	result *workertypes.TaskDetectionResult,
+) (*plugin_pb.JobProposal, error) {
+	if result == nil {
+		return nil, fmt.Errorf("task detection result is nil")
+	}
+	if result.TypedParams == nil {
+		return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
+	}
+	params := proto.Clone(result.TypedParams).(*worker_pb.TaskParams)
+	applyErasureCodingExecutionDefaults(params, nil)
+
+	paramsPayload, err := proto.Marshal(params)
+	if err != nil {
+		return nil, fmt.Errorf("marshal task params: %w", err)
+	}
+
+	proposalID := strings.TrimSpace(result.TaskID)
+	if proposalID == "" {
+		proposalID = fmt.Sprintf("erasure-coding-%d-%d", result.VolumeID, time.Now().UnixNano())
+	}
+
+	dedupeKey := fmt.Sprintf("erasure_coding:%d", result.VolumeID)
+	if result.Collection != "" {
+		dedupeKey += ":" + result.Collection
+	}
+
+	sourceNode := ""
+	if len(params.Sources) > 0 {
+		sourceNode = strings.TrimSpace(params.Sources[0].Node)
+	}
+
+	summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
+	if sourceNode != "" {
+		summary = fmt.Sprintf("Erasure code volume %d from %s", result.VolumeID, sourceNode)
+	}
+
+	return &plugin_pb.JobProposal{
+		ProposalId: proposalID,
+		DedupeKey:  dedupeKey,
+		JobType:    "erasure_coding",
+		Priority:   mapTaskPriority(result.Priority),
+		Summary:    summary,
+		Detail:     strings.TrimSpace(result.Reason),
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {
+				Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
+			},
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
+			},
+			"source_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
+			},
+			"target_count": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
+			},
+		},
+		Labels: map[string]string{
+			"task_type":    "erasure_coding",
+			"volume_id":    fmt.Sprintf("%d", result.VolumeID),
+			"collection":   result.Collection,
+			"source_node":  sourceNode,
+			"target_count": fmt.Sprintf("%d", len(params.Targets)),
+		},
+	}, nil
+}
+
+func decodeErasureCodingTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
+	if job == nil {
+		return nil, fmt.Errorf("job spec is nil")
+	}
+
+	if payload := readBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
+		params := &worker_pb.TaskParams{}
+		if err := proto.Unmarshal(payload, params); err != nil {
+			return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
+		}
+		if params.TaskId == "" {
+			params.TaskId = job.JobId
+		}
+		return params, nil
+	}
+
+	volumeID := readInt64Config(job.Parameters, "volume_id", 0)
+	sourceNode := strings.TrimSpace(readStringConfig(job.Parameters, "source_server", ""))
+	if sourceNode == "" {
+		sourceNode = strings.TrimSpace(readStringConfig(job.Parameters, "server", ""))
+	}
+	targetServers := readStringListConfig(job.Parameters, "target_servers")
+	if len(targetServers) == 0 {
+		targetServers = readStringListConfig(job.Parameters, "targets")
+	}
+	collection := readStringConfig(job.Parameters, "collection", "")
+
+	dataShards := int32(readInt64Config(job.Parameters, "data_shards", int64(ecstorage.DataShardsCount)))
+	if dataShards <= 0 {
+		dataShards = ecstorage.DataShardsCount
+	}
+	parityShards := int32(readInt64Config(job.Parameters, "parity_shards", int64(ecstorage.ParityShardsCount)))
+	if parityShards <= 0 {
+		parityShards = ecstorage.ParityShardsCount
+	}
+	totalShards := int(dataShards + parityShards)
+
+	if volumeID <= 0 {
+		return nil, fmt.Errorf("missing volume_id in job parameters")
+	}
+	if sourceNode == "" {
+		return nil, fmt.Errorf("missing source_server in job parameters")
+	}
+	if len(targetServers) == 0 {
+		return nil, fmt.Errorf("missing target_servers in job parameters")
+	}
+	if len(targetServers) < totalShards {
+		return nil, fmt.Errorf("insufficient target_servers: got %d, need at least %d", len(targetServers), totalShards)
+	}
+
+	shardAssignments := assignECShardIDs(totalShards, len(targetServers))
+	targets := make([]*worker_pb.TaskTarget, 0, len(targetServers))
+	for i := 0; i < len(targetServers); i++ {
+		targetNode := strings.TrimSpace(targetServers[i])
+		if targetNode == "" {
+			continue
+		}
+		targets = append(targets, &worker_pb.TaskTarget{
+			Node:     targetNode,
+			VolumeId: uint32(volumeID),
+			ShardIds: shardAssignments[i],
+		})
+	}
+	if len(targets) < totalShards {
+		return nil, fmt.Errorf("insufficient non-empty target_servers after normalization: got %d, need at least %d", len(targets), totalShards)
+	}
+
+	return &worker_pb.TaskParams{
+		TaskId:     job.JobId,
+		VolumeId:   uint32(volumeID),
+		Collection: collection,
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     sourceNode,
+				VolumeId: uint32(volumeID),
+			},
+		},
+		Targets: targets,
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:   dataShards,
+				ParityShards: parityShards,
+			},
+		},
+	}, nil
+}
+
+func applyErasureCodingExecutionDefaults(
+	params *worker_pb.TaskParams,
+	clusterContext *plugin_pb.ClusterContext,
+) {
+	if params == nil {
+		return
+	}
+
+	ecParams := params.GetErasureCodingParams()
+	if ecParams == nil {
+		ecParams = &worker_pb.ErasureCodingTaskParams{
+			DataShards:   ecstorage.DataShardsCount,
+			ParityShards: ecstorage.ParityShardsCount,
+		}
+		params.TaskParams = &worker_pb.TaskParams_ErasureCodingParams{ErasureCodingParams: ecParams}
+	}
+
+	if ecParams.DataShards <= 0 {
+		ecParams.DataShards = ecstorage.DataShardsCount
+	}
+	if ecParams.ParityShards <= 0 {
+		ecParams.ParityShards = ecstorage.ParityShardsCount
+	}
+	ecParams.WorkingDir = defaultErasureCodingWorkingDir()
+	ecParams.CleanupSource = true
+	if strings.TrimSpace(ecParams.MasterClient) == "" && clusterContext != nil && len(clusterContext.MasterGrpcAddresses) > 0 {
+		ecParams.MasterClient = clusterContext.MasterGrpcAddresses[0]
+	}
+
+	totalShards := int(ecParams.DataShards + ecParams.ParityShards)
+	if totalShards <= 0 {
+		totalShards = ecstorage.TotalShardsCount
+	}
+	needsShardAssignment := false
+	for _, target := range params.Targets {
+		if target == nil || len(target.ShardIds) == 0 {
+			needsShardAssignment = true
+			break
+		}
+	}
+	if needsShardAssignment && len(params.Targets) > 0 {
+		assignments := assignECShardIDs(totalShards, len(params.Targets))
+		for i := 0; i < len(params.Targets); i++ {
+			if params.Targets[i] == nil {
+				continue
+			}
+			if len(params.Targets[i].ShardIds) == 0 {
+				params.Targets[i].ShardIds = assignments[i]
+			}
+		}
+	}
+}
+
+func readStringListConfig(values map[string]*plugin_pb.ConfigValue, field string) []string {
+	if values == nil {
+		return nil
+	}
+	value := values[field]
+	if value == nil {
+		return nil
+	}
+
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_StringList:
+		return normalizeStringList(kind.StringList.GetValues())
+	case *plugin_pb.ConfigValue_ListValue:
+		out := make([]string, 0, len(kind.ListValue.GetValues()))
+		for _, item := range kind.ListValue.GetValues() {
+			itemText := readStringFromConfigValue(item)
+			if itemText != "" {
+				out = append(out, itemText)
+			}
+		}
+		return normalizeStringList(out)
+	case *plugin_pb.ConfigValue_StringValue:
+		return normalizeStringList(strings.Split(kind.StringValue, ","))
+	}
+
+	return nil
+}
+
+func readStringFromConfigValue(value *plugin_pb.ConfigValue) string {
+	if value == nil {
+		return ""
+	}
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_StringValue:
+		return strings.TrimSpace(kind.StringValue)
+	case *plugin_pb.ConfigValue_Int64Value:
+		return fmt.Sprintf("%d", kind.Int64Value)
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return fmt.Sprintf("%g", kind.DoubleValue)
+	case *plugin_pb.ConfigValue_BoolValue:
+		if kind.BoolValue {
+			return "true"
+		}
+		return "false"
+	}
+	return ""
+}
+
+func normalizeStringList(values []string) []string {
+	normalized := make([]string, 0, len(values))
+	seen := make(map[string]struct{}, len(values))
+	for _, value := range values {
+		item := strings.TrimSpace(value)
+		if item == "" {
+			continue
+		}
+		if _, found := seen[item]; found {
+			continue
+		}
+		seen[item] = struct{}{}
+		normalized = append(normalized, item)
+	}
+	return normalized
+}
+
+func assignECShardIDs(totalShards int, targetCount int) [][]uint32 {
+	if targetCount <= 0 {
+		return nil
+	}
+	if totalShards <= 0 {
+		totalShards = ecstorage.TotalShardsCount
+	}
+
+	assignments := make([][]uint32, targetCount)
+	for i := 0; i < totalShards; i++ {
+		targetIndex := i % targetCount
+		assignments[targetIndex] = append(assignments[targetIndex], uint32(i))
+	}
+	return assignments
+}
+
+func defaultErasureCodingWorkingDir() string {
+	return filepath.Join(os.TempDir(), "seaweedfs-ec")
+}
diff --git a/weed/plugin/worker/erasure_coding_handler_test.go b/weed/plugin/worker/erasure_coding_handler_test.go
new file mode 100644
index 000000000..28342c135
--- /dev/null
+++ b/weed/plugin/worker/erasure_coding_handler_test.go
@@ -0,0 +1,329 @@
+package pluginworker
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	erasurecodingtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestDecodeErasureCodingTaskParamsFromPayload(t *testing.T) {
+	expected := &worker_pb.TaskParams{
+		TaskId:     "task-ec-1",
+		VolumeId:   88,
+		Collection: "images",
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     "10.0.0.1:8080",
+				VolumeId: 88,
+			},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{
+				Node:     "10.0.0.2:8080",
+				VolumeId: 88,
+				ShardIds: []uint32{0, 10},
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:    ecstorage.DataShardsCount,
+				ParityShards:  ecstorage.ParityShardsCount,
+				WorkingDir:    "/tmp/ec-work",
+				CleanupSource: true,
+			},
+		},
+	}
+	payload, err := proto.Marshal(expected)
+	if err != nil {
+		t.Fatalf("marshal payload: %v", err)
+	}
+
+	job := &plugin_pb.JobSpec{
+		JobId: "job-from-admin",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
+		},
+	}
+
+	actual, err := decodeErasureCodingTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
+	}
+	if !proto.Equal(expected, actual) {
+		t.Fatalf("decoded params mismatch\nexpected: %+v\nactual:   %+v", expected, actual)
+	}
+}
+
+func TestDecodeErasureCodingTaskParamsFallback(t *testing.T) {
+	targetServers := make([]string, 0, ecstorage.TotalShardsCount)
+	for i := 0; i < ecstorage.TotalShardsCount; i++ {
+		targetServers = append(targetServers, "10.0.0."+string(rune('a'+i))+":8080")
+	}
+
+	job := &plugin_pb.JobSpec{
+		JobId: "job-ec-2",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7},
+			},
+			"source_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"},
+			},
+			"target_servers": {
+				Kind: &plugin_pb.ConfigValue_StringList{
+					StringList: &plugin_pb.StringList{Values: targetServers},
+				},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"},
+			},
+		},
+	}
+
+	params, err := decodeErasureCodingTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
+	}
+	if params.TaskId != "job-ec-2" || params.VolumeId != 7 || params.Collection != "videos" {
+		t.Fatalf("unexpected basic params: %+v", params)
+	}
+	if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
+		t.Fatalf("unexpected sources: %+v", params.Sources)
+	}
+	if len(params.Targets) != ecstorage.TotalShardsCount {
+		t.Fatalf("unexpected target count: %d", len(params.Targets))
+	}
+	if params.GetErasureCodingParams() == nil {
+		t.Fatalf("expected fallback erasure coding params")
+	}
+}
+
+func TestDeriveErasureCodingWorkerConfig(t *testing.T) {
+	values := map[string]*plugin_pb.ConfigValue{
+		"quiet_for_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 720},
+		},
+		"fullness_ratio": {
+			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.92},
+		},
+		"min_size_mb": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 128},
+		},
+		"min_interval_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 55},
+		},
+	}
+
+	cfg := deriveErasureCodingWorkerConfig(values)
+	if cfg.TaskConfig.QuietForSeconds != 720 {
+		t.Fatalf("expected quiet_for_seconds 720, got %d", cfg.TaskConfig.QuietForSeconds)
+	}
+	if cfg.TaskConfig.FullnessRatio != 0.92 {
+		t.Fatalf("expected fullness_ratio 0.92, got %v", cfg.TaskConfig.FullnessRatio)
+	}
+	if cfg.TaskConfig.MinSizeMB != 128 {
+		t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB)
+	}
+	if cfg.MinIntervalSeconds != 55 {
+		t.Fatalf("expected min_interval_seconds 55, got %d", cfg.MinIntervalSeconds)
+	}
+}
+
+func TestBuildErasureCodingProposal(t *testing.T) {
+	params := &worker_pb.TaskParams{
+		TaskId:     "ec-task-1",
+		VolumeId:   99,
+		Collection: "c1",
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     "source-a:8080",
+				VolumeId: 99,
+			},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{
+				Node:     "target-a:8080",
+				VolumeId: 99,
+				ShardIds: []uint32{0, 10},
+			},
+			{
+				Node:     "target-b:8080",
+				VolumeId: 99,
+				ShardIds: []uint32{1, 11},
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:   ecstorage.DataShardsCount,
+				ParityShards: ecstorage.ParityShardsCount,
+			},
+		},
+	}
+	result := &workertypes.TaskDetectionResult{
+		TaskID:      "ec-task-1",
+		TaskType:    workertypes.TaskTypeErasureCoding,
+		VolumeID:    99,
+		Server:      "source-a",
+		Collection:  "c1",
+		Priority:    workertypes.TaskPriorityLow,
+		Reason:      "quiet and full",
+		TypedParams: params,
+	}
+
+	proposal, err := buildErasureCodingProposal(result)
+	if err != nil {
+		t.Fatalf("buildErasureCodingProposal() err = %v", err)
+	}
+	if proposal.JobType != "erasure_coding" {
+		t.Fatalf("unexpected job type %q", proposal.JobType)
+	}
+	if proposal.Parameters["task_params_pb"] == nil {
+		t.Fatalf("expected serialized task params")
+	}
+	if proposal.Labels["source_node"] != "source-a:8080" {
+		t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
+	}
+}
+
+func TestErasureCodingHandlerRejectsUnsupportedJobType(t *testing.T) {
+	handler := NewErasureCodingHandler(nil)
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: "vacuum",
+	}, noopDetectionSender{})
+	if err == nil {
+		t.Fatalf("expected detect job type mismatch error")
+	}
+
+	err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
+	}, noopExecutionSender{})
+	if err == nil {
+		t.Fatalf("expected execute job type mismatch error")
+	}
+}
+
+func TestErasureCodingHandlerDetectSkipsByMinInterval(t *testing.T) {
+	handler := NewErasureCodingHandler(nil)
+	sender := &recordingDetectionSender{}
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:           "erasure_coding",
+		LastSuccessfulRun: timestamppb.New(time.Now().Add(-3 * time.Second)),
+		WorkerConfigValues: map[string]*plugin_pb.ConfigValue{
+			"min_interval_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10}},
+		},
+	}, sender)
+	if err != nil {
+		t.Fatalf("detect returned err = %v", err)
+	}
+	if sender.proposals == nil {
+		t.Fatalf("expected proposals message")
+	}
+	if len(sender.proposals.Proposals) != 0 {
+		t.Fatalf("expected zero proposals, got %d", len(sender.proposals.Proposals))
+	}
+	if sender.complete == nil || !sender.complete.Success {
+		t.Fatalf("expected successful completion message")
+	}
+	if len(sender.events) == 0 {
+		t.Fatalf("expected detector activity events")
+	}
+	if !strings.Contains(sender.events[0].Message, "min interval") {
+		t.Fatalf("unexpected skip-by-interval message: %q", sender.events[0].Message)
+	}
+}
+
+func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) {
+	sender := &recordingDetectionSender{}
+	config := erasurecodingtask.NewDefaultConfig()
+	config.QuietForSeconds = 5 * 60
+	config.MinSizeMB = 30
+	config.FullnessRatio = 0.91
+
+	metrics := []*workertypes.VolumeHealthMetrics{
+		{
+			VolumeID:      20,
+			Size:          0,
+			Age:           218*time.Hour + 41*time.Minute,
+			FullnessRatio: 0,
+		},
+		{
+			VolumeID:      27,
+			Size:          uint64(16 * 1024 * 1024 / 10),
+			Age:           91*time.Hour + time.Minute,
+			FullnessRatio: 0.002,
+		},
+		{
+			VolumeID:      12,
+			Size:          0,
+			Age:           219*time.Hour + 49*time.Minute,
+			FullnessRatio: 0,
+		},
+	}
+
+	if err := emitErasureCodingDetectionDecisionTrace(sender, metrics, config, nil); err != nil {
+		t.Fatalf("emitErasureCodingDetectionDecisionTrace error: %v", err)
+	}
+	if len(sender.events) < 4 {
+		t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
+	}
+
+	if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
+		t.Fatalf("expected detector source, got %v", sender.events[0].Source)
+	}
+	if !strings.Contains(sender.events[0].Message, "EC detection: No tasks created for 3 volumes") {
+		t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
+	}
+	if !strings.Contains(sender.events[1].Message, "ERASURE CODING: Volume 20: size=0.0MB") {
+		t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
+	}
+}
+
+func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) {
+	descriptor := NewErasureCodingHandler(nil).Descriptor()
+	if descriptor == nil || descriptor.WorkerConfigForm == nil {
+		t.Fatalf("expected worker config form in descriptor")
+	}
+	if workerConfigFormHasField(descriptor.WorkerConfigForm, "working_dir") {
+		t.Fatalf("unexpected working_dir in erasure coding worker config form")
+	}
+	if workerConfigFormHasField(descriptor.WorkerConfigForm, "cleanup_source") {
+		t.Fatalf("unexpected cleanup_source in erasure coding worker config form")
+	}
+}
+
+func TestApplyErasureCodingExecutionDefaultsForcesLocalFields(t *testing.T) {
+	params := &worker_pb.TaskParams{
+		TaskId:   "ec-test",
+		VolumeId: 100,
+		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
+			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
+				DataShards:    ecstorage.DataShardsCount,
+				ParityShards:  ecstorage.ParityShardsCount,
+				WorkingDir:    "/tmp/custom-from-job",
+				CleanupSource: false,
+			},
+		},
+	}
+
+	applyErasureCodingExecutionDefaults(params, nil)
+
+	ecParams := params.GetErasureCodingParams()
+	if ecParams == nil {
+		t.Fatalf("expected erasure coding params")
+	}
+	if ecParams.WorkingDir != defaultErasureCodingWorkingDir() {
+		t.Fatalf("expected local working_dir %q, got %q", defaultErasureCodingWorkingDir(), ecParams.WorkingDir)
+	}
+	if !ecParams.CleanupSource {
+		t.Fatalf("expected cleanup_source true")
+	}
+}
diff --git a/weed/plugin/worker/vacuum_handler.go b/weed/plugin/worker/vacuum_handler.go
new file mode 100644
index 000000000..4872e0538
--- /dev/null
+++ b/weed/plugin/worker/vacuum_handler.go
@@ -0,0 +1,870 @@
+package pluginworker
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	vacuumtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	defaultVacuumTaskBatchSize = int32(1000)
+)
+
+// VacuumHandler is the plugin job handler for vacuum job type.
+type VacuumHandler struct {
+	grpcDialOption grpc.DialOption
+}
+
+func NewVacuumHandler(grpcDialOption grpc.DialOption) *VacuumHandler {
+	return &VacuumHandler{grpcDialOption: grpcDialOption}
+}
+
+func (h *VacuumHandler) Capability() *plugin_pb.JobTypeCapability {
+	return &plugin_pb.JobTypeCapability{
+		JobType:                 "vacuum",
+		CanDetect:               true,
+		CanExecute:              true,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 2,
+		DisplayName:             "Volume Vacuum",
+		Description:             "Reclaims disk space by removing deleted files from volumes",
+	}
+}
+
+func (h *VacuumHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return &plugin_pb.JobTypeDescriptor{
+		JobType:           "vacuum",
+		DisplayName:       "Volume Vacuum",
+		Description:       "Detect and vacuum volumes with high garbage ratio",
+		Icon:              "fas fa-broom",
+		DescriptorVersion: 1,
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "vacuum-admin",
+			Title:       "Vacuum Admin Config",
+			Description: "Admin-side controls for vacuum detection scope.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "scope",
+					Title:       "Scope",
+					Description: "Optional filter to restrict detection.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "collection_filter",
+							Label:       "Collection Filter",
+							Description: "Only scan this collection when set.",
+							Placeholder: "all collections",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"collection_filter": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
+				},
+			},
+		},
+		WorkerConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "vacuum-worker",
+			Title:       "Vacuum Worker Config",
+			Description: "Worker-side vacuum thresholds.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "thresholds",
+					Title:       "Thresholds",
+					Description: "Detection thresholds and timing constraints.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "garbage_threshold",
+							Label:       "Garbage Threshold",
+							Description: "Detect volumes with garbage ratio >= threshold.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
+							MaxValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
+						},
+						{
+							Name:        "min_volume_age_seconds",
+							Label:       "Min Volume Age (s)",
+							Description: "Only detect volumes older than this age.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+						{
+							Name:        "min_interval_seconds",
+							Label:       "Min Interval (s)",
+							Description: "Minimum interval between vacuum on the same volume.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"garbage_threshold": {
+					Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
+				},
+				"min_volume_age_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 24 * 60 * 60},
+				},
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7 * 24 * 60 * 60},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			Enabled:                       true,
+			DetectionIntervalSeconds:      2 * 60 * 60,
+			DetectionTimeoutSeconds:       120,
+			MaxJobsPerDetection:           200,
+			GlobalExecutionConcurrency:    16,
+			PerWorkerExecutionConcurrency: 4,
+			RetryLimit:                    1,
+			RetryBackoffSeconds:           10,
+		},
+		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
+			"garbage_threshold": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
+			},
+			"min_volume_age_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 24 * 60 * 60},
+			},
+			"min_interval_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7 * 24 * 60 * 60},
+			},
+		},
+	}
+}
+
+func (h *VacuumHandler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender DetectionSender) error {
+	if request == nil {
+		return fmt.Errorf("run detection request is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("detection sender is nil")
+	}
+	if request.JobType != "" && request.JobType != "vacuum" {
+		return fmt.Errorf("job type %q is not handled by vacuum worker", request.JobType)
+	}
+
+	workerConfig := deriveVacuumConfig(request.GetWorkerConfigValues())
+	if shouldSkipDetectionByInterval(request.GetLastSuccessfulRun(), workerConfig.MinIntervalSeconds) {
+		minInterval := time.Duration(workerConfig.MinIntervalSeconds) * time.Second
+		_ = sender.SendActivity(buildDetectorActivity(
+			"skipped_by_interval",
+			fmt.Sprintf("VACUUM: Detection skipped due to min interval (%s)", minInterval),
+			map[string]*plugin_pb.ConfigValue{
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(workerConfig.MinIntervalSeconds)},
+				},
+			},
+		))
+		if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+			JobType:   "vacuum",
+			Proposals: []*plugin_pb.JobProposal{},
+			HasMore:   false,
+		}); err != nil {
+			return err
+		}
+		return sender.SendComplete(&plugin_pb.DetectionComplete{
+			JobType:        "vacuum",
+			Success:        true,
+			TotalProposals: 0,
+		})
+	}
+
+	collectionFilter := strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
+	masters := make([]string, 0)
+	if request.ClusterContext != nil {
+		masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
+	}
+	metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
+	if err != nil {
+		return err
+	}
+
+	clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
+	results, err := vacuumtask.Detection(metrics, clusterInfo, workerConfig)
+	if err != nil {
+		return err
+	}
+	if traceErr := emitVacuumDetectionDecisionTrace(sender, metrics, workerConfig, results); traceErr != nil {
+		glog.Warningf("Plugin worker failed to emit vacuum detection trace: %v", traceErr)
+	}
+
+	maxResults := int(request.MaxResults)
+	hasMore := false
+	if maxResults > 0 && len(results) > maxResults {
+		hasMore = true
+		results = results[:maxResults]
+	}
+
+	proposals := make([]*plugin_pb.JobProposal, 0, len(results))
+	for _, result := range results {
+		proposal, proposalErr := buildVacuumProposal(result)
+		if proposalErr != nil {
+			glog.Warningf("Plugin worker skip invalid vacuum proposal: %v", proposalErr)
+			continue
+		}
+		proposals = append(proposals, proposal)
+	}
+
+	if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+		JobType:   "vacuum",
+		Proposals: proposals,
+		HasMore:   hasMore,
+	}); err != nil {
+		return err
+	}
+
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		JobType:        "vacuum",
+		Success:        true,
+		TotalProposals: int32(len(proposals)),
+	})
+}
+
+func emitVacuumDetectionDecisionTrace(
+	sender DetectionSender,
+	metrics []*workertypes.VolumeHealthMetrics,
+	workerConfig *vacuumtask.Config,
+	results []*workertypes.TaskDetectionResult,
+) error {
+	if sender == nil || workerConfig == nil {
+		return nil
+	}
+
+	minVolumeAge := time.Duration(workerConfig.MinVolumeAgeSeconds) * time.Second
+	totalVolumes := len(metrics)
+
+	debugCount := 0
+	skippedDueToGarbage := 0
+	skippedDueToAge := 0
+	for _, metric := range metrics {
+		if metric == nil {
+			continue
+		}
+		if metric.GarbageRatio >= workerConfig.GarbageThreshold && metric.Age >= minVolumeAge {
+			continue
+		}
+		if debugCount < 5 {
+			if metric.GarbageRatio < workerConfig.GarbageThreshold {
+				skippedDueToGarbage++
+			}
+			if metric.Age < minVolumeAge {
+				skippedDueToAge++
+			}
+		}
+		debugCount++
+	}
+
+	summaryMessage := ""
+	summaryStage := "decision_summary"
+	if len(results) == 0 {
+		summaryMessage = fmt.Sprintf(
+			"VACUUM: No tasks created for %d volumes. Threshold=%.2f%%, MinAge=%s. Skipped: %d (garbage 0 {
+							metric.GarbageRatio = float64(metric.DeletedBytes) / float64(metric.Size)
+						}
+						if volumeSizeLimitBytes > 0 {
+							metric.FullnessRatio = float64(metric.Size) / float64(volumeSizeLimitBytes)
+						}
+						metric.Age = now.Sub(metric.LastModified)
+						metrics = append(metrics, metric)
+					}
+				}
+			}
+		}
+	}
+
+	replicaCounts := make(map[uint32]int)
+	for _, metric := range metrics {
+		replicaCounts[metric.VolumeID]++
+	}
+	for _, metric := range metrics {
+		metric.ReplicaCount = replicaCounts[metric.VolumeID]
+	}
+
+	return metrics, activeTopology, nil
+}
+
+func buildVacuumProposal(result *workertypes.TaskDetectionResult) (*plugin_pb.JobProposal, error) {
+	if result == nil {
+		return nil, fmt.Errorf("task detection result is nil")
+	}
+	if result.TypedParams == nil {
+		return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
+	}
+
+	paramsPayload, err := proto.Marshal(result.TypedParams)
+	if err != nil {
+		return nil, fmt.Errorf("marshal task params: %w", err)
+	}
+
+	proposalID := strings.TrimSpace(result.TaskID)
+	if proposalID == "" {
+		proposalID = fmt.Sprintf("vacuum-%d-%d", result.VolumeID, time.Now().UnixNano())
+	}
+
+	dedupeKey := fmt.Sprintf("vacuum:%d", result.VolumeID)
+	if result.Collection != "" {
+		dedupeKey = dedupeKey + ":" + result.Collection
+	}
+
+	summary := fmt.Sprintf("Vacuum volume %d", result.VolumeID)
+	if strings.TrimSpace(result.Server) != "" {
+		summary = summary + " on " + result.Server
+	}
+
+	return &plugin_pb.JobProposal{
+		ProposalId: proposalID,
+		DedupeKey:  dedupeKey,
+		JobType:    "vacuum",
+		Priority:   mapTaskPriority(result.Priority),
+		Summary:    summary,
+		Detail:     strings.TrimSpace(result.Reason),
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {
+				Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
+			},
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
+			},
+			"server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Server},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
+			},
+		},
+		Labels: map[string]string{
+			"task_type":   "vacuum",
+			"volume_id":   fmt.Sprintf("%d", result.VolumeID),
+			"collection":  result.Collection,
+			"source_node": result.Server,
+		},
+	}, nil
+}
+
+func decodeVacuumTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
+	if job == nil {
+		return nil, fmt.Errorf("job spec is nil")
+	}
+
+	if payload := readBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
+		params := &worker_pb.TaskParams{}
+		if err := proto.Unmarshal(payload, params); err != nil {
+			return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
+		}
+		if params.TaskId == "" {
+			params.TaskId = job.JobId
+		}
+		return params, nil
+	}
+
+	volumeID := readInt64Config(job.Parameters, "volume_id", 0)
+	server := readStringConfig(job.Parameters, "server", "")
+	collection := readStringConfig(job.Parameters, "collection", "")
+	if volumeID <= 0 {
+		return nil, fmt.Errorf("missing volume_id in job parameters")
+	}
+	if strings.TrimSpace(server) == "" {
+		return nil, fmt.Errorf("missing server in job parameters")
+	}
+
+	return &worker_pb.TaskParams{
+		TaskId:     job.JobId,
+		VolumeId:   uint32(volumeID),
+		Collection: collection,
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     server,
+				VolumeId: uint32(volumeID),
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_VacuumParams{
+			VacuumParams: &worker_pb.VacuumTaskParams{
+				GarbageThreshold: 0.3,
+				BatchSize:        defaultVacuumTaskBatchSize,
+				VerifyChecksum:   true,
+			},
+		},
+	}, nil
+}
+
+func readStringConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback string) string {
+	if values == nil {
+		return fallback
+	}
+	value := values[field]
+	if value == nil {
+		return fallback
+	}
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_StringValue:
+		return kind.StringValue
+	case *plugin_pb.ConfigValue_Int64Value:
+		return strconv.FormatInt(kind.Int64Value, 10)
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return strconv.FormatFloat(kind.DoubleValue, 'f', -1, 64)
+	case *plugin_pb.ConfigValue_BoolValue:
+		return strconv.FormatBool(kind.BoolValue)
+	}
+	return fallback
+}
+
+func readDoubleConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback float64) float64 {
+	if values == nil {
+		return fallback
+	}
+	value := values[field]
+	if value == nil {
+		return fallback
+	}
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return kind.DoubleValue
+	case *plugin_pb.ConfigValue_Int64Value:
+		return float64(kind.Int64Value)
+	case *plugin_pb.ConfigValue_StringValue:
+		parsed, err := strconv.ParseFloat(strings.TrimSpace(kind.StringValue), 64)
+		if err == nil {
+			return parsed
+		}
+	case *plugin_pb.ConfigValue_BoolValue:
+		if kind.BoolValue {
+			return 1
+		}
+		return 0
+	}
+	return fallback
+}
+
+func readInt64Config(values map[string]*plugin_pb.ConfigValue, field string, fallback int64) int64 {
+	if values == nil {
+		return fallback
+	}
+	value := values[field]
+	if value == nil {
+		return fallback
+	}
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_Int64Value:
+		return kind.Int64Value
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return int64(kind.DoubleValue)
+	case *plugin_pb.ConfigValue_StringValue:
+		parsed, err := strconv.ParseInt(strings.TrimSpace(kind.StringValue), 10, 64)
+		if err == nil {
+			return parsed
+		}
+	case *plugin_pb.ConfigValue_BoolValue:
+		if kind.BoolValue {
+			return 1
+		}
+		return 0
+	}
+	return fallback
+}
+
+func readBytesConfig(values map[string]*plugin_pb.ConfigValue, field string) []byte {
+	if values == nil {
+		return nil
+	}
+	value := values[field]
+	if value == nil {
+		return nil
+	}
+	if kind, ok := value.Kind.(*plugin_pb.ConfigValue_BytesValue); ok {
+		return kind.BytesValue
+	}
+	return nil
+}
+
+func mapTaskPriority(priority workertypes.TaskPriority) plugin_pb.JobPriority {
+	switch strings.ToLower(string(priority)) {
+	case "low":
+		return plugin_pb.JobPriority_JOB_PRIORITY_LOW
+	case "medium", "normal":
+		return plugin_pb.JobPriority_JOB_PRIORITY_NORMAL
+	case "high":
+		return plugin_pb.JobPriority_JOB_PRIORITY_HIGH
+	case "critical":
+		return plugin_pb.JobPriority_JOB_PRIORITY_CRITICAL
+	default:
+		return plugin_pb.JobPriority_JOB_PRIORITY_NORMAL
+	}
+}
+
+func masterAddressCandidates(address string) []string {
+	trimmed := strings.TrimSpace(address)
+	if trimmed == "" {
+		return nil
+	}
+	candidateSet := map[string]struct{}{
+		trimmed: {},
+	}
+	converted := pb.ServerToGrpcAddress(trimmed)
+	candidateSet[converted] = struct{}{}
+
+	candidates := make([]string, 0, len(candidateSet))
+	for candidate := range candidateSet {
+		candidates = append(candidates, candidate)
+	}
+	sort.Strings(candidates)
+	return candidates
+}
+
+func shouldSkipDetectionByInterval(lastSuccessfulRun *timestamppb.Timestamp, minIntervalSeconds int) bool {
+	if lastSuccessfulRun == nil || minIntervalSeconds <= 0 {
+		return false
+	}
+	lastRun := lastSuccessfulRun.AsTime()
+	if lastRun.IsZero() {
+		return false
+	}
+	return time.Since(lastRun) < time.Duration(minIntervalSeconds)*time.Second
+}
+
+func buildExecutorActivity(stage string, message string) *plugin_pb.ActivityEvent {
+	return &plugin_pb.ActivityEvent{
+		Source:    plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR,
+		Stage:     stage,
+		Message:   message,
+		CreatedAt: timestamppb.Now(),
+	}
+}
+
+func buildDetectorActivity(stage string, message string, details map[string]*plugin_pb.ConfigValue) *plugin_pb.ActivityEvent {
+	return &plugin_pb.ActivityEvent{
+		Source:    plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR,
+		Stage:     stage,
+		Message:   message,
+		Details:   details,
+		CreatedAt: timestamppb.Now(),
+	}
+}
diff --git a/weed/plugin/worker/vacuum_handler_test.go b/weed/plugin/worker/vacuum_handler_test.go
new file mode 100644
index 000000000..3f2528974
--- /dev/null
+++ b/weed/plugin/worker/vacuum_handler_test.go
@@ -0,0 +1,277 @@
+package pluginworker
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	vacuumtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestDecodeVacuumTaskParamsFromPayload(t *testing.T) {
+	expected := &worker_pb.TaskParams{
+		TaskId:     "task-1",
+		VolumeId:   42,
+		Collection: "photos",
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     "10.0.0.1:8080",
+				VolumeId: 42,
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_VacuumParams{
+			VacuumParams: &worker_pb.VacuumTaskParams{
+				GarbageThreshold: 0.33,
+				BatchSize:        500,
+				VerifyChecksum:   true,
+			},
+		},
+	}
+	payload, err := proto.Marshal(expected)
+	if err != nil {
+		t.Fatalf("marshal payload: %v", err)
+	}
+
+	job := &plugin_pb.JobSpec{
+		JobId: "job-from-admin",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
+		},
+	}
+
+	actual, err := decodeVacuumTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeVacuumTaskParams() err = %v", err)
+	}
+	if !proto.Equal(expected, actual) {
+		t.Fatalf("decoded params mismatch\nexpected: %+v\nactual:   %+v", expected, actual)
+	}
+}
+
+func TestDecodeVacuumTaskParamsFallback(t *testing.T) {
+	job := &plugin_pb.JobSpec{
+		JobId: "job-2",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id":  {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
+			"server":     {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
+			"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
+		},
+	}
+
+	params, err := decodeVacuumTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeVacuumTaskParams() err = %v", err)
+	}
+	if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
+		t.Fatalf("unexpected basic params: %+v", params)
+	}
+	if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
+		t.Fatalf("unexpected sources: %+v", params.Sources)
+	}
+	if params.GetVacuumParams() == nil {
+		t.Fatalf("expected fallback vacuum params")
+	}
+}
+
+func TestDeriveVacuumConfigAllowsZeroValues(t *testing.T) {
+	values := map[string]*plugin_pb.ConfigValue{
+		"garbage_threshold": {
+			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0},
+		},
+		"min_volume_age_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0},
+		},
+		"min_interval_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0},
+		},
+	}
+
+	cfg := deriveVacuumConfig(values)
+	if cfg.GarbageThreshold != 0 {
+		t.Fatalf("expected garbage_threshold 0, got %v", cfg.GarbageThreshold)
+	}
+	if cfg.MinVolumeAgeSeconds != 0 {
+		t.Fatalf("expected min_volume_age_seconds 0, got %d", cfg.MinVolumeAgeSeconds)
+	}
+	if cfg.MinIntervalSeconds != 0 {
+		t.Fatalf("expected min_interval_seconds 0, got %d", cfg.MinIntervalSeconds)
+	}
+}
+
+func TestMasterAddressCandidates(t *testing.T) {
+	candidates := masterAddressCandidates("localhost:9333")
+	if len(candidates) != 2 {
+		t.Fatalf("expected 2 candidates, got %d: %v", len(candidates), candidates)
+	}
+	seen := map[string]bool{}
+	for _, candidate := range candidates {
+		seen[candidate] = true
+	}
+	if !seen["localhost:9333"] {
+		t.Fatalf("expected original address in candidates: %v", candidates)
+	}
+	if !seen["localhost:19333"] {
+		t.Fatalf("expected grpc address in candidates: %v", candidates)
+	}
+}
+
+func TestShouldSkipDetectionByInterval(t *testing.T) {
+	if shouldSkipDetectionByInterval(nil, 10) {
+		t.Fatalf("expected false when timestamp is nil")
+	}
+	if shouldSkipDetectionByInterval(timestamppb.Now(), 0) {
+		t.Fatalf("expected false when min interval is zero")
+	}
+
+	recent := timestamppb.New(time.Now().Add(-5 * time.Second))
+	if !shouldSkipDetectionByInterval(recent, 10) {
+		t.Fatalf("expected true for recent successful run")
+	}
+
+	old := timestamppb.New(time.Now().Add(-30 * time.Second))
+	if shouldSkipDetectionByInterval(old, 10) {
+		t.Fatalf("expected false for old successful run")
+	}
+}
+
+func TestVacuumHandlerRejectsUnsupportedJobType(t *testing.T) {
+	handler := NewVacuumHandler(nil)
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: "balance",
+	}, noopDetectionSender{})
+	if err == nil {
+		t.Fatalf("expected detect job type mismatch error")
+	}
+
+	err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "balance"},
+	}, noopExecutionSender{})
+	if err == nil {
+		t.Fatalf("expected execute job type mismatch error")
+	}
+}
+
+func TestVacuumHandlerDetectSkipsByMinInterval(t *testing.T) {
+	handler := NewVacuumHandler(nil)
+	sender := &recordingDetectionSender{}
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:           "vacuum",
+		LastSuccessfulRun: timestamppb.New(time.Now().Add(-3 * time.Second)),
+		WorkerConfigValues: map[string]*plugin_pb.ConfigValue{
+			"min_interval_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10}},
+		},
+	}, sender)
+	if err != nil {
+		t.Fatalf("detect returned err = %v", err)
+	}
+	if sender.proposals == nil {
+		t.Fatalf("expected proposals message")
+	}
+	if len(sender.proposals.Proposals) != 0 {
+		t.Fatalf("expected zero proposals, got %d", len(sender.proposals.Proposals))
+	}
+	if sender.complete == nil || !sender.complete.Success {
+		t.Fatalf("expected successful completion message")
+	}
+}
+
+func TestBuildExecutorActivity(t *testing.T) {
+	activity := buildExecutorActivity("running", "vacuum in progress")
+	if activity == nil {
+		t.Fatalf("expected non-nil activity")
+	}
+	if activity.Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR {
+		t.Fatalf("unexpected source: %v", activity.Source)
+	}
+	if activity.Stage != "running" {
+		t.Fatalf("unexpected stage: %q", activity.Stage)
+	}
+	if activity.Message != "vacuum in progress" {
+		t.Fatalf("unexpected message: %q", activity.Message)
+	}
+	if activity.CreatedAt == nil {
+		t.Fatalf("expected created_at timestamp")
+	}
+}
+
+func TestEmitVacuumDetectionDecisionTraceNoTasks(t *testing.T) {
+	sender := &recordingDetectionSender{}
+	config := vacuumtask.NewDefaultConfig()
+	config.GarbageThreshold = 0.3
+	config.MinVolumeAgeSeconds = int((24 * time.Hour).Seconds())
+
+	metrics := []*workertypes.VolumeHealthMetrics{
+		{
+			VolumeID:     17,
+			GarbageRatio: 0,
+			Age:          218*time.Hour + 23*time.Minute,
+		},
+		{
+			VolumeID:     16,
+			GarbageRatio: 0,
+			Age:          218*time.Hour + 22*time.Minute,
+		},
+		{
+			VolumeID:     6,
+			GarbageRatio: 0,
+			Age:          90*time.Hour + 42*time.Minute,
+		},
+	}
+
+	if err := emitVacuumDetectionDecisionTrace(sender, metrics, config, nil); err != nil {
+		t.Fatalf("emitVacuumDetectionDecisionTrace error: %v", err)
+	}
+	if len(sender.events) < 4 {
+		t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
+	}
+
+	if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
+		t.Fatalf("expected detector source, got %v", sender.events[0].Source)
+	}
+	if !strings.Contains(sender.events[0].Message, "VACUUM: No tasks created for 3 volumes") {
+		t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
+	}
+	if !strings.Contains(sender.events[1].Message, "VACUUM: Volume 17: garbage=0.00%") {
+		t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
+	}
+}
+
+type noopDetectionSender struct{}
+
+func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
+func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error   { return nil }
+func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error       { return nil }
+
+type noopExecutionSender struct{}
+
+func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
+func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error     { return nil }
+
+type recordingDetectionSender struct {
+	proposals *plugin_pb.DetectionProposals
+	complete  *plugin_pb.DetectionComplete
+	events    []*plugin_pb.ActivityEvent
+}
+
+func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
+	r.proposals = proposals
+	return nil
+}
+
+func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
+	r.complete = complete
+	return nil
+}
+
+func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
+	if event != nil {
+		r.events = append(r.events, event)
+	}
+	return nil
+}
diff --git a/weed/plugin/worker/volume_balance_handler.go b/weed/plugin/worker/volume_balance_handler.go
new file mode 100644
index 000000000..b976b323d
--- /dev/null
+++ b/weed/plugin/worker/volume_balance_handler.go
@@ -0,0 +1,826 @@
+package pluginworker
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	balancetask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	defaultBalanceTimeoutSeconds = int32(10 * 60)
+)
+
+type volumeBalanceWorkerConfig struct {
+	TaskConfig         *balancetask.Config
+	MinIntervalSeconds int
+}
+
+// VolumeBalanceHandler is the plugin job handler for volume balancing.
+type VolumeBalanceHandler struct {
+	grpcDialOption grpc.DialOption
+}
+
+func NewVolumeBalanceHandler(grpcDialOption grpc.DialOption) *VolumeBalanceHandler {
+	return &VolumeBalanceHandler{grpcDialOption: grpcDialOption}
+}
+
+func (h *VolumeBalanceHandler) Capability() *plugin_pb.JobTypeCapability {
+	return &plugin_pb.JobTypeCapability{
+		JobType:                 "volume_balance",
+		CanDetect:               true,
+		CanExecute:              true,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 1,
+		DisplayName:             "Volume Balance",
+		Description:             "Moves volumes between servers to reduce skew in volume distribution",
+	}
+}
+
+func (h *VolumeBalanceHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return &plugin_pb.JobTypeDescriptor{
+		JobType:           "volume_balance",
+		DisplayName:       "Volume Balance",
+		Description:       "Detect and execute volume moves to balance server load",
+		Icon:              "fas fa-balance-scale",
+		DescriptorVersion: 1,
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "volume-balance-admin",
+			Title:       "Volume Balance Admin Config",
+			Description: "Admin-side controls for volume balance detection scope.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "scope",
+					Title:       "Scope",
+					Description: "Optional filters applied before balance detection.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "collection_filter",
+							Label:       "Collection Filter",
+							Description: "Only detect balance opportunities in this collection when set.",
+							Placeholder: "all collections",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"collection_filter": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
+				},
+			},
+		},
+		WorkerConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "volume-balance-worker",
+			Title:       "Volume Balance Worker Config",
+			Description: "Worker-side balance thresholds.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "thresholds",
+					Title:       "Detection Thresholds",
+					Description: "Controls for when balance jobs should be proposed.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        "imbalance_threshold",
+							Label:       "Imbalance Threshold",
+							Description: "Detect when skew exceeds this ratio.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
+							MaxValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
+						},
+						{
+							Name:        "min_server_count",
+							Label:       "Minimum Server Count",
+							Description: "Require at least this many servers for balancing.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2}},
+						},
+						{
+							Name:        "min_interval_seconds",
+							Label:       "Minimum Detection Interval (s)",
+							Description: "Skip detection if the last successful run is more recent than this interval.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							Required:    true,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				"imbalance_threshold": {
+					Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.2},
+				},
+				"min_server_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2},
+				},
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30 * 60},
+				},
+			},
+		},
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			Enabled:                       true,
+			DetectionIntervalSeconds:      30 * 60,
+			DetectionTimeoutSeconds:       120,
+			MaxJobsPerDetection:           100,
+			GlobalExecutionConcurrency:    16,
+			PerWorkerExecutionConcurrency: 4,
+			RetryLimit:                    1,
+			RetryBackoffSeconds:           15,
+		},
+		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
+			"imbalance_threshold": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.2},
+			},
+			"min_server_count": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2},
+			},
+			"min_interval_seconds": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30 * 60},
+			},
+		},
+	}
+}
+
+func (h *VolumeBalanceHandler) Detect(
+	ctx context.Context,
+	request *plugin_pb.RunDetectionRequest,
+	sender DetectionSender,
+) error {
+	if request == nil {
+		return fmt.Errorf("run detection request is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("detection sender is nil")
+	}
+	if request.JobType != "" && request.JobType != "volume_balance" {
+		return fmt.Errorf("job type %q is not handled by volume_balance worker", request.JobType)
+	}
+
+	workerConfig := deriveBalanceWorkerConfig(request.GetWorkerConfigValues())
+	if shouldSkipDetectionByInterval(request.GetLastSuccessfulRun(), workerConfig.MinIntervalSeconds) {
+		minInterval := time.Duration(workerConfig.MinIntervalSeconds) * time.Second
+		_ = sender.SendActivity(buildDetectorActivity(
+			"skipped_by_interval",
+			fmt.Sprintf("VOLUME BALANCE: Detection skipped due to min interval (%s)", minInterval),
+			map[string]*plugin_pb.ConfigValue{
+				"min_interval_seconds": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(workerConfig.MinIntervalSeconds)},
+				},
+			},
+		))
+		if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+			JobType:   "volume_balance",
+			Proposals: []*plugin_pb.JobProposal{},
+			HasMore:   false,
+		}); err != nil {
+			return err
+		}
+		return sender.SendComplete(&plugin_pb.DetectionComplete{
+			JobType:        "volume_balance",
+			Success:        true,
+			TotalProposals: 0,
+		})
+	}
+
+	collectionFilter := strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
+	masters := make([]string, 0)
+	if request.ClusterContext != nil {
+		masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
+	}
+
+	metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
+	if err != nil {
+		return err
+	}
+
+	clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
+	results, err := balancetask.Detection(metrics, clusterInfo, workerConfig.TaskConfig)
+	if err != nil {
+		return err
+	}
+	if traceErr := emitVolumeBalanceDetectionDecisionTrace(sender, metrics, workerConfig.TaskConfig, results); traceErr != nil {
+		glog.Warningf("Plugin worker failed to emit volume_balance detection trace: %v", traceErr)
+	}
+
+	maxResults := int(request.MaxResults)
+	hasMore := false
+	if maxResults > 0 && len(results) > maxResults {
+		hasMore = true
+		results = results[:maxResults]
+	}
+
+	proposals := make([]*plugin_pb.JobProposal, 0, len(results))
+	for _, result := range results {
+		proposal, proposalErr := buildVolumeBalanceProposal(result)
+		if proposalErr != nil {
+			glog.Warningf("Plugin worker skip invalid volume_balance proposal: %v", proposalErr)
+			continue
+		}
+		proposals = append(proposals, proposal)
+	}
+
+	if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+		JobType:   "volume_balance",
+		Proposals: proposals,
+		HasMore:   hasMore,
+	}); err != nil {
+		return err
+	}
+
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		JobType:        "volume_balance",
+		Success:        true,
+		TotalProposals: int32(len(proposals)),
+	})
+}
+
+func emitVolumeBalanceDetectionDecisionTrace(
+	sender DetectionSender,
+	metrics []*workertypes.VolumeHealthMetrics,
+	taskConfig *balancetask.Config,
+	results []*workertypes.TaskDetectionResult,
+) error {
+	if sender == nil || taskConfig == nil {
+		return nil
+	}
+
+	totalVolumes := len(metrics)
+	summaryMessage := ""
+	if len(results) == 0 {
+		summaryMessage = fmt.Sprintf(
+			"BALANCE: No tasks created for %d volumes across %d disk type(s). Threshold=%.1f%%, MinServers=%d",
+			totalVolumes,
+			countBalanceDiskTypes(metrics),
+			taskConfig.ImbalanceThreshold*100,
+			taskConfig.MinServerCount,
+		)
+	} else {
+		summaryMessage = fmt.Sprintf(
+			"BALANCE: Created %d task(s) for %d volumes across %d disk type(s). Threshold=%.1f%%, MinServers=%d",
+			len(results),
+			totalVolumes,
+			countBalanceDiskTypes(metrics),
+			taskConfig.ImbalanceThreshold*100,
+			taskConfig.MinServerCount,
+		)
+	}
+
+	if err := sender.SendActivity(buildDetectorActivity("decision_summary", summaryMessage, map[string]*plugin_pb.ConfigValue{
+		"total_volumes": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
+		},
+		"selected_tasks": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
+		},
+		"imbalance_threshold_percent": {
+			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.ImbalanceThreshold * 100},
+		},
+		"min_server_count": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinServerCount)},
+		},
+	})); err != nil {
+		return err
+	}
+
+	volumesByDiskType := make(map[string][]*workertypes.VolumeHealthMetrics)
+	for _, metric := range metrics {
+		if metric == nil {
+			continue
+		}
+		diskType := strings.TrimSpace(metric.DiskType)
+		if diskType == "" {
+			diskType = "unknown"
+		}
+		volumesByDiskType[diskType] = append(volumesByDiskType[diskType], metric)
+	}
+
+	diskTypes := make([]string, 0, len(volumesByDiskType))
+	for diskType := range volumesByDiskType {
+		diskTypes = append(diskTypes, diskType)
+	}
+	sort.Strings(diskTypes)
+
+	const minVolumeCount = 2
+	detailCount := 0
+	for _, diskType := range diskTypes {
+		diskMetrics := volumesByDiskType[diskType]
+		volumeCount := len(diskMetrics)
+		if volumeCount < minVolumeCount {
+			message := fmt.Sprintf(
+				"BALANCE [%s]: No tasks created - cluster too small (%d volumes, need ≥%d)",
+				diskType,
+				volumeCount,
+				minVolumeCount,
+			)
+			if err := sender.SendActivity(buildDetectorActivity("decision_disk_type", message, map[string]*plugin_pb.ConfigValue{
+				"disk_type": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: diskType},
+				},
+				"volume_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeCount)},
+				},
+				"required_min_volume_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: minVolumeCount},
+				},
+			})); err != nil {
+				return err
+			}
+			detailCount++
+			if detailCount >= 3 {
+				break
+			}
+			continue
+		}
+
+		serverVolumeCounts := make(map[string]int)
+		for _, metric := range diskMetrics {
+			serverVolumeCounts[metric.Server]++
+		}
+		if len(serverVolumeCounts) < taskConfig.MinServerCount {
+			message := fmt.Sprintf(
+				"BALANCE [%s]: No tasks created - too few servers (%d servers, need ≥%d)",
+				diskType,
+				len(serverVolumeCounts),
+				taskConfig.MinServerCount,
+			)
+			if err := sender.SendActivity(buildDetectorActivity("decision_disk_type", message, map[string]*plugin_pb.ConfigValue{
+				"disk_type": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: diskType},
+				},
+				"server_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(serverVolumeCounts))},
+				},
+				"required_min_server_count": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinServerCount)},
+				},
+			})); err != nil {
+				return err
+			}
+			detailCount++
+			if detailCount >= 3 {
+				break
+			}
+			continue
+		}
+
+		totalDiskTypeVolumes := len(diskMetrics)
+		avgVolumesPerServer := float64(totalDiskTypeVolumes) / float64(len(serverVolumeCounts))
+		maxVolumes := 0
+		minVolumes := totalDiskTypeVolumes
+		maxServer := ""
+		minServer := ""
+		for server, count := range serverVolumeCounts {
+			if count > maxVolumes {
+				maxVolumes = count
+				maxServer = server
+			}
+			if count < minVolumes {
+				minVolumes = count
+				minServer = server
+			}
+		}
+
+		imbalanceRatio := 0.0
+		if avgVolumesPerServer > 0 {
+			imbalanceRatio = float64(maxVolumes-minVolumes) / avgVolumesPerServer
+		}
+
+		stage := "decision_disk_type"
+		message := ""
+		if imbalanceRatio <= taskConfig.ImbalanceThreshold {
+			message = fmt.Sprintf(
+				"BALANCE [%s]: No tasks created - cluster well balanced. Imbalance=%.1f%% (threshold=%.1f%%). Max=%d volumes on %s, Min=%d on %s, Avg=%.1f",
+				diskType,
+				imbalanceRatio*100,
+				taskConfig.ImbalanceThreshold*100,
+				maxVolumes,
+				maxServer,
+				minVolumes,
+				minServer,
+				avgVolumesPerServer,
+			)
+		} else {
+			stage = "decision_candidate"
+			message = fmt.Sprintf(
+				"BALANCE [%s]: Candidate detected. Imbalance=%.1f%% (threshold=%.1f%%). Max=%d volumes on %s, Min=%d on %s, Avg=%.1f",
+				diskType,
+				imbalanceRatio*100,
+				taskConfig.ImbalanceThreshold*100,
+				maxVolumes,
+				maxServer,
+				minVolumes,
+				minServer,
+				avgVolumesPerServer,
+			)
+		}
+
+		if err := sender.SendActivity(buildDetectorActivity(stage, message, map[string]*plugin_pb.ConfigValue{
+			"disk_type": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: diskType},
+			},
+			"volume_count": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalDiskTypeVolumes)},
+			},
+			"server_count": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(serverVolumeCounts))},
+			},
+			"imbalance_percent": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: imbalanceRatio * 100},
+			},
+			"threshold_percent": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.ImbalanceThreshold * 100},
+			},
+			"max_volumes": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(maxVolumes)},
+			},
+			"min_volumes": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(minVolumes)},
+			},
+			"avg_volumes_per_server": {
+				Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: avgVolumesPerServer},
+			},
+		})); err != nil {
+			return err
+		}
+
+		detailCount++
+		if detailCount >= 3 {
+			break
+		}
+	}
+
+	return nil
+}
+
+func countBalanceDiskTypes(metrics []*workertypes.VolumeHealthMetrics) int {
+	diskTypes := make(map[string]struct{})
+	for _, metric := range metrics {
+		if metric == nil {
+			continue
+		}
+		diskType := strings.TrimSpace(metric.DiskType)
+		if diskType == "" {
+			diskType = "unknown"
+		}
+		diskTypes[diskType] = struct{}{}
+	}
+	return len(diskTypes)
+}
+
+func (h *VolumeBalanceHandler) Execute(
+	ctx context.Context,
+	request *plugin_pb.ExecuteJobRequest,
+	sender ExecutionSender,
+) error {
+	if request == nil || request.Job == nil {
+		return fmt.Errorf("execute request/job is nil")
+	}
+	if sender == nil {
+		return fmt.Errorf("execution sender is nil")
+	}
+	if request.Job.JobType != "" && request.Job.JobType != "volume_balance" {
+		return fmt.Errorf("job type %q is not handled by volume_balance worker", request.Job.JobType)
+	}
+
+	params, err := decodeVolumeBalanceTaskParams(request.Job)
+	if err != nil {
+		return err
+	}
+	if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
+		return fmt.Errorf("volume balance source node is required")
+	}
+	if len(params.Targets) == 0 || strings.TrimSpace(params.Targets[0].Node) == "" {
+		return fmt.Errorf("volume balance target node is required")
+	}
+
+	applyBalanceExecutionDefaults(params)
+
+	task := balancetask.NewBalanceTask(
+		request.Job.JobId,
+		params.Sources[0].Node,
+		params.VolumeId,
+		params.Collection,
+	)
+	task.SetProgressCallback(func(progress float64, stage string) {
+		message := fmt.Sprintf("balance progress %.0f%%", progress)
+		if strings.TrimSpace(stage) != "" {
+			message = stage
+		}
+		_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+			JobId:           request.Job.JobId,
+			JobType:         request.Job.JobType,
+			State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+			ProgressPercent: progress,
+			Stage:           stage,
+			Message:         message,
+			Activities: []*plugin_pb.ActivityEvent{
+				buildExecutorActivity(stage, message),
+			},
+		})
+	})
+
+	if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
+		JobId:           request.Job.JobId,
+		JobType:         request.Job.JobType,
+		State:           plugin_pb.JobState_JOB_STATE_ASSIGNED,
+		ProgressPercent: 0,
+		Stage:           "assigned",
+		Message:         "volume balance job accepted",
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("assigned", "volume balance job accepted"),
+		},
+	}); err != nil {
+		return err
+	}
+
+	if err := task.Execute(ctx, params); err != nil {
+		_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+			JobId:           request.Job.JobId,
+			JobType:         request.Job.JobType,
+			State:           plugin_pb.JobState_JOB_STATE_FAILED,
+			ProgressPercent: 100,
+			Stage:           "failed",
+			Message:         err.Error(),
+			Activities: []*plugin_pb.ActivityEvent{
+				buildExecutorActivity("failed", err.Error()),
+			},
+		})
+		return err
+	}
+
+	sourceNode := params.Sources[0].Node
+	targetNode := params.Targets[0].Node
+	resultSummary := fmt.Sprintf("volume %d moved from %s to %s", params.VolumeId, sourceNode, targetNode)
+
+	return sender.SendCompleted(&plugin_pb.JobCompleted{
+		JobId:   request.Job.JobId,
+		JobType: request.Job.JobType,
+		Success: true,
+		Result: &plugin_pb.JobResult{
+			Summary: resultSummary,
+			OutputValues: map[string]*plugin_pb.ConfigValue{
+				"volume_id": {
+					Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
+				},
+				"source_server": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
+				},
+				"target_server": {
+					Kind: &plugin_pb.ConfigValue_StringValue{StringValue: targetNode},
+				},
+			},
+		},
+		Activities: []*plugin_pb.ActivityEvent{
+			buildExecutorActivity("completed", resultSummary),
+		},
+	})
+}
+
+func (h *VolumeBalanceHandler) collectVolumeMetrics(
+	ctx context.Context,
+	masterAddresses []string,
+	collectionFilter string,
+) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
+	// Reuse the same master topology fetch/build flow used by the vacuum handler.
+	helper := &VacuumHandler{grpcDialOption: h.grpcDialOption}
+	return helper.collectVolumeMetrics(ctx, masterAddresses, collectionFilter)
+}
+
+func deriveBalanceWorkerConfig(values map[string]*plugin_pb.ConfigValue) *volumeBalanceWorkerConfig {
+	taskConfig := balancetask.NewDefaultConfig()
+
+	imbalanceThreshold := readDoubleConfig(values, "imbalance_threshold", taskConfig.ImbalanceThreshold)
+	if imbalanceThreshold < 0 {
+		imbalanceThreshold = 0
+	}
+	if imbalanceThreshold > 1 {
+		imbalanceThreshold = 1
+	}
+	taskConfig.ImbalanceThreshold = imbalanceThreshold
+
+	minServerCount := int(readInt64Config(values, "min_server_count", int64(taskConfig.MinServerCount)))
+	if minServerCount < 2 {
+		minServerCount = 2
+	}
+	taskConfig.MinServerCount = minServerCount
+
+	minIntervalSeconds := int(readInt64Config(values, "min_interval_seconds", 0))
+	if minIntervalSeconds < 0 {
+		minIntervalSeconds = 0
+	}
+
+	return &volumeBalanceWorkerConfig{
+		TaskConfig:         taskConfig,
+		MinIntervalSeconds: minIntervalSeconds,
+	}
+}
+
+func buildVolumeBalanceProposal(
+	result *workertypes.TaskDetectionResult,
+) (*plugin_pb.JobProposal, error) {
+	if result == nil {
+		return nil, fmt.Errorf("task detection result is nil")
+	}
+	if result.TypedParams == nil {
+		return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
+	}
+
+	params := proto.Clone(result.TypedParams).(*worker_pb.TaskParams)
+	applyBalanceExecutionDefaults(params)
+
+	paramsPayload, err := proto.Marshal(params)
+	if err != nil {
+		return nil, fmt.Errorf("marshal task params: %w", err)
+	}
+
+	proposalID := strings.TrimSpace(result.TaskID)
+	if proposalID == "" {
+		proposalID = fmt.Sprintf("volume-balance-%d-%d", result.VolumeID, time.Now().UnixNano())
+	}
+
+	dedupeKey := fmt.Sprintf("volume_balance:%d", result.VolumeID)
+	if result.Collection != "" {
+		dedupeKey += ":" + result.Collection
+	}
+
+	sourceNode := ""
+	if len(params.Sources) > 0 {
+		sourceNode = strings.TrimSpace(params.Sources[0].Node)
+	}
+	targetNode := ""
+	if len(params.Targets) > 0 {
+		targetNode = strings.TrimSpace(params.Targets[0].Node)
+	}
+
+	summary := fmt.Sprintf("Balance volume %d", result.VolumeID)
+	if sourceNode != "" && targetNode != "" {
+		summary = fmt.Sprintf("Move volume %d from %s to %s", result.VolumeID, sourceNode, targetNode)
+	}
+
+	return &plugin_pb.JobProposal{
+		ProposalId: proposalID,
+		DedupeKey:  dedupeKey,
+		JobType:    "volume_balance",
+		Priority:   mapTaskPriority(result.Priority),
+		Summary:    summary,
+		Detail:     strings.TrimSpace(result.Reason),
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {
+				Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
+			},
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
+			},
+			"source_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
+			},
+			"target_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: targetNode},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
+			},
+		},
+		Labels: map[string]string{
+			"task_type":     "balance",
+			"volume_id":     fmt.Sprintf("%d", result.VolumeID),
+			"collection":    result.Collection,
+			"source_node":   sourceNode,
+			"target_node":   targetNode,
+			"source_server": sourceNode,
+			"target_server": targetNode,
+		},
+	}, nil
+}
+
+func decodeVolumeBalanceTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
+	if job == nil {
+		return nil, fmt.Errorf("job spec is nil")
+	}
+
+	if payload := readBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
+		params := &worker_pb.TaskParams{}
+		if err := proto.Unmarshal(payload, params); err != nil {
+			return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
+		}
+		if params.TaskId == "" {
+			params.TaskId = job.JobId
+		}
+		return params, nil
+	}
+
+	volumeID := readInt64Config(job.Parameters, "volume_id", 0)
+	sourceNode := strings.TrimSpace(readStringConfig(job.Parameters, "source_server", ""))
+	if sourceNode == "" {
+		sourceNode = strings.TrimSpace(readStringConfig(job.Parameters, "server", ""))
+	}
+	targetNode := strings.TrimSpace(readStringConfig(job.Parameters, "target_server", ""))
+	if targetNode == "" {
+		targetNode = strings.TrimSpace(readStringConfig(job.Parameters, "target", ""))
+	}
+	collection := readStringConfig(job.Parameters, "collection", "")
+	timeoutSeconds := int32(readInt64Config(job.Parameters, "timeout_seconds", int64(defaultBalanceTimeoutSeconds)))
+	if timeoutSeconds <= 0 {
+		timeoutSeconds = defaultBalanceTimeoutSeconds
+	}
+	forceMove := readBoolConfig(job.Parameters, "force_move", false)
+
+	if volumeID <= 0 {
+		return nil, fmt.Errorf("missing volume_id in job parameters")
+	}
+	if sourceNode == "" {
+		return nil, fmt.Errorf("missing source_server in job parameters")
+	}
+	if targetNode == "" {
+		return nil, fmt.Errorf("missing target_server in job parameters")
+	}
+
+	return &worker_pb.TaskParams{
+		TaskId:     job.JobId,
+		VolumeId:   uint32(volumeID),
+		Collection: collection,
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     sourceNode,
+				VolumeId: uint32(volumeID),
+			},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{
+				Node:     targetNode,
+				VolumeId: uint32(volumeID),
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_BalanceParams{
+			BalanceParams: &worker_pb.BalanceTaskParams{
+				ForceMove:      forceMove,
+				TimeoutSeconds: timeoutSeconds,
+			},
+		},
+	}, nil
+}
+
+func applyBalanceExecutionDefaults(params *worker_pb.TaskParams) {
+	if params == nil {
+		return
+	}
+
+	balanceParams := params.GetBalanceParams()
+	if balanceParams == nil {
+		params.TaskParams = &worker_pb.TaskParams_BalanceParams{
+			BalanceParams: &worker_pb.BalanceTaskParams{
+				ForceMove:      false,
+				TimeoutSeconds: defaultBalanceTimeoutSeconds,
+			},
+		}
+		return
+	}
+
+	if balanceParams.TimeoutSeconds <= 0 {
+		balanceParams.TimeoutSeconds = defaultBalanceTimeoutSeconds
+	}
+}
+
+func readBoolConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback bool) bool {
+	if values == nil {
+		return fallback
+	}
+	value := values[field]
+	if value == nil {
+		return fallback
+	}
+	switch kind := value.Kind.(type) {
+	case *plugin_pb.ConfigValue_BoolValue:
+		return kind.BoolValue
+	case *plugin_pb.ConfigValue_Int64Value:
+		return kind.Int64Value != 0
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return kind.DoubleValue != 0
+	case *plugin_pb.ConfigValue_StringValue:
+		text := strings.TrimSpace(strings.ToLower(kind.StringValue))
+		switch text {
+		case "1", "true", "yes", "on":
+			return true
+		case "0", "false", "no", "off":
+			return false
+		}
+	}
+	return fallback
+}
diff --git a/weed/plugin/worker/volume_balance_handler_test.go b/weed/plugin/worker/volume_balance_handler_test.go
new file mode 100644
index 000000000..86c4453c1
--- /dev/null
+++ b/weed/plugin/worker/volume_balance_handler_test.go
@@ -0,0 +1,283 @@
+package pluginworker
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	balancetask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestDecodeVolumeBalanceTaskParamsFromPayload(t *testing.T) {
+	expected := &worker_pb.TaskParams{
+		TaskId:     "task-1",
+		VolumeId:   42,
+		Collection: "photos",
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     "10.0.0.1:8080",
+				VolumeId: 42,
+			},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{
+				Node:     "10.0.0.2:8080",
+				VolumeId: 42,
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_BalanceParams{
+			BalanceParams: &worker_pb.BalanceTaskParams{
+				ForceMove:      true,
+				TimeoutSeconds: 1200,
+			},
+		},
+	}
+	payload, err := proto.Marshal(expected)
+	if err != nil {
+		t.Fatalf("marshal payload: %v", err)
+	}
+
+	job := &plugin_pb.JobSpec{
+		JobId: "job-from-admin",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
+		},
+	}
+
+	actual, err := decodeVolumeBalanceTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
+	}
+	if !proto.Equal(expected, actual) {
+		t.Fatalf("decoded params mismatch\nexpected: %+v\nactual:   %+v", expected, actual)
+	}
+}
+
+func TestDecodeVolumeBalanceTaskParamsFallback(t *testing.T) {
+	job := &plugin_pb.JobSpec{
+		JobId: "job-2",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id":     {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
+			"source_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
+			"target_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.2:8080"}},
+			"collection":    {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
+		},
+	}
+
+	params, err := decodeVolumeBalanceTaskParams(job)
+	if err != nil {
+		t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
+	}
+	if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
+		t.Fatalf("unexpected basic params: %+v", params)
+	}
+	if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
+		t.Fatalf("unexpected sources: %+v", params.Sources)
+	}
+	if len(params.Targets) != 1 || params.Targets[0].Node != "127.0.0.2:8080" {
+		t.Fatalf("unexpected targets: %+v", params.Targets)
+	}
+	if params.GetBalanceParams() == nil {
+		t.Fatalf("expected fallback balance params")
+	}
+}
+
+func TestDeriveBalanceWorkerConfig(t *testing.T) {
+	values := map[string]*plugin_pb.ConfigValue{
+		"imbalance_threshold": {
+			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.45},
+		},
+		"min_server_count": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 5},
+		},
+		"min_interval_seconds": {
+			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 33},
+		},
+	}
+
+	cfg := deriveBalanceWorkerConfig(values)
+	if cfg.TaskConfig.ImbalanceThreshold != 0.45 {
+		t.Fatalf("expected imbalance_threshold 0.45, got %v", cfg.TaskConfig.ImbalanceThreshold)
+	}
+	if cfg.TaskConfig.MinServerCount != 5 {
+		t.Fatalf("expected min_server_count 5, got %d", cfg.TaskConfig.MinServerCount)
+	}
+	if cfg.MinIntervalSeconds != 33 {
+		t.Fatalf("expected min_interval_seconds 33, got %d", cfg.MinIntervalSeconds)
+	}
+}
+
+func TestBuildVolumeBalanceProposal(t *testing.T) {
+	params := &worker_pb.TaskParams{
+		TaskId:     "balance-task-1",
+		VolumeId:   55,
+		Collection: "images",
+		Sources: []*worker_pb.TaskSource{
+			{
+				Node:     "source-a:8080",
+				VolumeId: 55,
+			},
+		},
+		Targets: []*worker_pb.TaskTarget{
+			{
+				Node:     "target-b:8080",
+				VolumeId: 55,
+			},
+		},
+		TaskParams: &worker_pb.TaskParams_BalanceParams{
+			BalanceParams: &worker_pb.BalanceTaskParams{
+				TimeoutSeconds: 600,
+			},
+		},
+	}
+	result := &workertypes.TaskDetectionResult{
+		TaskID:      "balance-task-1",
+		TaskType:    workertypes.TaskTypeBalance,
+		VolumeID:    55,
+		Server:      "source-a",
+		Collection:  "images",
+		Priority:    workertypes.TaskPriorityHigh,
+		Reason:      "imbalanced load",
+		TypedParams: params,
+	}
+
+	proposal, err := buildVolumeBalanceProposal(result)
+	if err != nil {
+		t.Fatalf("buildVolumeBalanceProposal() err = %v", err)
+	}
+	if proposal.JobType != "volume_balance" {
+		t.Fatalf("unexpected job type %q", proposal.JobType)
+	}
+	if proposal.DedupeKey == "" {
+		t.Fatalf("expected dedupe key")
+	}
+	if proposal.Parameters["task_params_pb"] == nil {
+		t.Fatalf("expected serialized task params")
+	}
+	if proposal.Labels["source_node"] != "source-a:8080" {
+		t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
+	}
+	if proposal.Labels["target_node"] != "target-b:8080" {
+		t.Fatalf("unexpected target label %q", proposal.Labels["target_node"])
+	}
+}
+
+func TestVolumeBalanceHandlerRejectsUnsupportedJobType(t *testing.T) {
+	handler := NewVolumeBalanceHandler(nil)
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: "vacuum",
+	}, noopDetectionSender{})
+	if err == nil {
+		t.Fatalf("expected detect job type mismatch error")
+	}
+
+	err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
+	}, noopExecutionSender{})
+	if err == nil {
+		t.Fatalf("expected execute job type mismatch error")
+	}
+}
+
+func TestVolumeBalanceHandlerDetectSkipsByMinInterval(t *testing.T) {
+	handler := NewVolumeBalanceHandler(nil)
+	sender := &recordingDetectionSender{}
+	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:           "volume_balance",
+		LastSuccessfulRun: timestamppb.New(time.Now().Add(-3 * time.Second)),
+		WorkerConfigValues: map[string]*plugin_pb.ConfigValue{
+			"min_interval_seconds": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10}},
+		},
+	}, sender)
+	if err != nil {
+		t.Fatalf("detect returned err = %v", err)
+	}
+	if sender.proposals == nil {
+		t.Fatalf("expected proposals message")
+	}
+	if len(sender.proposals.Proposals) != 0 {
+		t.Fatalf("expected zero proposals, got %d", len(sender.proposals.Proposals))
+	}
+	if sender.complete == nil || !sender.complete.Success {
+		t.Fatalf("expected successful completion message")
+	}
+	if len(sender.events) == 0 {
+		t.Fatalf("expected detector activity events")
+	}
+	if !strings.Contains(sender.events[0].Message, "min interval") {
+		t.Fatalf("unexpected skip-by-interval message: %q", sender.events[0].Message)
+	}
+}
+
+func TestEmitVolumeBalanceDetectionDecisionTraceNoTasks(t *testing.T) {
+	sender := &recordingDetectionSender{}
+	config := balancetask.NewDefaultConfig()
+	config.ImbalanceThreshold = 0.2
+	config.MinServerCount = 2
+
+	metrics := []*workertypes.VolumeHealthMetrics{
+		{VolumeID: 1, Server: "server-a", DiskType: "hdd"},
+		{VolumeID: 2, Server: "server-a", DiskType: "hdd"},
+		{VolumeID: 3, Server: "server-b", DiskType: "hdd"},
+		{VolumeID: 4, Server: "server-b", DiskType: "hdd"},
+	}
+
+	if err := emitVolumeBalanceDetectionDecisionTrace(sender, metrics, config, nil); err != nil {
+		t.Fatalf("emitVolumeBalanceDetectionDecisionTrace error: %v", err)
+	}
+	if len(sender.events) < 2 {
+		t.Fatalf("expected at least 2 detection events, got %d", len(sender.events))
+	}
+	if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
+		t.Fatalf("expected detector source, got %v", sender.events[0].Source)
+	}
+	if !strings.Contains(sender.events[0].Message, "BALANCE: No tasks created for 4 volumes") {
+		t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
+	}
+	foundDiskTypeDecision := false
+	for _, event := range sender.events {
+		if strings.Contains(event.Message, "BALANCE [hdd]: No tasks created - cluster well balanced") {
+			foundDiskTypeDecision = true
+			break
+		}
+	}
+	if !foundDiskTypeDecision {
+		t.Fatalf("expected per-disk-type decision message")
+	}
+}
+
+func TestVolumeBalanceDescriptorOmitsExecutionTuningFields(t *testing.T) {
+	descriptor := NewVolumeBalanceHandler(nil).Descriptor()
+	if descriptor == nil || descriptor.WorkerConfigForm == nil {
+		t.Fatalf("expected worker config form in descriptor")
+	}
+	if workerConfigFormHasField(descriptor.WorkerConfigForm, "timeout_seconds") {
+		t.Fatalf("unexpected timeout_seconds in volume balance worker config form")
+	}
+	if workerConfigFormHasField(descriptor.WorkerConfigForm, "force_move") {
+		t.Fatalf("unexpected force_move in volume balance worker config form")
+	}
+}
+
+func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
+	if form == nil {
+		return false
+	}
+	for _, section := range form.Sections {
+		if section == nil {
+			continue
+		}
+		for _, field := range section.Fields {
+			if field != nil && field.Name == fieldName {
+				return true
+			}
+		}
+	}
+	return false
+}
diff --git a/weed/plugin/worker/worker.go b/weed/plugin/worker/worker.go
new file mode 100644
index 000000000..279ffbe9d
--- /dev/null
+++ b/weed/plugin/worker/worker.go
@@ -0,0 +1,939 @@
+package pluginworker
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+	defaultHeartbeatInterval = 15 * time.Second
+	defaultReconnectDelay    = 5 * time.Second
+	defaultSendBufferSize    = 256
+)
+
+// DetectionSender sends detection responses for one request.
+type DetectionSender interface {
+	SendProposals(*plugin_pb.DetectionProposals) error
+	SendComplete(*plugin_pb.DetectionComplete) error
+	SendActivity(*plugin_pb.ActivityEvent) error
+}
+
+// ExecutionSender sends execution progress/completion responses for one request.
+type ExecutionSender interface {
+	SendProgress(*plugin_pb.JobProgressUpdate) error
+	SendCompleted(*plugin_pb.JobCompleted) error
+}
+
+// JobHandler implements one plugin job type on the worker side.
+type JobHandler interface {
+	Capability() *plugin_pb.JobTypeCapability
+	Descriptor() *plugin_pb.JobTypeDescriptor
+	Detect(context.Context, *plugin_pb.RunDetectionRequest, DetectionSender) error
+	Execute(context.Context, *plugin_pb.ExecuteJobRequest, ExecutionSender) error
+}
+
+// WorkerOptions configures one plugin worker process.
+type WorkerOptions struct {
+	AdminServer             string
+	WorkerID                string
+	WorkerVersion           string
+	WorkerAddress           string
+	HeartbeatInterval       time.Duration
+	ReconnectDelay          time.Duration
+	MaxDetectionConcurrency int
+	MaxExecutionConcurrency int
+	GrpcDialOption          grpc.DialOption
+	Handlers                []JobHandler
+	Handler                 JobHandler
+}
+
+// Worker runs one plugin job handler over plugin.proto stream.
+type Worker struct {
+	opts WorkerOptions
+
+	detectSlots chan struct{}
+	execSlots   chan struct{}
+
+	handlers map[string]JobHandler
+
+	runningMu   sync.RWMutex
+	runningWork map[string]*plugin_pb.RunningWork
+
+	workCancelMu sync.Mutex
+	workCancel   map[string]context.CancelFunc
+
+	workerID string
+
+	connectionMu sync.RWMutex
+	connected    bool
+}
+
+// NewWorker creates a plugin worker instance.
+func NewWorker(options WorkerOptions) (*Worker, error) {
+	if strings.TrimSpace(options.AdminServer) == "" {
+		return nil, fmt.Errorf("admin server is required")
+	}
+	if options.GrpcDialOption == nil {
+		return nil, fmt.Errorf("grpc dial option is required")
+	}
+	if options.HeartbeatInterval <= 0 {
+		options.HeartbeatInterval = defaultHeartbeatInterval
+	}
+	if options.ReconnectDelay <= 0 {
+		options.ReconnectDelay = defaultReconnectDelay
+	}
+	if options.MaxDetectionConcurrency <= 0 {
+		options.MaxDetectionConcurrency = 1
+	}
+	if options.MaxExecutionConcurrency <= 0 {
+		options.MaxExecutionConcurrency = 1
+	}
+	if strings.TrimSpace(options.WorkerVersion) == "" {
+		options.WorkerVersion = "dev"
+	}
+
+	workerID := strings.TrimSpace(options.WorkerID)
+	if workerID == "" {
+		workerID = generateWorkerID()
+	}
+
+	workerAddress := strings.TrimSpace(options.WorkerAddress)
+	if workerAddress == "" {
+		hostname, _ := os.Hostname()
+		workerAddress = hostname
+	}
+	opts := options
+	opts.WorkerAddress = workerAddress
+
+	allHandlers := make([]JobHandler, 0, len(opts.Handlers)+1)
+	if opts.Handler != nil {
+		allHandlers = append(allHandlers, opts.Handler)
+	}
+	allHandlers = append(allHandlers, opts.Handlers...)
+	if len(allHandlers) == 0 {
+		return nil, fmt.Errorf("at least one job handler is required")
+	}
+
+	handlers := make(map[string]JobHandler, len(allHandlers))
+	for i, handler := range allHandlers {
+		if handler == nil {
+			return nil, fmt.Errorf("job handler at index %d is nil", i)
+		}
+		handlerJobType, err := resolveHandlerJobType(handler)
+		if err != nil {
+			return nil, fmt.Errorf("resolve job handler at index %d: %w", i, err)
+		}
+		key := normalizeJobTypeKey(handlerJobType)
+		if key == "" {
+			return nil, fmt.Errorf("job handler at index %d has empty job type", i)
+		}
+		if _, found := handlers[key]; found {
+			return nil, fmt.Errorf("duplicate job handler for job type %q", handlerJobType)
+		}
+		handlers[key] = handler
+	}
+	if opts.Handler == nil {
+		opts.Handler = allHandlers[0]
+	}
+
+	w := &Worker{
+		opts:        opts,
+		detectSlots: make(chan struct{}, opts.MaxDetectionConcurrency),
+		execSlots:   make(chan struct{}, opts.MaxExecutionConcurrency),
+		handlers:    handlers,
+		runningWork: make(map[string]*plugin_pb.RunningWork),
+		workCancel:  make(map[string]context.CancelFunc),
+		workerID:    workerID,
+	}
+	return w, nil
+}
+
+// Run keeps the plugin worker connected and reconnects on stream failures.
+func (w *Worker) Run(ctx context.Context) error {
+	adminAddress := pb.ServerToGrpcAddress(w.opts.AdminServer)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+		}
+
+		if err := w.runOnce(ctx, adminAddress); err != nil {
+			if ctx.Err() != nil {
+				return nil
+			}
+			glog.Warningf("Plugin worker %s stream ended: %v", w.workerID, err)
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-time.After(w.opts.ReconnectDelay):
+		}
+	}
+}
+
+func (w *Worker) runOnce(ctx context.Context, adminAddress string) error {
+	defer w.setConnected(false)
+
+	dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second)
+	defer cancelDial()
+
+	conn, err := pb.GrpcDial(dialCtx, adminAddress, false, w.opts.GrpcDialOption)
+	if err != nil {
+		return fmt.Errorf("dial admin %s: %w", adminAddress, err)
+	}
+	defer conn.Close()
+
+	client := plugin_pb.NewPluginControlServiceClient(conn)
+	connCtx, cancelConn := context.WithCancel(ctx)
+	defer cancelConn()
+
+	stream, err := client.WorkerStream(connCtx)
+	if err != nil {
+		return fmt.Errorf("open worker stream: %w", err)
+	}
+	w.setConnected(true)
+
+	sendCh := make(chan *plugin_pb.WorkerToAdminMessage, defaultSendBufferSize)
+	sendErrCh := make(chan error, 1)
+
+	send := func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		if msg == nil {
+			return false
+		}
+		msg.WorkerId = w.workerID
+		if msg.SentAt == nil {
+			msg.SentAt = timestamppb.Now()
+		}
+		select {
+		case <-connCtx.Done():
+			return false
+		case sendCh <- msg:
+			return true
+		}
+	}
+
+	go func() {
+		for {
+			select {
+			case <-connCtx.Done():
+				return
+			case msg := <-sendCh:
+				if msg == nil {
+					continue
+				}
+				if err := stream.Send(msg); err != nil {
+					select {
+					case sendErrCh <- err:
+					default:
+					}
+					cancelConn()
+					return
+				}
+			}
+		}
+	}()
+
+	if !send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Hello{Hello: w.buildHello()},
+	}) {
+		return fmt.Errorf("send worker hello: stream closed")
+	}
+
+	heartbeatTicker := time.NewTicker(w.opts.HeartbeatInterval)
+	defer heartbeatTicker.Stop()
+
+	go func() {
+		for {
+			select {
+			case <-connCtx.Done():
+				return
+			case <-heartbeatTicker.C:
+				w.sendHeartbeat(send)
+			}
+		}
+	}()
+
+	for {
+		select {
+		case <-connCtx.Done():
+			return connCtx.Err()
+		case err := <-sendErrCh:
+			return fmt.Errorf("send to admin stream: %w", err)
+		default:
+		}
+
+		message, err := stream.Recv()
+		if err != nil {
+			return fmt.Errorf("recv admin message: %w", err)
+		}
+
+		w.handleAdminMessage(connCtx, message, send)
+	}
+}
+
+// IsConnected reports whether the worker currently has an active stream to admin.
+func (w *Worker) IsConnected() bool {
+	w.connectionMu.RLock()
+	defer w.connectionMu.RUnlock()
+	return w.connected
+}
+
+func (w *Worker) setConnected(connected bool) {
+	w.connectionMu.Lock()
+	w.connected = connected
+	w.connectionMu.Unlock()
+}
+
+func (w *Worker) handleAdminMessage(
+	ctx context.Context,
+	message *plugin_pb.AdminToWorkerMessage,
+	send func(*plugin_pb.WorkerToAdminMessage) bool,
+) {
+	if message == nil {
+		return
+	}
+
+	switch body := message.Body.(type) {
+	case *plugin_pb.AdminToWorkerMessage_Hello:
+		_ = body
+	case *plugin_pb.AdminToWorkerMessage_RequestConfigSchema:
+		w.handleSchemaRequest(message.GetRequestId(), body.RequestConfigSchema, send)
+	case *plugin_pb.AdminToWorkerMessage_RunDetectionRequest:
+		w.handleDetectionRequest(ctx, message.GetRequestId(), body.RunDetectionRequest, send)
+	case *plugin_pb.AdminToWorkerMessage_ExecuteJobRequest:
+		w.handleExecuteRequest(ctx, message.GetRequestId(), body.ExecuteJobRequest, send)
+	case *plugin_pb.AdminToWorkerMessage_CancelRequest:
+		cancel := body.CancelRequest
+		targetID := ""
+		if cancel != nil {
+			targetID = strings.TrimSpace(cancel.TargetId)
+		}
+		accepted := false
+		ackMessage := "cancel target is required"
+		if targetID != "" {
+			if w.cancelWork(targetID) {
+				accepted = true
+				ackMessage = "cancel request accepted"
+			} else {
+				ackMessage = "cancel target not found"
+			}
+		}
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_Acknowledge{Acknowledge: &plugin_pb.WorkerAcknowledge{
+				RequestId: message.GetRequestId(),
+				Accepted:  accepted,
+				Message:   ackMessage,
+			}},
+		})
+	case *plugin_pb.AdminToWorkerMessage_Shutdown:
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_Acknowledge{Acknowledge: &plugin_pb.WorkerAcknowledge{
+				RequestId: message.GetRequestId(),
+				Accepted:  true,
+				Message:   "shutdown acknowledged",
+			}},
+		})
+	default:
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_Acknowledge{Acknowledge: &plugin_pb.WorkerAcknowledge{
+				RequestId: message.GetRequestId(),
+				Accepted:  false,
+				Message:   "unsupported request body",
+			}},
+		})
+	}
+}
+
+func (w *Worker) handleSchemaRequest(requestID string, request *plugin_pb.RequestConfigSchema, send func(*plugin_pb.WorkerToAdminMessage) bool) {
+	jobType := ""
+	if request != nil {
+		jobType = strings.TrimSpace(request.JobType)
+	}
+
+	handler, resolvedJobType, err := w.findHandler(jobType)
+	if err != nil {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse{ConfigSchemaResponse: &plugin_pb.ConfigSchemaResponse{
+				RequestId:    requestID,
+				JobType:      jobType,
+				Success:      false,
+				ErrorMessage: err.Error(),
+			}},
+		})
+		return
+	}
+
+	descriptor := handler.Descriptor()
+	if descriptor == nil || descriptor.JobType == "" {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse{ConfigSchemaResponse: &plugin_pb.ConfigSchemaResponse{
+				RequestId:    requestID,
+				JobType:      resolvedJobType,
+				Success:      false,
+				ErrorMessage: "handler descriptor is not configured",
+			}},
+		})
+		return
+	}
+
+	send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse{ConfigSchemaResponse: &plugin_pb.ConfigSchemaResponse{
+			RequestId:         requestID,
+			JobType:           descriptor.JobType,
+			Success:           true,
+			JobTypeDescriptor: descriptor,
+		}},
+	})
+}
+
+func (w *Worker) handleDetectionRequest(
+	ctx context.Context,
+	requestID string,
+	request *plugin_pb.RunDetectionRequest,
+	send func(*plugin_pb.WorkerToAdminMessage) bool,
+) {
+	if request == nil {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_DetectionComplete{DetectionComplete: &plugin_pb.DetectionComplete{
+				RequestId:    requestID,
+				Success:      false,
+				ErrorMessage: "run detection request is nil",
+			}},
+		})
+		return
+	}
+
+	handler, resolvedJobType, err := w.findHandler(request.JobType)
+	if err != nil {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_DetectionComplete{DetectionComplete: &plugin_pb.DetectionComplete{
+				RequestId:    requestID,
+				JobType:      request.JobType,
+				Success:      false,
+				ErrorMessage: err.Error(),
+			}},
+		})
+		return
+	}
+
+	workKey := "detect:" + requestID
+	w.setRunningWork(workKey, &plugin_pb.RunningWork{
+		WorkId:          requestID,
+		Kind:            plugin_pb.WorkKind_WORK_KIND_DETECTION,
+		JobType:         resolvedJobType,
+		State:           plugin_pb.JobState_JOB_STATE_ASSIGNED,
+		ProgressPercent: 0,
+		Stage:           "queued",
+	})
+	w.sendHeartbeat(send)
+
+	requestCtx, cancelRequest := context.WithCancel(ctx)
+	w.setWorkCancel(cancelRequest, requestID)
+
+	send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Acknowledge{Acknowledge: &plugin_pb.WorkerAcknowledge{
+			RequestId: requestID,
+			Accepted:  true,
+			Message:   "detection request accepted",
+		}},
+	})
+
+	go func() {
+		detectionSender := &detectionSender{
+			requestID: requestID,
+			jobType:   resolvedJobType,
+			send:      send,
+		}
+		defer func() {
+			w.clearWorkCancel(requestID)
+			cancelRequest()
+			w.clearRunningWork(workKey)
+			w.sendHeartbeat(send)
+		}()
+
+		select {
+		case <-requestCtx.Done():
+			detectionSender.SendComplete(&plugin_pb.DetectionComplete{
+				Success:      false,
+				ErrorMessage: requestCtx.Err().Error(),
+			})
+			return
+		case w.detectSlots <- struct{}{}:
+		}
+		defer func() {
+			<-w.detectSlots
+			w.sendHeartbeat(send)
+		}()
+
+		w.setRunningWork(workKey, &plugin_pb.RunningWork{
+			WorkId:          requestID,
+			Kind:            plugin_pb.WorkKind_WORK_KIND_DETECTION,
+			JobType:         resolvedJobType,
+			State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+			ProgressPercent: 0,
+			Stage:           "detecting",
+		})
+		w.sendHeartbeat(send)
+
+		if err := handler.Detect(requestCtx, request, detectionSender); err != nil {
+			detectionSender.SendComplete(&plugin_pb.DetectionComplete{
+				Success:      false,
+				ErrorMessage: err.Error(),
+			})
+		}
+	}()
+}
+
+func (w *Worker) handleExecuteRequest(
+	ctx context.Context,
+	requestID string,
+	request *plugin_pb.ExecuteJobRequest,
+	send func(*plugin_pb.WorkerToAdminMessage) bool,
+) {
+	if request == nil || request.Job == nil {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_JobCompleted{JobCompleted: &plugin_pb.JobCompleted{
+				RequestId:    requestID,
+				Success:      false,
+				ErrorMessage: "execute request/job is nil",
+			}},
+		})
+		return
+	}
+
+	handler, resolvedJobType, err := w.findHandler(request.Job.JobType)
+	if err != nil {
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_JobCompleted{JobCompleted: &plugin_pb.JobCompleted{
+				RequestId:    requestID,
+				JobId:        request.Job.JobId,
+				JobType:      request.Job.JobType,
+				Success:      false,
+				ErrorMessage: err.Error(),
+			}},
+		})
+		return
+	}
+
+	select {
+	case w.execSlots <- struct{}{}:
+	default:
+		send(&plugin_pb.WorkerToAdminMessage{
+			Body: &plugin_pb.WorkerToAdminMessage_JobCompleted{JobCompleted: &plugin_pb.JobCompleted{
+				RequestId:    requestID,
+				JobId:        request.Job.JobId,
+				JobType:      resolvedJobType,
+				Success:      false,
+				ErrorMessage: "executor is at capacity",
+			}},
+		})
+		return
+	}
+	w.sendHeartbeat(send)
+
+	workKey := "exec:" + requestID
+	w.setRunningWork(workKey, &plugin_pb.RunningWork{
+		WorkId:          request.Job.JobId,
+		Kind:            plugin_pb.WorkKind_WORK_KIND_EXECUTION,
+		JobType:         resolvedJobType,
+		State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+		ProgressPercent: 0,
+		Stage:           "starting",
+	})
+	w.sendHeartbeat(send)
+
+	send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Acknowledge{Acknowledge: &plugin_pb.WorkerAcknowledge{
+			RequestId: requestID,
+			Accepted:  true,
+			Message:   "execute request accepted",
+		}},
+	})
+
+	go func() {
+		requestCtx, cancelRequest := context.WithCancel(ctx)
+		w.setWorkCancel(cancelRequest, requestID, request.Job.JobId)
+		defer func() {
+			w.clearWorkCancel(requestID, request.Job.JobId)
+			cancelRequest()
+			<-w.execSlots
+			w.clearRunningWork(workKey)
+			w.sendHeartbeat(send)
+		}()
+
+		executionSender := &executionSender{
+			requestID: requestID,
+			jobID:     request.Job.JobId,
+			jobType:   resolvedJobType,
+			send:      send,
+			onProgress: func(progress float64, stage string) {
+				w.updateRunningExecution(workKey, progress, stage)
+			},
+		}
+		if err := handler.Execute(requestCtx, request, executionSender); err != nil {
+			executionSender.SendCompleted(&plugin_pb.JobCompleted{
+				Success:      false,
+				ErrorMessage: err.Error(),
+			})
+		}
+	}()
+}
+
+func (w *Worker) buildHello() *plugin_pb.WorkerHello {
+	jobTypeKeys := make([]string, 0, len(w.handlers))
+	for key := range w.handlers {
+		jobTypeKeys = append(jobTypeKeys, key)
+	}
+	sort.Strings(jobTypeKeys)
+
+	capabilities := make([]*plugin_pb.JobTypeCapability, 0, len(jobTypeKeys))
+	jobTypes := make([]string, 0, len(jobTypeKeys))
+
+	for _, key := range jobTypeKeys {
+		handler := w.handlers[key]
+		if handler == nil {
+			continue
+		}
+		jobType, _ := resolveHandlerJobType(handler)
+		capability := handler.Capability()
+		if capability == nil {
+			capability = &plugin_pb.JobTypeCapability{}
+		} else {
+			capability = proto.Clone(capability).(*plugin_pb.JobTypeCapability)
+		}
+		if strings.TrimSpace(capability.JobType) == "" {
+			capability.JobType = jobType
+		}
+		capability.MaxDetectionConcurrency = int32(cap(w.detectSlots))
+		capability.MaxExecutionConcurrency = int32(cap(w.execSlots))
+		capabilities = append(capabilities, capability)
+		if capability.JobType != "" {
+			jobTypes = append(jobTypes, capability.JobType)
+		}
+	}
+
+	instanceID := generateWorkerID()
+	return &plugin_pb.WorkerHello{
+		WorkerId:         w.workerID,
+		WorkerInstanceId: "inst-" + instanceID,
+		Address:          w.opts.WorkerAddress,
+		WorkerVersion:    w.opts.WorkerVersion,
+		ProtocolVersion:  "plugin.v1",
+		Capabilities:     capabilities,
+		Metadata: map[string]string{
+			"runtime":   "plugin",
+			"job_types": strings.Join(jobTypes, ","),
+		},
+	}
+}
+
+func (w *Worker) buildHeartbeat() *plugin_pb.WorkerHeartbeat {
+	w.runningMu.RLock()
+	running := make([]*plugin_pb.RunningWork, 0, len(w.runningWork))
+	for _, work := range w.runningWork {
+		if work == nil {
+			continue
+		}
+		cloned := *work
+		running = append(running, &cloned)
+	}
+	w.runningMu.RUnlock()
+
+	detectUsed := len(w.detectSlots)
+	execUsed := len(w.execSlots)
+	return &plugin_pb.WorkerHeartbeat{
+		WorkerId:            w.workerID,
+		RunningWork:         running,
+		DetectionSlotsUsed:  int32(detectUsed),
+		DetectionSlotsTotal: int32(cap(w.detectSlots)),
+		ExecutionSlotsUsed:  int32(execUsed),
+		ExecutionSlotsTotal: int32(cap(w.execSlots)),
+		QueuedJobsByType:    map[string]int32{},
+		Metadata: map[string]string{
+			"runtime": "plugin",
+		},
+	}
+}
+
+func (w *Worker) sendHeartbeat(send func(*plugin_pb.WorkerToAdminMessage) bool) {
+	if send == nil {
+		return
+	}
+	send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_Heartbeat{
+			Heartbeat: w.buildHeartbeat(),
+		},
+	})
+}
+
+func (w *Worker) setRunningWork(key string, work *plugin_pb.RunningWork) {
+	if strings.TrimSpace(key) == "" || work == nil {
+		return
+	}
+	w.runningMu.Lock()
+	w.runningWork[key] = work
+	w.runningMu.Unlock()
+}
+
+func (w *Worker) clearRunningWork(key string) {
+	w.runningMu.Lock()
+	delete(w.runningWork, key)
+	w.runningMu.Unlock()
+}
+
+func (w *Worker) updateRunningExecution(key string, progress float64, stage string) {
+	w.runningMu.Lock()
+	if running := w.runningWork[key]; running != nil {
+		running.ProgressPercent = progress
+		if strings.TrimSpace(stage) != "" {
+			running.Stage = stage
+		}
+		running.State = plugin_pb.JobState_JOB_STATE_RUNNING
+	}
+	w.runningMu.Unlock()
+}
+
+type detectionSender struct {
+	requestID string
+	jobType   string
+	send      func(*plugin_pb.WorkerToAdminMessage) bool
+}
+
+func (s *detectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
+	if proposals == nil {
+		return fmt.Errorf("detection proposals are nil")
+	}
+	if proposals.RequestId == "" {
+		proposals.RequestId = s.requestID
+	}
+	if proposals.JobType == "" {
+		proposals.JobType = s.jobType
+	}
+	if !s.send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_DetectionProposals{DetectionProposals: proposals},
+	}) {
+		return fmt.Errorf("stream closed")
+	}
+	return nil
+}
+
+func (s *detectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
+	if complete == nil {
+		return fmt.Errorf("detection complete is nil")
+	}
+	if complete.RequestId == "" {
+		complete.RequestId = s.requestID
+	}
+	if complete.JobType == "" {
+		complete.JobType = s.jobType
+	}
+	if !s.send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_DetectionComplete{DetectionComplete: complete},
+	}) {
+		return fmt.Errorf("stream closed")
+	}
+	return nil
+}
+
+func (s *detectionSender) SendActivity(activity *plugin_pb.ActivityEvent) error {
+	if activity == nil {
+		return fmt.Errorf("detection activity is nil")
+	}
+	if activity.CreatedAt == nil {
+		activity.CreatedAt = timestamppb.Now()
+	}
+	if activity.Source == plugin_pb.ActivitySource_ACTIVITY_SOURCE_UNSPECIFIED {
+		activity.Source = plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR
+	}
+
+	update := &plugin_pb.JobProgressUpdate{
+		RequestId:       s.requestID,
+		JobType:         s.jobType,
+		State:           plugin_pb.JobState_JOB_STATE_RUNNING,
+		ProgressPercent: 0,
+		Stage:           activity.Stage,
+		Message:         activity.Message,
+		Activities:      []*plugin_pb.ActivityEvent{activity},
+		UpdatedAt:       timestamppb.Now(),
+	}
+
+	if !s.send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_JobProgressUpdate{JobProgressUpdate: update},
+	}) {
+		return fmt.Errorf("stream closed")
+	}
+	return nil
+}
+
+type executionSender struct {
+	requestID  string
+	jobID      string
+	jobType    string
+	send       func(*plugin_pb.WorkerToAdminMessage) bool
+	onProgress func(progress float64, stage string)
+}
+
+func (s *executionSender) SendProgress(progress *plugin_pb.JobProgressUpdate) error {
+	if progress == nil {
+		return fmt.Errorf("job progress is nil")
+	}
+	if progress.RequestId == "" {
+		progress.RequestId = s.requestID
+	}
+	if progress.JobId == "" {
+		progress.JobId = s.jobID
+	}
+	if progress.JobType == "" {
+		progress.JobType = s.jobType
+	}
+	if progress.UpdatedAt == nil {
+		progress.UpdatedAt = timestamppb.Now()
+	}
+	if s.onProgress != nil {
+		s.onProgress(progress.ProgressPercent, progress.Stage)
+	}
+	if !s.send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_JobProgressUpdate{JobProgressUpdate: progress},
+	}) {
+		return fmt.Errorf("stream closed")
+	}
+	return nil
+}
+
+func (s *executionSender) SendCompleted(completed *plugin_pb.JobCompleted) error {
+	if completed == nil {
+		return fmt.Errorf("job completed is nil")
+	}
+	if completed.RequestId == "" {
+		completed.RequestId = s.requestID
+	}
+	if completed.JobId == "" {
+		completed.JobId = s.jobID
+	}
+	if completed.JobType == "" {
+		completed.JobType = s.jobType
+	}
+	if completed.CompletedAt == nil {
+		completed.CompletedAt = timestamppb.Now()
+	}
+	if !s.send(&plugin_pb.WorkerToAdminMessage{
+		Body: &plugin_pb.WorkerToAdminMessage_JobCompleted{JobCompleted: completed},
+	}) {
+		return fmt.Errorf("stream closed")
+	}
+	return nil
+}
+
+func generateWorkerID() string {
+	random := make([]byte, 3)
+	if _, err := rand.Read(random); err != nil {
+		return fmt.Sprintf("plugin-%d", time.Now().UnixNano())
+	}
+	return "plugin-" + hex.EncodeToString(random)
+}
+
+func (w *Worker) setWorkCancel(cancel context.CancelFunc, keys ...string) {
+	if cancel == nil {
+		return
+	}
+	w.workCancelMu.Lock()
+	defer w.workCancelMu.Unlock()
+	for _, key := range keys {
+		key = strings.TrimSpace(key)
+		if key == "" {
+			continue
+		}
+		w.workCancel[key] = cancel
+	}
+}
+
+func (w *Worker) clearWorkCancel(keys ...string) {
+	w.workCancelMu.Lock()
+	defer w.workCancelMu.Unlock()
+	for _, key := range keys {
+		key = strings.TrimSpace(key)
+		if key == "" {
+			continue
+		}
+		delete(w.workCancel, key)
+	}
+}
+
+func (w *Worker) cancelWork(targetID string) bool {
+	targetID = strings.TrimSpace(targetID)
+	if targetID == "" {
+		return false
+	}
+
+	w.workCancelMu.Lock()
+	cancel := w.workCancel[targetID]
+	w.workCancelMu.Unlock()
+	if cancel == nil {
+		return false
+	}
+	cancel()
+	return true
+}
+
+func (w *Worker) findHandler(jobType string) (JobHandler, string, error) {
+	trimmed := strings.TrimSpace(jobType)
+	if trimmed == "" {
+		if len(w.handlers) == 1 {
+			for _, handler := range w.handlers {
+				resolvedJobType, err := resolveHandlerJobType(handler)
+				return handler, resolvedJobType, err
+			}
+		}
+		return nil, "", fmt.Errorf("job type is required when worker serves multiple job types")
+	}
+
+	key := normalizeJobTypeKey(trimmed)
+	handler := w.handlers[key]
+	if handler == nil {
+		return nil, "", fmt.Errorf("job type %q is not handled by this worker", trimmed)
+	}
+	resolvedJobType, err := resolveHandlerJobType(handler)
+	if err != nil {
+		return nil, "", err
+	}
+	return handler, resolvedJobType, nil
+}
+
+func resolveHandlerJobType(handler JobHandler) (string, error) {
+	if handler == nil {
+		return "", fmt.Errorf("job handler is nil")
+	}
+
+	if descriptor := handler.Descriptor(); descriptor != nil {
+		if jobType := strings.TrimSpace(descriptor.JobType); jobType != "" {
+			return jobType, nil
+		}
+	}
+	if capability := handler.Capability(); capability != nil {
+		if jobType := strings.TrimSpace(capability.JobType); jobType != "" {
+			return jobType, nil
+		}
+	}
+	return "", fmt.Errorf("handler job type is not configured")
+}
+
+func normalizeJobTypeKey(jobType string) string {
+	return strings.ToLower(strings.TrimSpace(jobType))
+}
diff --git a/weed/plugin/worker/worker_test.go b/weed/plugin/worker/worker_test.go
new file mode 100644
index 000000000..d540d048d
--- /dev/null
+++ b/weed/plugin/worker/worker_test.go
@@ -0,0 +1,599 @@
+package pluginworker
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestWorkerBuildHelloUsesConfiguredConcurrency(t *testing.T) {
+	handler := &testJobHandler{
+		capability: &plugin_pb.JobTypeCapability{
+			JobType:                 "vacuum",
+			CanDetect:               true,
+			CanExecute:              true,
+			MaxDetectionConcurrency: 99,
+			MaxExecutionConcurrency: 88,
+		},
+		descriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+	}
+
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:             "localhost:23646",
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler:                 handler,
+		MaxDetectionConcurrency: 3,
+		MaxExecutionConcurrency: 4,
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	hello := worker.buildHello()
+	if hello == nil || len(hello.Capabilities) != 1 {
+		t.Fatalf("expected one capability in hello")
+	}
+	capability := hello.Capabilities[0]
+	if capability.MaxDetectionConcurrency != 3 {
+		t.Fatalf("expected max_detection_concurrency=3, got=%d", capability.MaxDetectionConcurrency)
+	}
+	if capability.MaxExecutionConcurrency != 4 {
+		t.Fatalf("expected max_execution_concurrency=4, got=%d", capability.MaxExecutionConcurrency)
+	}
+	if capability.JobType != "vacuum" {
+		t.Fatalf("expected job type vacuum, got=%q", capability.JobType)
+	}
+}
+
+func TestWorkerBuildHelloIncludesMultipleCapabilities(t *testing.T) {
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:    "localhost:23646",
+		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handlers: []JobHandler{
+			&testJobHandler{
+				capability: &plugin_pb.JobTypeCapability{JobType: "vacuum", CanDetect: true, CanExecute: true},
+				descriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+			},
+			&testJobHandler{
+				capability: &plugin_pb.JobTypeCapability{JobType: "volume_balance", CanDetect: true, CanExecute: true},
+				descriptor: &plugin_pb.JobTypeDescriptor{JobType: "volume_balance"},
+			},
+		},
+		MaxDetectionConcurrency: 2,
+		MaxExecutionConcurrency: 3,
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	hello := worker.buildHello()
+	if hello == nil || len(hello.Capabilities) != 2 {
+		t.Fatalf("expected two capabilities in hello")
+	}
+
+	found := map[string]bool{}
+	for _, capability := range hello.Capabilities {
+		found[capability.JobType] = true
+		if capability.MaxDetectionConcurrency != 2 {
+			t.Fatalf("expected max_detection_concurrency=2, got=%d", capability.MaxDetectionConcurrency)
+		}
+		if capability.MaxExecutionConcurrency != 3 {
+			t.Fatalf("expected max_execution_concurrency=3, got=%d", capability.MaxExecutionConcurrency)
+		}
+	}
+	if !found["vacuum"] || !found["volume_balance"] {
+		t.Fatalf("expected capabilities for vacuum and volume_balance, got=%v", found)
+	}
+}
+
+func TestWorkerCancelWorkByTargetID(t *testing.T) {
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:    "localhost:23646",
+		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler: &testJobHandler{
+			capability: &plugin_pb.JobTypeCapability{JobType: "vacuum"},
+			descriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	worker.setWorkCancel(cancel, "request-1", "job-1")
+
+	if !worker.cancelWork("request-1") {
+		t.Fatalf("expected cancel by request id to succeed")
+	}
+	select {
+	case <-ctx.Done():
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("expected context to be canceled")
+	}
+
+	if !worker.cancelWork("job-1") {
+		t.Fatalf("expected cancel by job id to succeed")
+	}
+	if worker.cancelWork("unknown-target") {
+		t.Fatalf("expected cancel unknown target to fail")
+	}
+}
+
+func TestWorkerHandleCancelRequestAck(t *testing.T) {
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:    "localhost:23646",
+		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler: &testJobHandler{
+			capability: &plugin_pb.JobTypeCapability{JobType: "vacuum"},
+			descriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	canceled := false
+	worker.setWorkCancel(func() { canceled = true }, "job-42")
+
+	var response *plugin_pb.WorkerToAdminMessage
+	ok := worker.handleAdminMessageForTest(&plugin_pb.AdminToWorkerMessage{
+		RequestId: "cancel-req-1",
+		Body: &plugin_pb.AdminToWorkerMessage_CancelRequest{
+			CancelRequest: &plugin_pb.CancelRequest{TargetId: "job-42"},
+		},
+	}, func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		response = msg
+		return true
+	})
+	if !ok {
+		t.Fatalf("expected send callback to be invoked")
+	}
+	if !canceled {
+		t.Fatalf("expected registered work cancel function to be called")
+	}
+	if response == nil || response.GetAcknowledge() == nil || !response.GetAcknowledge().Accepted {
+		t.Fatalf("expected accepted acknowledge response, got=%+v", response)
+	}
+
+	response = nil
+	ok = worker.handleAdminMessageForTest(&plugin_pb.AdminToWorkerMessage{
+		RequestId: "cancel-req-2",
+		Body: &plugin_pb.AdminToWorkerMessage_CancelRequest{
+			CancelRequest: &plugin_pb.CancelRequest{TargetId: "missing"},
+		},
+	}, func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		response = msg
+		return true
+	})
+	if !ok {
+		t.Fatalf("expected send callback to be invoked")
+	}
+	if response == nil || response.GetAcknowledge() == nil || response.GetAcknowledge().Accepted {
+		t.Fatalf("expected rejected acknowledge for missing target, got=%+v", response)
+	}
+}
+
+func TestWorkerSchemaRequestRequiresJobTypeWhenMultipleHandlers(t *testing.T) {
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:    "localhost:23646",
+		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handlers: []JobHandler{
+			&testJobHandler{
+				capability: &plugin_pb.JobTypeCapability{JobType: "vacuum"},
+				descriptor: &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+			},
+			&testJobHandler{
+				capability: &plugin_pb.JobTypeCapability{JobType: "erasure_coding"},
+				descriptor: &plugin_pb.JobTypeDescriptor{JobType: "erasure_coding"},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	var response *plugin_pb.WorkerToAdminMessage
+	ok := worker.handleAdminMessageForTest(&plugin_pb.AdminToWorkerMessage{
+		RequestId: "schema-req-1",
+		Body: &plugin_pb.AdminToWorkerMessage_RequestConfigSchema{
+			RequestConfigSchema: &plugin_pb.RequestConfigSchema{},
+		},
+	}, func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		response = msg
+		return true
+	})
+	if !ok {
+		t.Fatalf("expected send callback to be invoked")
+	}
+	schema := response.GetConfigSchemaResponse()
+	if schema == nil || schema.Success {
+		t.Fatalf("expected schema error response, got=%+v", response)
+	}
+}
+
+func TestWorkerHandleDetectionQueuesWhenAtCapacity(t *testing.T) {
+	handler := &detectionQueueTestHandler{
+		capability: &plugin_pb.JobTypeCapability{
+			JobType:    "vacuum",
+			CanDetect:  true,
+			CanExecute: false,
+		},
+		descriptor:     &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+		detectEntered:  make(chan struct{}, 2),
+		detectContinue: make(chan struct{}, 2),
+	}
+
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:             "localhost:23646",
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler:                 handler,
+		MaxDetectionConcurrency: 1,
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	msgCh := make(chan *plugin_pb.WorkerToAdminMessage, 8)
+	send := func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		msgCh <- msg
+		return true
+	}
+
+	sendDetection := func(requestID string) {
+		worker.handleAdminMessage(context.Background(), &plugin_pb.AdminToWorkerMessage{
+			RequestId: requestID,
+			Body: &plugin_pb.AdminToWorkerMessage_RunDetectionRequest{
+				RunDetectionRequest: &plugin_pb.RunDetectionRequest{
+					JobType: "vacuum",
+				},
+			},
+		}, send)
+	}
+
+	sendDetection("detect-1")
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		ack := message.GetAcknowledge()
+		return ack != nil && ack.RequestId == "detect-1" && ack.Accepted
+	}, "detection acknowledge detect-1")
+	<-handler.detectEntered
+
+	sendDetection("detect-2")
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		ack := message.GetAcknowledge()
+		return ack != nil && ack.RequestId == "detect-2" && ack.Accepted
+	}, "detection acknowledge detect-2")
+
+	select {
+	case unexpected := <-msgCh:
+		t.Fatalf("did not expect detection completion before slot is available, got=%+v", unexpected)
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	handler.detectContinue <- struct{}{}
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		complete := message.GetDetectionComplete()
+		return complete != nil && complete.RequestId == "detect-1" && complete.Success
+	}, "detection complete detect-1")
+
+	<-handler.detectEntered
+	handler.detectContinue <- struct{}{}
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		complete := message.GetDetectionComplete()
+		return complete != nil && complete.RequestId == "detect-2" && complete.Success
+	}, "detection complete detect-2")
+}
+
+func TestWorkerHeartbeatReflectsActiveDetectionLoad(t *testing.T) {
+	handler := &detectionQueueTestHandler{
+		capability: &plugin_pb.JobTypeCapability{
+			JobType:    "vacuum",
+			CanDetect:  true,
+			CanExecute: false,
+		},
+		descriptor:     &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+		detectEntered:  make(chan struct{}, 1),
+		detectContinue: make(chan struct{}, 1),
+	}
+
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:             "localhost:23646",
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler:                 handler,
+		MaxDetectionConcurrency: 1,
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	msgCh := make(chan *plugin_pb.WorkerToAdminMessage, 16)
+	send := func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		msgCh <- msg
+		return true
+	}
+
+	requestID := "detect-heartbeat-1"
+	worker.handleAdminMessage(context.Background(), &plugin_pb.AdminToWorkerMessage{
+		RequestId: requestID,
+		Body: &plugin_pb.AdminToWorkerMessage_RunDetectionRequest{
+			RunDetectionRequest: &plugin_pb.RunDetectionRequest{
+				JobType: "vacuum",
+			},
+		},
+	}, send)
+
+	<-handler.detectEntered
+
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		heartbeat := message.GetHeartbeat()
+		return heartbeat != nil &&
+			heartbeat.DetectionSlotsUsed > 0 &&
+			heartbeatHasRunningWork(heartbeat, requestID, plugin_pb.WorkKind_WORK_KIND_DETECTION)
+	}, "active detection heartbeat")
+
+	handler.detectContinue <- struct{}{}
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		complete := message.GetDetectionComplete()
+		return complete != nil && complete.RequestId == requestID && complete.Success
+	}, "detection complete")
+
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		heartbeat := message.GetHeartbeat()
+		return heartbeat != nil && heartbeat.DetectionSlotsUsed == 0 &&
+			!heartbeatHasRunningWork(heartbeat, requestID, plugin_pb.WorkKind_WORK_KIND_DETECTION)
+	}, "idle detection heartbeat")
+}
+
+func TestWorkerHeartbeatReflectsActiveExecutionLoad(t *testing.T) {
+	handler := &executionHeartbeatTestHandler{
+		capability: &plugin_pb.JobTypeCapability{
+			JobType:    "vacuum",
+			CanDetect:  false,
+			CanExecute: true,
+		},
+		descriptor:     &plugin_pb.JobTypeDescriptor{JobType: "vacuum"},
+		executeEntered: make(chan struct{}, 1),
+		executeDone:    make(chan struct{}, 1),
+	}
+
+	worker, err := NewWorker(WorkerOptions{
+		AdminServer:             "localhost:23646",
+		GrpcDialOption:          grpc.WithTransportCredentials(insecure.NewCredentials()),
+		Handler:                 handler,
+		MaxExecutionConcurrency: 1,
+	})
+	if err != nil {
+		t.Fatalf("NewWorker error = %v", err)
+	}
+
+	msgCh := make(chan *plugin_pb.WorkerToAdminMessage, 16)
+	send := func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		msgCh <- msg
+		return true
+	}
+
+	requestID := "exec-heartbeat-1"
+	jobID := "job-heartbeat-1"
+	worker.handleAdminMessage(context.Background(), &plugin_pb.AdminToWorkerMessage{
+		RequestId: requestID,
+		Body: &plugin_pb.AdminToWorkerMessage_ExecuteJobRequest{
+			ExecuteJobRequest: &plugin_pb.ExecuteJobRequest{
+				Job: &plugin_pb.JobSpec{
+					JobId:   jobID,
+					JobType: "vacuum",
+				},
+			},
+		},
+	}, send)
+
+	<-handler.executeEntered
+
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		heartbeat := message.GetHeartbeat()
+		return heartbeat != nil &&
+			heartbeat.ExecutionSlotsUsed > 0 &&
+			heartbeatHasRunningWork(heartbeat, jobID, plugin_pb.WorkKind_WORK_KIND_EXECUTION)
+	}, "active execution heartbeat")
+
+	handler.executeDone <- struct{}{}
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		completed := message.GetJobCompleted()
+		return completed != nil && completed.RequestId == requestID && completed.Success
+	}, "execution complete")
+
+	waitForWorkerMessage(t, msgCh, func(message *plugin_pb.WorkerToAdminMessage) bool {
+		heartbeat := message.GetHeartbeat()
+		return heartbeat != nil && heartbeat.ExecutionSlotsUsed == 0 &&
+			!heartbeatHasRunningWork(heartbeat, jobID, plugin_pb.WorkKind_WORK_KIND_EXECUTION)
+	}, "idle execution heartbeat")
+}
+
+type testJobHandler struct {
+	capability *plugin_pb.JobTypeCapability
+	descriptor *plugin_pb.JobTypeDescriptor
+}
+
+func (h *testJobHandler) Capability() *plugin_pb.JobTypeCapability {
+	return h.capability
+}
+
+func (h *testJobHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return h.descriptor
+}
+
+func (h *testJobHandler) Detect(context.Context, *plugin_pb.RunDetectionRequest, DetectionSender) error {
+	return nil
+}
+
+func (h *testJobHandler) Execute(context.Context, *plugin_pb.ExecuteJobRequest, ExecutionSender) error {
+	return nil
+}
+
+type detectionQueueTestHandler struct {
+	capability *plugin_pb.JobTypeCapability
+	descriptor *plugin_pb.JobTypeDescriptor
+
+	detectEntered  chan struct{}
+	detectContinue chan struct{}
+}
+
+func (h *detectionQueueTestHandler) Capability() *plugin_pb.JobTypeCapability {
+	return h.capability
+}
+
+func (h *detectionQueueTestHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return h.descriptor
+}
+
+func (h *detectionQueueTestHandler) Detect(ctx context.Context, _ *plugin_pb.RunDetectionRequest, sender DetectionSender) error {
+	select {
+	case h.detectEntered <- struct{}{}:
+	default:
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-h.detectContinue:
+	}
+
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		Success: true,
+	})
+}
+
+func (h *detectionQueueTestHandler) Execute(context.Context, *plugin_pb.ExecuteJobRequest, ExecutionSender) error {
+	return nil
+}
+
+type executionHeartbeatTestHandler struct {
+	capability *plugin_pb.JobTypeCapability
+	descriptor *plugin_pb.JobTypeDescriptor
+
+	executeEntered chan struct{}
+	executeDone    chan struct{}
+}
+
+func (h *executionHeartbeatTestHandler) Capability() *plugin_pb.JobTypeCapability {
+	return h.capability
+}
+
+func (h *executionHeartbeatTestHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return h.descriptor
+}
+
+func (h *executionHeartbeatTestHandler) Detect(context.Context, *plugin_pb.RunDetectionRequest, DetectionSender) error {
+	return nil
+}
+
+func (h *executionHeartbeatTestHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender ExecutionSender) error {
+	select {
+	case h.executeEntered <- struct{}{}:
+	default:
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-h.executeDone:
+	}
+
+	return sender.SendCompleted(&plugin_pb.JobCompleted{
+		JobId:   request.Job.JobId,
+		JobType: request.Job.JobType,
+		Success: true,
+	})
+}
+
+func recvWorkerMessage(t *testing.T, msgCh <-chan *plugin_pb.WorkerToAdminMessage) *plugin_pb.WorkerToAdminMessage {
+	t.Helper()
+	select {
+	case msg := <-msgCh:
+		return msg
+	case <-time.After(2 * time.Second):
+		t.Fatal("timed out waiting for worker message")
+		return nil
+	}
+}
+
+func expectDetectionAck(t *testing.T, message *plugin_pb.WorkerToAdminMessage, requestID string) {
+	t.Helper()
+	ack := message.GetAcknowledge()
+	if ack == nil {
+		t.Fatalf("expected acknowledge for request %q, got=%+v", requestID, message)
+	}
+	if ack.RequestId != requestID {
+		t.Fatalf("expected acknowledge request_id=%q, got=%q", requestID, ack.RequestId)
+	}
+	if !ack.Accepted {
+		t.Fatalf("expected acknowledge accepted for request %q, got=%+v", requestID, ack)
+	}
+}
+
+func expectDetectionCompleteSuccess(t *testing.T, message *plugin_pb.WorkerToAdminMessage, requestID string) {
+	t.Helper()
+	complete := message.GetDetectionComplete()
+	if complete == nil {
+		t.Fatalf("expected detection complete for request %q, got=%+v", requestID, message)
+	}
+	if complete.RequestId != requestID {
+		t.Fatalf("expected detection complete request_id=%q, got=%q", requestID, complete.RequestId)
+	}
+	if !complete.Success {
+		t.Fatalf("expected successful detection complete for request %q, got=%+v", requestID, complete)
+	}
+}
+
+func waitForWorkerMessage(
+	t *testing.T,
+	msgCh <-chan *plugin_pb.WorkerToAdminMessage,
+	predicate func(*plugin_pb.WorkerToAdminMessage) bool,
+	description string,
+) *plugin_pb.WorkerToAdminMessage {
+	t.Helper()
+
+	timeout := time.NewTimer(3 * time.Second)
+	defer timeout.Stop()
+
+	for {
+		select {
+		case message := <-msgCh:
+			if predicate(message) {
+				return message
+			}
+		case <-timeout.C:
+			t.Fatalf("timed out waiting for %s", description)
+			return nil
+		}
+	}
+}
+
+func heartbeatHasRunningWork(heartbeat *plugin_pb.WorkerHeartbeat, workID string, kind plugin_pb.WorkKind) bool {
+	if heartbeat == nil || workID == "" {
+		return false
+	}
+	for _, work := range heartbeat.RunningWork {
+		if work == nil {
+			continue
+		}
+		if work.WorkId == workID && work.Kind == kind {
+			return true
+		}
+	}
+	return false
+}
+
+func (w *Worker) handleAdminMessageForTest(
+	message *plugin_pb.AdminToWorkerMessage,
+	send func(*plugin_pb.WorkerToAdminMessage) bool,
+) bool {
+	called := false
+	w.handleAdminMessage(context.Background(), message, func(msg *plugin_pb.WorkerToAdminMessage) bool {
+		called = true
+		return send(msg)
+	})
+	return called
+}
diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go
index 11412195d..a905ae16c 100644
--- a/weed/topology/data_node.go
+++ b/weed/topology/data_node.go
@@ -269,11 +269,24 @@ func (dn *DataNode) ToInfo() (info DataNodeInfo) {
 
 func (dn *DataNode) ToDataNodeInfo() *master_pb.DataNodeInfo {
 	m := &master_pb.DataNodeInfo{
-		Id:        string(dn.Id()),
-		DiskInfos: make(map[string]*master_pb.DiskInfo),
+		Id: string(dn.Id()),
+		// Start from disk usage counters so empty disks are still represented
+		// even when there are no volumes/EC shards on this data node yet.
+		DiskInfos: dn.diskUsages.ToDiskInfo(),
 		GrpcPort:  uint32(dn.GrpcPort),
 		Address:   dn.Url(), // ip:port for connecting to the volume server
 	}
+	if m.DiskInfos == nil {
+		m.DiskInfos = make(map[string]*master_pb.DiskInfo)
+	}
+	for diskType, diskInfo := range m.DiskInfos {
+		if diskInfo == nil {
+			m.DiskInfos[diskType] = &master_pb.DiskInfo{Type: diskType}
+			continue
+		}
+		diskInfo.Type = diskType
+	}
+
 	for _, c := range dn.Children() {
 		disk := c.(*Disk)
 		m.DiskInfos[string(disk.Id())] = disk.ToDiskInfo()
diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.go
index fa34e4db2..5885bbbb4 100644
--- a/weed/topology/topology_test.go
+++ b/weed/topology/topology_test.go
@@ -165,6 +165,29 @@ func TestHandlingVolumeServerHeartbeat(t *testing.T) {
 
 }
 
+func TestDataNodeToDataNodeInfo_IncludeEmptyDiskFromUsage(t *testing.T) {
+	dn := NewDataNode("node-1")
+	dn.Ip = "127.0.0.1"
+	dn.Port = 18080
+	dn.GrpcPort = 28080
+
+	// Simulate a node that has slot counters but no mounted volumes yet.
+	usage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
+	usage.maxVolumeCount = 8
+
+	info := dn.ToDataNodeInfo()
+	diskInfo, found := info.DiskInfos[""]
+	if !found {
+		t.Fatalf("expected default disk entry for empty node")
+	}
+	if diskInfo.MaxVolumeCount != 8 {
+		t.Fatalf("unexpected max volume count: got=%d want=8", diskInfo.MaxVolumeCount)
+	}
+	if len(diskInfo.VolumeInfos) != 0 {
+		t.Fatalf("expected no volumes for empty disk, got=%d", len(diskInfo.VolumeInfos))
+	}
+}
+
 func assert(t *testing.T, message string, actual, expected int) {
 	if actual != expected {
 		t.Fatalf("unexpected %s: %d, expected: %d", message, actual, expected)