plugin worker: support job type categories (all, default, heavy) (#8547)

* plugin worker: add handler registry with job categories

Introduce a self-registration pattern for plugin worker job handlers.
Each handler can register itself via init() with a HandlerFactory that
declares its job type, category (default/heavy), CLI aliases, and a
builder function.

ResolveHandlerFactories accepts a mix of category names ("all",
"default", "heavy") and explicit job type names/aliases, returning the
matching factories. This enables workers to be configured by resource
profile rather than requiring explicit job type enumeration.

* plugin worker: register all handlers via init()

Each job handler now self-registers into the global handler registry
with its canonical job type, category, CLI aliases, and build function:

  - vacuum:              category=default
  - volume_balance:      category=default
  - admin_script:        category=default
  - erasure_coding:      category=heavy
  - iceberg_maintenance: category=heavy

Adding a new job type now only requires adding the init() call in the
handler file itself — no other files need to be touched.

* plugin worker: replace hardcoded job type switch with registry

Remove buildPluginWorkerHandler, parsePluginWorkerJobTypes, and
canonicalPluginWorkerJobType from worker_runtime.go. The simplified
buildPluginWorkerHandlers now delegates to
pluginworker.ResolveHandlerFactories, which resolves category names
("all", "default", "heavy") and explicit job type names/aliases.

The default job type is changed from an explicit list to "all", so new
handlers registered via init() are automatically picked up.

Update all tests to use the new API.

* plugin worker: update CLI help text for job categories

Update the -jobType flag description and command examples to document
category support (all, default, heavy) alongside explicit job type names.

* plugin worker: address review feedback

- Add CategoryAll constant; use typed constants in tokenAsCategory
- Pre-allocate result slice in ResolveHandlerFactories
- Add vacuum aliases (vol.vacuum, volume.vacuum)
- List alias examples (ec, balance, iceberg) in -jobType flag help
- Create handlers aggregator package for subpackage blank imports so
  new handler subpackages only need to be added in one place
- Make category tests relationship-based (subset/union checks) instead
  of asserting exact handler counts
- Add clarifying comments to worker_test.go and mini_plugin_test.go
  listing expected handler names next to count assertions

---------

Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
Chris Lu
2026-03-07 18:30:58 -08:00
committed by GitHub
parent f249fb7e63
commit 587c24ec89
12 changed files with 350 additions and 176 deletions

View File

@@ -15,7 +15,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/seaweedfs/seaweedfs/weed/glog"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
icebergworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker/iceberg"
_ "github.com/seaweedfs/seaweedfs/weed/plugin/worker/handlers" // register all handler subpackages
"github.com/seaweedfs/seaweedfs/weed/security"
statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -24,7 +24,7 @@ import (
"google.golang.org/grpc"
)
const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding,admin_script,iceberg_maintenance"
const defaultPluginWorkerJobTypes = "all"
type pluginWorkerRunOptions struct {
AdminServer string
@@ -138,102 +138,37 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
return worker.GenerateOrLoadWorkerID(workingDir)
}
// buildPluginWorkerHandler constructs the JobHandler for the given job type.
// maxExecute is forwarded to handlers that use it to report their own
// MaxExecutionConcurrency in Capability for consistency and future-proofing.
// The scheduler's effective per-worker MaxExecutionConcurrency is derived from
// the worker-level configuration (e.g. WorkerOptions.MaxExecutionConcurrency),
// not directly from the handler's Capability.
func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExecute int, workingDir string) (pluginworker.JobHandler, error) {
canonicalJobType, err := canonicalPluginWorkerJobType(jobType)
if err != nil {
return nil, err
}
switch canonicalJobType {
case "vacuum":
return pluginworker.NewVacuumHandler(dialOption, int32(maxExecute)), nil
case "volume_balance":
return pluginworker.NewVolumeBalanceHandler(dialOption), nil
case "erasure_coding":
return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
case "admin_script":
return pluginworker.NewAdminScriptHandler(dialOption), nil
case "iceberg_maintenance":
return icebergworker.NewHandler(dialOption), nil
default:
return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
}
}
// buildPluginWorkerHandlers constructs a deduplicated slice of JobHandlers for
// the comma-separated jobTypes string, forwarding maxExecute to each handler.
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
// (which may contain category names like "all", "default", "heavy" and/or
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
parsedJobTypes, err := parsePluginWorkerJobTypes(jobTypes)
jobTypes = strings.TrimSpace(jobTypes)
if jobTypes == "" {
jobTypes = defaultPluginWorkerJobTypes
}
factories, err := pluginworker.ResolveHandlerFactories(jobTypes)
if err != nil {
return nil, err
}
handlers := make([]pluginworker.JobHandler, 0, len(parsedJobTypes))
for _, jobType := range parsedJobTypes {
handler, buildErr := buildPluginWorkerHandler(jobType, dialOption, maxExecute, workingDir)
opts := pluginworker.HandlerBuildOptions{
GrpcDialOption: dialOption,
MaxExecute: maxExecute,
WorkingDir: workingDir,
}
handlers := make([]pluginworker.JobHandler, 0, len(factories))
for _, f := range factories {
handler, buildErr := f.Build(opts)
if buildErr != nil {
return nil, buildErr
return nil, fmt.Errorf("building handler for %q: %w", f.JobType, buildErr)
}
handlers = append(handlers, handler)
}
return handlers, nil
}
func parsePluginWorkerJobTypes(jobTypes string) ([]string, error) {
jobTypes = strings.TrimSpace(jobTypes)
if jobTypes == "" {
return []string{"vacuum"}, nil
}
parts := strings.Split(jobTypes, ",")
parsed := make([]string, 0, len(parts))
seen := make(map[string]struct{}, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
canonical, err := canonicalPluginWorkerJobType(part)
if err != nil {
return nil, err
}
if _, found := seen[canonical]; found {
continue
}
seen[canonical] = struct{}{}
parsed = append(parsed, canonical)
}
if len(parsed) == 0 {
return []string{"vacuum"}, nil
}
return parsed, nil
}
func canonicalPluginWorkerJobType(jobType string) (string, error) {
switch strings.ToLower(strings.TrimSpace(jobType)) {
case "", "vacuum":
return "vacuum", nil
case "volume_balance", "balance", "volume.balance", "volume-balance":
return "volume_balance", nil
case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
return "erasure_coding", nil
case "admin_script", "admin-script", "admin.script", "script", "admin":
return "admin_script", nil
case "iceberg_maintenance", "iceberg-maintenance", "iceberg.maintenance", "iceberg":
return "iceberg_maintenance", nil
default:
return "", fmt.Errorf("unsupported plugin job type %q", jobType)
}
}
func resolvePluginWorkerAdminServer(adminServer string) string {
adminServer = strings.TrimSpace(adminServer)
host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress(adminServer)