plugin worker: support job type categories (all, default, heavy) (#8547)
* plugin worker: add handler registry with job categories
Introduce a self-registration pattern for plugin worker job handlers.
Each handler can register itself via init() with a HandlerFactory that
declares its job type, category (default/heavy), CLI aliases, and a
builder function.
ResolveHandlerFactories accepts a mix of category names ("all",
"default", "heavy") and explicit job type names/aliases, returning the
matching factories. This enables workers to be configured by resource
profile rather than requiring explicit job type enumeration.
* plugin worker: register all handlers via init()
Each job handler now self-registers into the global handler registry
with its canonical job type, category, CLI aliases, and build function:
- vacuum: category=default
- volume_balance: category=default
- admin_script: category=default
- erasure_coding: category=heavy
- iceberg_maintenance: category=heavy
Adding a new job type now only requires adding the init() call in the
handler file itself — no other files need to be touched.
* plugin worker: replace hardcoded job type switch with registry
Remove buildPluginWorkerHandler, parsePluginWorkerJobTypes, and
canonicalPluginWorkerJobType from worker_runtime.go. The simplified
buildPluginWorkerHandlers now delegates to
pluginworker.ResolveHandlerFactories, which resolves category names
("all", "default", "heavy") and explicit job type names/aliases.
The default job type is changed from an explicit list to "all", so new
handlers registered via init() are automatically picked up.
Update all tests to use the new API.
* plugin worker: update CLI help text for job categories
Update the -jobType flag description and command examples to document
category support (all, default, heavy) alongside explicit job type names.
* plugin worker: address review feedback
- Add CategoryAll constant; use typed constants in tokenAsCategory
- Pre-allocate result slice in ResolveHandlerFactories
- Add vacuum aliases (vol.vacuum, volume.vacuum)
- List alias examples (ec, balance, iceberg) in -jobType flag help
- Create handlers aggregator package for subpackage blank imports so
new handler subpackages only need to be added in one place
- Make category tests relationship-based (subset/union checks) instead
of asserting exact handler counts
- Add clarifying comments to worker_test.go and mini_plugin_test.go
listing expected handler names next to count assertions
---------
Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
@@ -15,7 +15,7 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
icebergworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker/iceberg"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/plugin/worker/handlers" // register all handler subpackages
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding,admin_script,iceberg_maintenance"
|
||||
const defaultPluginWorkerJobTypes = "all"
|
||||
|
||||
type pluginWorkerRunOptions struct {
|
||||
AdminServer string
|
||||
@@ -138,102 +138,37 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
|
||||
return worker.GenerateOrLoadWorkerID(workingDir)
|
||||
}
|
||||
|
||||
// buildPluginWorkerHandler constructs the JobHandler for the given job type.
|
||||
// maxExecute is forwarded to handlers that use it to report their own
|
||||
// MaxExecutionConcurrency in Capability for consistency and future-proofing.
|
||||
// The scheduler's effective per-worker MaxExecutionConcurrency is derived from
|
||||
// the worker-level configuration (e.g. WorkerOptions.MaxExecutionConcurrency),
|
||||
// not directly from the handler's Capability.
|
||||
func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExecute int, workingDir string) (pluginworker.JobHandler, error) {
|
||||
canonicalJobType, err := canonicalPluginWorkerJobType(jobType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch canonicalJobType {
|
||||
case "vacuum":
|
||||
return pluginworker.NewVacuumHandler(dialOption, int32(maxExecute)), nil
|
||||
case "volume_balance":
|
||||
return pluginworker.NewVolumeBalanceHandler(dialOption), nil
|
||||
case "erasure_coding":
|
||||
return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
|
||||
case "admin_script":
|
||||
return pluginworker.NewAdminScriptHandler(dialOption), nil
|
||||
case "iceberg_maintenance":
|
||||
return icebergworker.NewHandler(dialOption), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
|
||||
}
|
||||
}
|
||||
|
||||
// buildPluginWorkerHandlers constructs a deduplicated slice of JobHandlers for
|
||||
// the comma-separated jobTypes string, forwarding maxExecute to each handler.
|
||||
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
|
||||
// (which may contain category names like "all", "default", "heavy" and/or
|
||||
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
|
||||
parsedJobTypes, err := parsePluginWorkerJobTypes(jobTypes)
|
||||
jobTypes = strings.TrimSpace(jobTypes)
|
||||
if jobTypes == "" {
|
||||
jobTypes = defaultPluginWorkerJobTypes
|
||||
}
|
||||
|
||||
factories, err := pluginworker.ResolveHandlerFactories(jobTypes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
handlers := make([]pluginworker.JobHandler, 0, len(parsedJobTypes))
|
||||
for _, jobType := range parsedJobTypes {
|
||||
handler, buildErr := buildPluginWorkerHandler(jobType, dialOption, maxExecute, workingDir)
|
||||
opts := pluginworker.HandlerBuildOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
MaxExecute: maxExecute,
|
||||
WorkingDir: workingDir,
|
||||
}
|
||||
|
||||
handlers := make([]pluginworker.JobHandler, 0, len(factories))
|
||||
for _, f := range factories {
|
||||
handler, buildErr := f.Build(opts)
|
||||
if buildErr != nil {
|
||||
return nil, buildErr
|
||||
return nil, fmt.Errorf("building handler for %q: %w", f.JobType, buildErr)
|
||||
}
|
||||
handlers = append(handlers, handler)
|
||||
}
|
||||
return handlers, nil
|
||||
}
|
||||
|
||||
func parsePluginWorkerJobTypes(jobTypes string) ([]string, error) {
|
||||
jobTypes = strings.TrimSpace(jobTypes)
|
||||
if jobTypes == "" {
|
||||
return []string{"vacuum"}, nil
|
||||
}
|
||||
|
||||
parts := strings.Split(jobTypes, ",")
|
||||
parsed := make([]string, 0, len(parts))
|
||||
seen := make(map[string]struct{}, len(parts))
|
||||
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
canonical, err := canonicalPluginWorkerJobType(part)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, found := seen[canonical]; found {
|
||||
continue
|
||||
}
|
||||
seen[canonical] = struct{}{}
|
||||
parsed = append(parsed, canonical)
|
||||
}
|
||||
|
||||
if len(parsed) == 0 {
|
||||
return []string{"vacuum"}, nil
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func canonicalPluginWorkerJobType(jobType string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(jobType)) {
|
||||
case "", "vacuum":
|
||||
return "vacuum", nil
|
||||
case "volume_balance", "balance", "volume.balance", "volume-balance":
|
||||
return "volume_balance", nil
|
||||
case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
|
||||
return "erasure_coding", nil
|
||||
case "admin_script", "admin-script", "admin.script", "script", "admin":
|
||||
return "admin_script", nil
|
||||
case "iceberg_maintenance", "iceberg-maintenance", "iceberg.maintenance", "iceberg":
|
||||
return "iceberg_maintenance", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported plugin job type %q", jobType)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvePluginWorkerAdminServer(adminServer string) string {
|
||||
adminServer = strings.TrimSpace(adminServer)
|
||||
host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress(adminServer)
|
||||
|
||||
Reference in New Issue
Block a user