gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove ~50k lines of unreachable dead code (#8913)

Browse Source 
* chore: remove unreachable dead code across the codebase

Remove ~50,000 lines of unreachable code identified by static analysis.

Major removals:
- weed/filer/redis_lua: entire unused Redis Lua filer store implementation
- weed/wdclient/net2, resource_pool: unused connection/resource pool packages
- weed/plugin/worker/lifecycle: unused lifecycle plugin worker
- weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy,
  multipart IAM, key rotation, and various SSE helper functions
- weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions
- weed/mq/offset: unused SQL storage and migration code
- weed/worker: unused registry, task, and monitoring functions
- weed/query: unused SQL engine, parquet scanner, and type functions
- weed/shell: unused EC proportional rebalance functions
- weed/storage/erasure_coding/distribution: unused distribution analysis functions
- Individual unreachable functions removed from 150+ files across admin,
  credential, filer, iam, kms, mount, mq, operation, pb, s3api, server,
  shell, storage, topology, and util packages

* fix(s3): reset shared memory store in IAM test to prevent flaky failure

TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because
the MemoryStore credential backend is a singleton registered via init().
Earlier tests that create anonymous identities pollute the shared store,
causing LookupAnonymous() to unexpectedly return true.

Fix by calling Reset() on the memory store before the test runs.

* style: run gofmt on changed files

* fix: restore KMS functions used by integration tests

* fix(plugin): prevent panic on send to closed worker session channel

The Plugin.sendToWorker method could panic with "send on closed channel"
when a worker disconnected while a message was being sent. The race was
between streamSession.close() closing the outgoing channel and sendToWorker
writing to it concurrently.

Add a done channel to streamSession that is closed before the outgoing
channel, and check it in sendToWorker's select to safely detect closed
sessions without panicking.
This commit is contained in:
Chris Lu
2026-04-03 16:04:27 -07:00
committed by GitHub
parent 8fad85aed7
commit 995dfc4d5d
264 changed files with 62 additions and 46027 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
weed/admin/dash/types.go
View File

@@ -447,11 +447,6 @@ type QueueStats = maintenance.QueueStats
type WorkerDetailsData = maintenance.WorkerDetailsData
type WorkerPerformance = maintenance.WorkerPerformance
// GetTaskIcon returns the icon CSS class for a task type from its UI provider
func GetTaskIcon(taskType MaintenanceTaskType) string {
return maintenance.GetTaskIcon(taskType)
}
// Status constants (these are still static)
const (
TaskStatusPending = maintenance.TaskStatusPending

23
weed/admin/handlers/cluster_handlers.go
View File

@@ -312,29 +312,6 @@ func (h *ClusterHandlers) ShowClusterFilers(w http.ResponseWriter, r *http.Reque
}
}
// ShowClusterBrokers renders the cluster message brokers page
func (h *ClusterHandlers) ShowClusterBrokers(w http.ResponseWriter, r *http.Request) {
// Get cluster brokers data
brokersData, err := h.adminServer.GetClusterBrokers()
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get cluster brokers: "+err.Error())
return
}
username := usernameOrDefault(r)
brokersData.Username = username
// Render HTML template
w.Header().Set("Content-Type", "text/html")
brokersComponent := app.ClusterBrokers(*brokersData)
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
layoutComponent := layout.Layout(viewCtx, brokersComponent)
if err := layoutComponent.Render(r.Context(), w); err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
return
}
}
// GetClusterTopology returns the cluster topology as JSON
func (h *ClusterHandlers) GetClusterTopology(w http.ResponseWriter, r *http.Request) {
topology, err := h.adminServer.GetClusterTopology()

28
weed/admin/handlers/mq_handlers.go
View File

@@ -78,34 +78,6 @@ func (h *MessageQueueHandlers) ShowTopics(w http.ResponseWriter, r *http.Request
}
}
// ShowSubscribers renders the message queue subscribers page
func (h *MessageQueueHandlers) ShowSubscribers(w http.ResponseWriter, r *http.Request) {
// Get subscribers data
subscribersData, err := h.adminServer.GetSubscribers()
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get subscribers: "+err.Error())
return
}
// Set username
username := dash.UsernameFromContext(r.Context())
if username == "" {
username = "admin"
}
subscribersData.Username = username
// Render HTML template
w.Header().Set("Content-Type", "text/html")
subscribersComponent := app.Subscribers(*subscribersData)
viewCtx := layout.NewViewContext(r, username, dash.CSRFTokenFromContext(r.Context()))
layoutComponent := layout.Layout(viewCtx, subscribersComponent)
err = layoutComponent.Render(r.Context(), w)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to render template: "+err.Error())
return
}
}
// ShowTopicDetails renders the topic details page
func (h *MessageQueueHandlers) ShowTopicDetails(w http.ResponseWriter, r *http.Request) {
// Get topic parameters from URL

124
weed/admin/maintenance/config_verification.go
View File

@@ -1,124 +0,0 @@
package maintenance
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
func VerifyProtobufConfig() error {
// Create configuration manager
configManager := NewMaintenanceConfigManager()
config := configManager.GetConfig()
// Verify basic configuration
if !config.Enabled {
return fmt.Errorf("expected config to be enabled by default")
}
if config.ScanIntervalSeconds != 30*60 {
return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
}
// Verify policy configuration
if config.Policy == nil {
return fmt.Errorf("expected policy to be configured")
}
if config.Policy.GlobalMaxConcurrent != 4 {
return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
}
// Verify task policies
vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
if vacuumPolicy == nil {
return fmt.Errorf("expected vacuum policy to be configured")
}
if !vacuumPolicy.Enabled {
return fmt.Errorf("expected vacuum policy to be enabled")
}
// Verify typed configuration access
vacuumConfig := vacuumPolicy.GetVacuumConfig()
if vacuumConfig == nil {
return fmt.Errorf("expected vacuum config to be accessible")
}
if vacuumConfig.GarbageThreshold != 0.3 {
return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
}
// Verify helper functions work
if !IsTaskEnabled(config.Policy, "vacuum") {
return fmt.Errorf("expected vacuum task to be enabled via helper function")
}
maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
if maxConcurrent != 2 {
return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
}
// Verify erasure coding configuration
ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
if ecPolicy == nil {
return fmt.Errorf("expected EC policy to be configured")
}
ecConfig := ecPolicy.GetErasureCodingConfig()
if ecConfig == nil {
return fmt.Errorf("expected EC config to be accessible")
}
// Verify configurable EC fields only
if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
}
return nil
}
// GetProtobufConfigSummary returns a summary of the current protobuf configuration
func GetProtobufConfigSummary() string {
configManager := NewMaintenanceConfigManager()
config := configManager.GetConfig()
summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
summary += fmt.Sprintf(" Enabled: %v\n", config.Enabled)
summary += fmt.Sprintf(" Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
summary += fmt.Sprintf(" Max Retries: %d\n", config.MaxRetries)
summary += fmt.Sprintf(" Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
summary += fmt.Sprintf(" Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
for taskType, policy := range config.Policy.TaskPolicies {
summary += fmt.Sprintf(" %s: enabled=%v, max_concurrent=%d\n",
taskType, policy.Enabled, policy.MaxConcurrent)
}
return summary
}
// CreateCustomConfig demonstrates creating a custom protobuf configuration
func CreateCustomConfig() *worker_pb.MaintenanceConfig {
return &worker_pb.MaintenanceConfig{
Enabled: true,
ScanIntervalSeconds: 60 * 60, // 1 hour
MaxRetries: 5,
Policy: &worker_pb.MaintenancePolicy{
GlobalMaxConcurrent: 8,
TaskPolicies: map[string]*worker_pb.TaskPolicy{
"custom_vacuum": {
Enabled: true,
MaxConcurrent: 4,
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{
GarbageThreshold: 0.5,
MinVolumeAgeHours: 48,
},
},
},
},
},
}
}

265
weed/admin/maintenance/maintenance_config_proto.go
View File

@@ -1,24 +1,9 @@
package maintenance
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// MaintenanceConfigManager handles protobuf-based configuration
type MaintenanceConfigManager struct {
config *worker_pb.MaintenanceConfig
}
// NewMaintenanceConfigManager creates a new config manager with defaults
func NewMaintenanceConfigManager() *MaintenanceConfigManager {
return &MaintenanceConfigManager{
config: DefaultMaintenanceConfigProto(),
}
}
// DefaultMaintenanceConfigProto returns default configuration as protobuf
func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
return &worker_pb.MaintenanceConfig{
@@ -34,253 +19,3 @@ func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
Policy: nil,
}
}
// GetConfig returns the current configuration
func (mcm *MaintenanceConfigManager) GetConfig() *worker_pb.MaintenanceConfig {
return mcm.config
}
// Type-safe configuration accessors
// GetVacuumConfig returns vacuum-specific configuration for a task type
func (mcm *MaintenanceConfigManager) GetVacuumConfig(taskType string) *worker_pb.VacuumTaskConfig {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
return vacuumConfig
}
}
// Return defaults if not configured
return &worker_pb.VacuumTaskConfig{
GarbageThreshold: 0.3,
MinVolumeAgeHours: 24,
}
}
// GetErasureCodingConfig returns EC-specific configuration for a task type
func (mcm *MaintenanceConfigManager) GetErasureCodingConfig(taskType string) *worker_pb.ErasureCodingTaskConfig {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
return ecConfig
}
}
// Return defaults if not configured
return &worker_pb.ErasureCodingTaskConfig{
FullnessRatio: 0.95,
QuietForSeconds: 3600,
MinVolumeSizeMb: 100,
CollectionFilter: "",
}
}
// GetBalanceConfig returns balance-specific configuration for a task type
func (mcm *MaintenanceConfigManager) GetBalanceConfig(taskType string) *worker_pb.BalanceTaskConfig {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
return balanceConfig
}
}
// Return defaults if not configured
return &worker_pb.BalanceTaskConfig{
ImbalanceThreshold: 0.2,
MinServerCount: 2,
}
}
// GetReplicationConfig returns replication-specific configuration for a task type
func (mcm *MaintenanceConfigManager) GetReplicationConfig(taskType string) *worker_pb.ReplicationTaskConfig {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
if replicationConfig := policy.GetReplicationConfig(); replicationConfig != nil {
return replicationConfig
}
}
// Return defaults if not configured
return &worker_pb.ReplicationTaskConfig{
TargetReplicaCount: 2,
}
}
// Typed convenience methods for getting task configurations
// GetVacuumTaskConfigForType returns vacuum configuration for a specific task type
func (mcm *MaintenanceConfigManager) GetVacuumTaskConfigForType(taskType string) *worker_pb.VacuumTaskConfig {
return GetVacuumTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
}
// GetErasureCodingTaskConfigForType returns erasure coding configuration for a specific task type
func (mcm *MaintenanceConfigManager) GetErasureCodingTaskConfigForType(taskType string) *worker_pb.ErasureCodingTaskConfig {
return GetErasureCodingTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
}
// GetBalanceTaskConfigForType returns balance configuration for a specific task type
func (mcm *MaintenanceConfigManager) GetBalanceTaskConfigForType(taskType string) *worker_pb.BalanceTaskConfig {
return GetBalanceTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
}
// GetReplicationTaskConfigForType returns replication configuration for a specific task type
func (mcm *MaintenanceConfigManager) GetReplicationTaskConfigForType(taskType string) *worker_pb.ReplicationTaskConfig {
return GetReplicationTaskConfig(mcm.config.Policy, MaintenanceTaskType(taskType))
}
// Helper methods
func (mcm *MaintenanceConfigManager) getTaskPolicy(taskType string) *worker_pb.TaskPolicy {
if mcm.config.Policy != nil && mcm.config.Policy.TaskPolicies != nil {
return mcm.config.Policy.TaskPolicies[taskType]
}
return nil
}
// IsTaskEnabled returns whether a task type is enabled
func (mcm *MaintenanceConfigManager) IsTaskEnabled(taskType string) bool {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
return policy.Enabled
}
return false
}
// GetMaxConcurrent returns the max concurrent limit for a task type
func (mcm *MaintenanceConfigManager) GetMaxConcurrent(taskType string) int32 {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
return policy.MaxConcurrent
}
return 1 // Default
}
// GetRepeatInterval returns the repeat interval for a task type in seconds
func (mcm *MaintenanceConfigManager) GetRepeatInterval(taskType string) int32 {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
return policy.RepeatIntervalSeconds
}
return mcm.config.Policy.DefaultRepeatIntervalSeconds
}
// GetCheckInterval returns the check interval for a task type in seconds
func (mcm *MaintenanceConfigManager) GetCheckInterval(taskType string) int32 {
if policy := mcm.getTaskPolicy(taskType); policy != nil {
return policy.CheckIntervalSeconds
}
return mcm.config.Policy.DefaultCheckIntervalSeconds
}
// Duration accessor methods
// GetScanInterval returns the scan interval as a time.Duration
func (mcm *MaintenanceConfigManager) GetScanInterval() time.Duration {
return time.Duration(mcm.config.ScanIntervalSeconds) * time.Second
}
// GetWorkerTimeout returns the worker timeout as a time.Duration
func (mcm *MaintenanceConfigManager) GetWorkerTimeout() time.Duration {
return time.Duration(mcm.config.WorkerTimeoutSeconds) * time.Second
}
// GetTaskTimeout returns the task timeout as a time.Duration
func (mcm *MaintenanceConfigManager) GetTaskTimeout() time.Duration {
return time.Duration(mcm.config.TaskTimeoutSeconds) * time.Second
}
// GetRetryDelay returns the retry delay as a time.Duration
func (mcm *MaintenanceConfigManager) GetRetryDelay() time.Duration {
return time.Duration(mcm.config.RetryDelaySeconds) * time.Second
}
// GetCleanupInterval returns the cleanup interval as a time.Duration
func (mcm *MaintenanceConfigManager) GetCleanupInterval() time.Duration {
return time.Duration(mcm.config.CleanupIntervalSeconds) * time.Second
}
// GetTaskRetention returns the task retention period as a time.Duration
func (mcm *MaintenanceConfigManager) GetTaskRetention() time.Duration {
return time.Duration(mcm.config.TaskRetentionSeconds) * time.Second
}
// ValidateMaintenanceConfigWithSchema validates protobuf maintenance configuration using ConfigField rules
func ValidateMaintenanceConfigWithSchema(config *worker_pb.MaintenanceConfig) error {
if config == nil {
return fmt.Errorf("configuration cannot be nil")
}
// Get the schema to access field validation rules
schema := GetMaintenanceConfigSchema()
// Validate each field individually using the ConfigField rules
if err := validateFieldWithSchema(schema, "enabled", config.Enabled); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "scan_interval_seconds", int(config.ScanIntervalSeconds)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "worker_timeout_seconds", int(config.WorkerTimeoutSeconds)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "task_timeout_seconds", int(config.TaskTimeoutSeconds)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "retry_delay_seconds", int(config.RetryDelaySeconds)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "max_retries", int(config.MaxRetries)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "cleanup_interval_seconds", int(config.CleanupIntervalSeconds)); err != nil {
return err
}
if err := validateFieldWithSchema(schema, "task_retention_seconds", int(config.TaskRetentionSeconds)); err != nil {
return err
}
// Validate policy fields if present
if config.Policy != nil {
// Note: These field names might need to be adjusted based on the actual schema
if err := validatePolicyField("global_max_concurrent", int(config.Policy.GlobalMaxConcurrent)); err != nil {
return err
}
if err := validatePolicyField("default_repeat_interval_seconds", int(config.Policy.DefaultRepeatIntervalSeconds)); err != nil {
return err
}
if err := validatePolicyField("default_check_interval_seconds", int(config.Policy.DefaultCheckIntervalSeconds)); err != nil {
return err
}
}
return nil
}
// validateFieldWithSchema validates a single field using its ConfigField definition
func validateFieldWithSchema(schema *MaintenanceConfigSchema, fieldName string, value interface{}) error {
field := schema.GetFieldByName(fieldName)
if field == nil {
// Field not in schema, skip validation
return nil
}
return field.ValidateValue(value)
}
// validatePolicyField validates policy fields (simplified validation for now)
func validatePolicyField(fieldName string, value int) error {
switch fieldName {
case "global_max_concurrent":
if value < 1 || value > 20 {
return fmt.Errorf("Global Max Concurrent must be between 1 and 20, got %d", value)
}
case "default_repeat_interval":
if value < 1 || value > 168 {
return fmt.Errorf("Default Repeat Interval must be between 1 and 168 hours, got %d", value)
}
case "default_check_interval":
if value < 1 || value > 168 {
return fmt.Errorf("Default Check Interval must be between 1 and 168 hours, got %d", value)
}
}
return nil
}

22
weed/admin/maintenance/maintenance_queue.go
View File

@@ -1055,28 +1055,6 @@ func (mq *MaintenanceQueue) getMaxConcurrentForTaskType(taskType MaintenanceTask
return 1
}
// getRunningTasks returns all currently running tasks
func (mq *MaintenanceQueue) getRunningTasks() []*MaintenanceTask {
var runningTasks []*MaintenanceTask
for _, task := range mq.tasks {
if task.Status == TaskStatusAssigned || task.Status == TaskStatusInProgress {
runningTasks = append(runningTasks, task)
}
}
return runningTasks
}
// getAvailableWorkers returns all workers that can take more work
func (mq *MaintenanceQueue) getAvailableWorkers() []*MaintenanceWorker {
var availableWorkers []*MaintenanceWorker
for _, worker := range mq.workers {
if worker.Status == "active" && worker.CurrentLoad < worker.MaxConcurrent {
availableWorkers = append(availableWorkers, worker)
}
}
return availableWorkers
}
// trackPendingOperation adds a task to the pending operations tracker
func (mq *MaintenanceQueue) trackPendingOperation(task *MaintenanceTask) {
if mq.integration == nil {

309
weed/admin/maintenance/maintenance_types.go
View File

@@ -2,15 +2,11 @@ package maintenance
import (
"html/template"
"sort"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// AdminClient interface defines what the maintenance system needs from the admin server
@@ -21,51 +17,6 @@ type AdminClient interface {
// MaintenanceTaskType represents different types of maintenance operations
type MaintenanceTaskType string
// GetRegisteredMaintenanceTaskTypes returns all registered task types as MaintenanceTaskType values
// sorted alphabetically for consistent menu ordering
func GetRegisteredMaintenanceTaskTypes() []MaintenanceTaskType {
typesRegistry := tasks.GetGlobalTypesRegistry()
var taskTypes []MaintenanceTaskType
for workerTaskType := range typesRegistry.GetAllDetectors() {
maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
taskTypes = append(taskTypes, maintenanceTaskType)
}
// Sort task types alphabetically to ensure consistent menu ordering
sort.Slice(taskTypes, func(i, j int) bool {
return string(taskTypes[i]) < string(taskTypes[j])
})
return taskTypes
}
// GetMaintenanceTaskType returns a specific task type if it's registered, or empty string if not found
func GetMaintenanceTaskType(taskTypeName string) MaintenanceTaskType {
typesRegistry := tasks.GetGlobalTypesRegistry()
for workerTaskType := range typesRegistry.GetAllDetectors() {
if string(workerTaskType) == taskTypeName {
return MaintenanceTaskType(taskTypeName)
}
}
return MaintenanceTaskType("")
}
// IsMaintenanceTaskTypeRegistered checks if a task type is registered
func IsMaintenanceTaskTypeRegistered(taskType MaintenanceTaskType) bool {
typesRegistry := tasks.GetGlobalTypesRegistry()
for workerTaskType := range typesRegistry.GetAllDetectors() {
if string(workerTaskType) == string(taskType) {
return true
}
}
return false
}
// MaintenanceTaskPriority represents task execution priority
type MaintenanceTaskPriority int
@@ -200,14 +151,6 @@ func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPol
return mp.TaskPolicies[string(taskType)]
}
// SetTaskPolicy sets the policy for a specific task type
func SetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType, policy *TaskPolicy) {
if mp.TaskPolicies == nil {
mp.TaskPolicies = make(map[string]*TaskPolicy)
}
mp.TaskPolicies[string(taskType)] = policy
}
// IsTaskEnabled returns whether a task type is enabled
func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
policy := GetTaskPolicy(mp, taskType)
@@ -235,84 +178,6 @@ func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int
return int(policy.RepeatIntervalSeconds)
}
// GetVacuumTaskConfig returns the vacuum task configuration
func GetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.VacuumTaskConfig {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return nil
}
return policy.GetVacuumConfig()
}
// GetErasureCodingTaskConfig returns the erasure coding task configuration
func GetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ErasureCodingTaskConfig {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return nil
}
return policy.GetErasureCodingConfig()
}
// GetBalanceTaskConfig returns the balance task configuration
func GetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.BalanceTaskConfig {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return nil
}
return policy.GetBalanceConfig()
}
// GetReplicationTaskConfig returns the replication task configuration
func GetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType) *worker_pb.ReplicationTaskConfig {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return nil
}
return policy.GetReplicationConfig()
}
// Note: GetTaskConfig was removed - use typed getters: GetVacuumTaskConfig, GetErasureCodingTaskConfig, GetBalanceTaskConfig, or GetReplicationTaskConfig
// SetVacuumTaskConfig sets the vacuum task configuration
func SetVacuumTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.VacuumTaskConfig) {
policy := GetTaskPolicy(mp, taskType)
if policy != nil {
policy.TaskConfig = &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: config,
}
}
}
// SetErasureCodingTaskConfig sets the erasure coding task configuration
func SetErasureCodingTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ErasureCodingTaskConfig) {
policy := GetTaskPolicy(mp, taskType)
if policy != nil {
policy.TaskConfig = &worker_pb.TaskPolicy_ErasureCodingConfig{
ErasureCodingConfig: config,
}
}
}
// SetBalanceTaskConfig sets the balance task configuration
func SetBalanceTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.BalanceTaskConfig) {
policy := GetTaskPolicy(mp, taskType)
if policy != nil {
policy.TaskConfig = &worker_pb.TaskPolicy_BalanceConfig{
BalanceConfig: config,
}
}
}
// SetReplicationTaskConfig sets the replication task configuration
func SetReplicationTaskConfig(mp *MaintenancePolicy, taskType MaintenanceTaskType, config *worker_pb.ReplicationTaskConfig) {
policy := GetTaskPolicy(mp, taskType)
if policy != nil {
policy.TaskConfig = &worker_pb.TaskPolicy_ReplicationConfig{
ReplicationConfig: config,
}
}
}
// SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
// Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig
@@ -475,180 +340,6 @@ type ClusterReplicationTask struct {
Metadata map[string]string `json:"metadata,omitempty"`
}
// BuildMaintenancePolicyFromTasks creates a maintenance policy with configurations
// from all registered tasks using their UI providers
func BuildMaintenancePolicyFromTasks() *MaintenancePolicy {
policy := &MaintenancePolicy{
TaskPolicies: make(map[string]*TaskPolicy),
GlobalMaxConcurrent: 4,
DefaultRepeatIntervalSeconds: 6 * 3600, // 6 hours in seconds
DefaultCheckIntervalSeconds: 12 * 3600, // 12 hours in seconds
}
// Get all registered task types from the UI registry
uiRegistry := tasks.GetGlobalUIRegistry()
typesRegistry := tasks.GetGlobalTypesRegistry()
for taskType, provider := range uiRegistry.GetAllProviders() {
// Convert task type to maintenance task type
maintenanceTaskType := MaintenanceTaskType(string(taskType))
// Get the default configuration from the UI provider
defaultConfig := provider.GetCurrentConfig()
// Create task policy from UI configuration
taskPolicy := &TaskPolicy{
Enabled: true, // Default enabled
MaxConcurrent: 2, // Default concurrency
RepeatIntervalSeconds: policy.DefaultRepeatIntervalSeconds,
CheckIntervalSeconds: policy.DefaultCheckIntervalSeconds,
}
// Extract configuration using TaskConfig interface - no more map conversions!
if taskConfig, ok := defaultConfig.(interface{ ToTaskPolicy() *worker_pb.TaskPolicy }); ok {
// Use protobuf directly for clean, type-safe config extraction
pbTaskPolicy := taskConfig.ToTaskPolicy()
taskPolicy.Enabled = pbTaskPolicy.Enabled
taskPolicy.MaxConcurrent = pbTaskPolicy.MaxConcurrent
if pbTaskPolicy.RepeatIntervalSeconds > 0 {
taskPolicy.RepeatIntervalSeconds = pbTaskPolicy.RepeatIntervalSeconds
}
if pbTaskPolicy.CheckIntervalSeconds > 0 {
taskPolicy.CheckIntervalSeconds = pbTaskPolicy.CheckIntervalSeconds
}
}
// Also get defaults from scheduler if available (using types.TaskScheduler explicitly)
var scheduler types.TaskScheduler = typesRegistry.GetScheduler(taskType)
if scheduler != nil {
if taskPolicy.MaxConcurrent <= 0 {
taskPolicy.MaxConcurrent = int32(scheduler.GetMaxConcurrent())
}
// Convert default repeat interval to seconds
if repeatInterval := scheduler.GetDefaultRepeatInterval(); repeatInterval > 0 {
taskPolicy.RepeatIntervalSeconds = int32(repeatInterval.Seconds())
}
}
// Also get defaults from detector if available (using types.TaskDetector explicitly)
var detector types.TaskDetector = typesRegistry.GetDetector(taskType)
if detector != nil {
// Convert scan interval to check interval (seconds)
if scanInterval := detector.ScanInterval(); scanInterval > 0 {
taskPolicy.CheckIntervalSeconds = int32(scanInterval.Seconds())
}
}
policy.TaskPolicies[string(maintenanceTaskType)] = taskPolicy
glog.V(3).Infof("Built policy for task type %s: enabled=%v, max_concurrent=%d",
maintenanceTaskType, taskPolicy.Enabled, taskPolicy.MaxConcurrent)
}
glog.V(2).Infof("Built maintenance policy with %d task configurations", len(policy.TaskPolicies))
return policy
}
// SetPolicyFromTasks sets the maintenance policy from registered tasks
func SetPolicyFromTasks(policy *MaintenancePolicy) {
if policy == nil {
return
}
// Build new policy from tasks
newPolicy := BuildMaintenancePolicyFromTasks()
// Copy task policies
policy.TaskPolicies = newPolicy.TaskPolicies
glog.V(1).Infof("Updated maintenance policy with %d task configurations from registered tasks", len(policy.TaskPolicies))
}
// GetTaskIcon returns the icon CSS class for a task type from its UI provider
func GetTaskIcon(taskType MaintenanceTaskType) string {
typesRegistry := tasks.GetGlobalTypesRegistry()
uiRegistry := tasks.GetGlobalUIRegistry()
// Convert MaintenanceTaskType to TaskType
for workerTaskType := range typesRegistry.GetAllDetectors() {
if string(workerTaskType) == string(taskType) {
// Get the UI provider for this task type
provider := uiRegistry.GetProvider(workerTaskType)
if provider != nil {
return provider.GetIcon()
}
break
}
}
// Default icon if no UI provider found
return "fas fa-cog text-muted"
}
// GetTaskDisplayName returns the display name for a task type from its UI provider
func GetTaskDisplayName(taskType MaintenanceTaskType) string {
typesRegistry := tasks.GetGlobalTypesRegistry()
uiRegistry := tasks.GetGlobalUIRegistry()
// Convert MaintenanceTaskType to TaskType
for workerTaskType := range typesRegistry.GetAllDetectors() {
if string(workerTaskType) == string(taskType) {
// Get the UI provider for this task type
provider := uiRegistry.GetProvider(workerTaskType)
if provider != nil {
return provider.GetDisplayName()
}
break
}
}
// Fallback to the task type string
return string(taskType)
}
// GetTaskDescription returns the description for a task type from its UI provider
func GetTaskDescription(taskType MaintenanceTaskType) string {
typesRegistry := tasks.GetGlobalTypesRegistry()
uiRegistry := tasks.GetGlobalUIRegistry()
// Convert MaintenanceTaskType to TaskType
for workerTaskType := range typesRegistry.GetAllDetectors() {
if string(workerTaskType) == string(taskType) {
// Get the UI provider for this task type
provider := uiRegistry.GetProvider(workerTaskType)
if provider != nil {
return provider.GetDescription()
}
break
}
}
// Fallback to a generic description
return "Configure detailed settings for " + string(taskType) + " tasks."
}
// BuildMaintenanceMenuItems creates menu items for all registered task types
func BuildMaintenanceMenuItems() []*MaintenanceMenuItem {
var menuItems []*MaintenanceMenuItem
// Get all registered task types
registeredTypes := GetRegisteredMaintenanceTaskTypes()
for _, taskType := range registeredTypes {
menuItem := &MaintenanceMenuItem{
TaskType: taskType,
DisplayName: GetTaskDisplayName(taskType),
Description: GetTaskDescription(taskType),
Icon: GetTaskIcon(taskType),
IsEnabled: IsMaintenanceTaskTypeRegistered(taskType),
Path: "/maintenance/config/" + string(taskType),
}
menuItems = append(menuItems, menuItem)
}
return menuItems
}
// Helper functions to extract configuration fields
// Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField

421
weed/admin/maintenance/maintenance_worker.go
View File

@@ -1,421 +0,0 @@
package maintenance
import (
"context"
"fmt"
"os"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/worker"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
// Import task packages to trigger their auto-registration
_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
_ "github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
)
// MaintenanceWorkerService manages maintenance task execution
// TaskExecutor defines the function signature for task execution
type TaskExecutor func(*MaintenanceWorkerService, *MaintenanceTask) error
// TaskExecutorFactory creates a task executor for a given worker service
type TaskExecutorFactory func() TaskExecutor
// Global registry for task executor factories
var taskExecutorFactories = make(map[MaintenanceTaskType]TaskExecutorFactory)
var executorRegistryMutex sync.RWMutex
var executorRegistryInitOnce sync.Once
// initializeExecutorFactories dynamically registers executor factories for all auto-registered task types
func initializeExecutorFactories() {
executorRegistryInitOnce.Do(func() {
// Get all registered task types from the global registry
typesRegistry := tasks.GetGlobalTypesRegistry()
var taskTypes []MaintenanceTaskType
for workerTaskType := range typesRegistry.GetAllDetectors() {
// Convert types.TaskType to MaintenanceTaskType by string conversion
maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
taskTypes = append(taskTypes, maintenanceTaskType)
}
// Register generic executor for all task types
for _, taskType := range taskTypes {
RegisterTaskExecutorFactory(taskType, createGenericTaskExecutor)
}
glog.V(1).Infof("Dynamically registered generic task executor for %d task types: %v", len(taskTypes), taskTypes)
})
}
// RegisterTaskExecutorFactory registers a factory function for creating task executors
func RegisterTaskExecutorFactory(taskType MaintenanceTaskType, factory TaskExecutorFactory) {
executorRegistryMutex.Lock()
defer executorRegistryMutex.Unlock()
taskExecutorFactories[taskType] = factory
glog.V(2).Infof("Registered executor factory for task type: %s", taskType)
}
// GetTaskExecutorFactory returns the factory for a task type
func GetTaskExecutorFactory(taskType MaintenanceTaskType) (TaskExecutorFactory, bool) {
// Ensure executor factories are initialized
initializeExecutorFactories()
executorRegistryMutex.RLock()
defer executorRegistryMutex.RUnlock()
factory, exists := taskExecutorFactories[taskType]
return factory, exists
}
// GetSupportedExecutorTaskTypes returns all task types with registered executor factories
func GetSupportedExecutorTaskTypes() []MaintenanceTaskType {
// Ensure executor factories are initialized
initializeExecutorFactories()
executorRegistryMutex.RLock()
defer executorRegistryMutex.RUnlock()
taskTypes := make([]MaintenanceTaskType, 0, len(taskExecutorFactories))
for taskType := range taskExecutorFactories {
taskTypes = append(taskTypes, taskType)
}
return taskTypes
}
// createGenericTaskExecutor creates a generic task executor that uses the task registry
func createGenericTaskExecutor() TaskExecutor {
return func(mws *MaintenanceWorkerService, task *MaintenanceTask) error {
return mws.executeGenericTask(task)
}
}
// init does minimal initialization - actual registration happens lazily
func init() {
// Executor factory registration will happen lazily when first accessed
glog.V(1).Infof("Maintenance worker initialized - executor factories will be registered on first access")
}
type MaintenanceWorkerService struct {
workerID string
address string
adminServer string
capabilities []MaintenanceTaskType
maxConcurrent int
currentTasks map[string]*MaintenanceTask
queue *MaintenanceQueue
adminClient AdminClient
running bool
stopChan chan struct{}
// Task execution registry
taskExecutors map[MaintenanceTaskType]TaskExecutor
// Task registry for creating task instances
taskRegistry *tasks.TaskRegistry
}
// NewMaintenanceWorkerService creates a new maintenance worker service
func NewMaintenanceWorkerService(workerID, address, adminServer string) *MaintenanceWorkerService {
// Get all registered maintenance task types dynamically
capabilities := GetRegisteredMaintenanceTaskTypes()
worker := &MaintenanceWorkerService{
workerID: workerID,
address: address,
adminServer: adminServer,
capabilities: capabilities,
maxConcurrent: 2, // Default concurrent task limit
currentTasks: make(map[string]*MaintenanceTask),
stopChan: make(chan struct{}),
taskExecutors: make(map[MaintenanceTaskType]TaskExecutor),
taskRegistry: tasks.GetGlobalTaskRegistry(), // Use global registry with auto-registered tasks
}
// Initialize task executor registry
worker.initializeTaskExecutors()
glog.V(1).Infof("Created maintenance worker with %d registered task types", len(worker.taskRegistry.GetAll()))
return worker
}
// executeGenericTask executes a task using the task registry instead of hardcoded methods
func (mws *MaintenanceWorkerService) executeGenericTask(task *MaintenanceTask) error {
glog.V(2).Infof("Executing generic task %s: %s for volume %d", task.ID, task.Type, task.VolumeID)
// Validate that task has proper typed parameters
if task.TypedParams == nil {
return fmt.Errorf("task %s has no typed parameters - task was not properly planned (insufficient destinations)", task.ID)
}
// Convert MaintenanceTask to types.TaskType
taskType := types.TaskType(string(task.Type))
// Create task instance using the registry
taskInstance, err := mws.taskRegistry.Get(taskType).Create(task.TypedParams)
if err != nil {
return fmt.Errorf("failed to create task instance: %w", err)
}
// Update progress to show task has started
mws.updateTaskProgress(task.ID, 5)
// Execute the task
err = taskInstance.Execute(context.Background(), task.TypedParams)
if err != nil {
return fmt.Errorf("task execution failed: %w", err)
}
// Update progress to show completion
mws.updateTaskProgress(task.ID, 100)
glog.V(2).Infof("Generic task %s completed successfully", task.ID)
return nil
}
// initializeTaskExecutors sets up the task execution registry dynamically
func (mws *MaintenanceWorkerService) initializeTaskExecutors() {
mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
// Get all registered executor factories and create executors
executorRegistryMutex.RLock()
defer executorRegistryMutex.RUnlock()
for taskType, factory := range taskExecutorFactories {
executor := factory()
mws.taskExecutors[taskType] = executor
glog.V(3).Infof("Initialized executor for task type: %s", taskType)
}
glog.V(2).Infof("Initialized %d task executors", len(mws.taskExecutors))
}
// RegisterTaskExecutor allows dynamic registration of new task executors
func (mws *MaintenanceWorkerService) RegisterTaskExecutor(taskType MaintenanceTaskType, executor TaskExecutor) {
if mws.taskExecutors == nil {
mws.taskExecutors = make(map[MaintenanceTaskType]TaskExecutor)
}
mws.taskExecutors[taskType] = executor
glog.V(1).Infof("Registered executor for task type: %s", taskType)
}
// GetSupportedTaskTypes returns all task types that this worker can execute
func (mws *MaintenanceWorkerService) GetSupportedTaskTypes() []MaintenanceTaskType {
return GetSupportedExecutorTaskTypes()
}
// Start begins the worker service
func (mws *MaintenanceWorkerService) Start() error {
mws.running = true
// Register with admin server
worker := &MaintenanceWorker{
ID: mws.workerID,
Address: mws.address,
Capabilities: mws.capabilities,
MaxConcurrent: mws.maxConcurrent,
}
if mws.queue != nil {
mws.queue.RegisterWorker(worker)
}
// Start worker loop
go mws.workerLoop()
glog.Infof("Maintenance worker %s started at %s", mws.workerID, mws.address)
return nil
}
// Stop terminates the worker service
func (mws *MaintenanceWorkerService) Stop() {
mws.running = false
close(mws.stopChan)
// Wait for current tasks to complete or timeout
timeout := time.NewTimer(30 * time.Second)
defer timeout.Stop()
for len(mws.currentTasks) > 0 {
select {
case <-timeout.C:
glog.Warningf("Worker %s stopping with %d tasks still running", mws.workerID, len(mws.currentTasks))
return
case <-time.After(time.Second):
// Check again
}
}
glog.Infof("Maintenance worker %s stopped", mws.workerID)
}
// workerLoop is the main worker event loop
func (mws *MaintenanceWorkerService) workerLoop() {
heartbeatTicker := time.NewTicker(30 * time.Second)
defer heartbeatTicker.Stop()
taskRequestTicker := time.NewTicker(5 * time.Second)
defer taskRequestTicker.Stop()
for mws.running {
select {
case <-mws.stopChan:
return
case <-heartbeatTicker.C:
mws.sendHeartbeat()
case <-taskRequestTicker.C:
mws.requestTasks()
}
}
}
// sendHeartbeat sends heartbeat to admin server
func (mws *MaintenanceWorkerService) sendHeartbeat() {
if mws.queue != nil {
mws.queue.UpdateWorkerHeartbeat(mws.workerID)
}
}
// requestTasks requests new tasks from the admin server
func (mws *MaintenanceWorkerService) requestTasks() {
if len(mws.currentTasks) >= mws.maxConcurrent {
return // Already at capacity
}
if mws.queue != nil {
task := mws.queue.GetNextTask(mws.workerID, mws.capabilities)
if task != nil {
mws.executeTask(task)
}
}
}
// executeTask executes a maintenance task
func (mws *MaintenanceWorkerService) executeTask(task *MaintenanceTask) {
mws.currentTasks[task.ID] = task
go func() {
defer func() {
delete(mws.currentTasks, task.ID)
}()
glog.Infof("Worker %s executing task %s: %s", mws.workerID, task.ID, task.Type)
// Execute task using dynamic executor registry
var err error
if executor, exists := mws.taskExecutors[task.Type]; exists {
err = executor(mws, task)
} else {
err = fmt.Errorf("unsupported task type: %s", task.Type)
glog.Errorf("No executor registered for task type: %s", task.Type)
}
// Report task completion
if mws.queue != nil {
errorMsg := ""
if err != nil {
errorMsg = err.Error()
}
mws.queue.CompleteTask(task.ID, errorMsg)
}
if err != nil {
glog.Errorf("Worker %s failed to execute task %s: %v", mws.workerID, task.ID, err)
} else {
glog.Infof("Worker %s completed task %s successfully", mws.workerID, task.ID)
}
}()
}
// updateTaskProgress updates the progress of a task
func (mws *MaintenanceWorkerService) updateTaskProgress(taskID string, progress float64) {
if mws.queue != nil {
mws.queue.UpdateTaskProgress(taskID, progress)
}
}
// GetStatus returns the current status of the worker
func (mws *MaintenanceWorkerService) GetStatus() map[string]interface{} {
return map[string]interface{}{
"worker_id": mws.workerID,
"address": mws.address,
"running": mws.running,
"capabilities": mws.capabilities,
"max_concurrent": mws.maxConcurrent,
"current_tasks": len(mws.currentTasks),
"task_details": mws.currentTasks,
}
}
// SetQueue sets the maintenance queue for the worker
func (mws *MaintenanceWorkerService) SetQueue(queue *MaintenanceQueue) {
mws.queue = queue
}
// SetAdminClient sets the admin client for the worker
func (mws *MaintenanceWorkerService) SetAdminClient(client AdminClient) {
mws.adminClient = client
}
// SetCapabilities sets the worker capabilities
func (mws *MaintenanceWorkerService) SetCapabilities(capabilities []MaintenanceTaskType) {
mws.capabilities = capabilities
}
// SetMaxConcurrent sets the maximum concurrent tasks
func (mws *MaintenanceWorkerService) SetMaxConcurrent(max int) {
mws.maxConcurrent = max
}
// SetHeartbeatInterval sets the heartbeat interval (placeholder for future use)
func (mws *MaintenanceWorkerService) SetHeartbeatInterval(interval time.Duration) {
// Future implementation for configurable heartbeat
}
// SetTaskRequestInterval sets the task request interval (placeholder for future use)
func (mws *MaintenanceWorkerService) SetTaskRequestInterval(interval time.Duration) {
// Future implementation for configurable task requests
}
// MaintenanceWorkerCommand represents a standalone maintenance worker command
type MaintenanceWorkerCommand struct {
workerService *MaintenanceWorkerService
}
// NewMaintenanceWorkerCommand creates a new worker command
func NewMaintenanceWorkerCommand(workerID, address, adminServer string) *MaintenanceWorkerCommand {
return &MaintenanceWorkerCommand{
workerService: NewMaintenanceWorkerService(workerID, address, adminServer),
}
}
// Run starts the maintenance worker as a standalone service
func (mwc *MaintenanceWorkerCommand) Run() error {
// Generate or load persistent worker ID if not provided
if mwc.workerService.workerID == "" {
// Get current working directory for worker ID persistence
wd, err := os.Getwd()
if err != nil {
return fmt.Errorf("failed to get working directory: %w", err)
}
workerID, err := worker.GenerateOrLoadWorkerID(wd)
if err != nil {
return fmt.Errorf("failed to generate or load worker ID: %w", err)
}
mwc.workerService.workerID = workerID
}
// Start the worker service
err := mwc.workerService.Start()
if err != nil {
return fmt.Errorf("failed to start maintenance worker: %w", err)
}
// Wait for interrupt signal
select {}
}

12
weed/admin/plugin/plugin.go
View File

@@ -122,6 +122,7 @@ type Plugin struct {
type streamSession struct {
workerID string
outgoing chan *plugin_pb.AdminToWorkerMessage
done chan struct{}
closeOnce sync.Once
}
@@ -274,6 +275,7 @@ func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStream
session := &streamSession{
workerID: workerID,
outgoing: make(chan *plugin_pb.AdminToWorkerMessage, r.outgoingBuffer),
done: make(chan struct{}),
}
r.putSession(session)
defer r.cleanupSession(workerID)
@@ -908,8 +910,10 @@ func (r *Plugin) sendLoop(
return nil
case <-r.shutdownCh:
return nil
case msg, ok := <-session.outgoing:
if !ok {
case <-session.done:
return nil
case msg := <-session.outgoing:
if msg == nil {
return nil
}
if err := stream.Send(msg); err != nil {
@@ -930,6 +934,8 @@ func (r *Plugin) sendToWorker(workerID string, message *plugin_pb.AdminToWorkerM
select {
case <-r.shutdownCh:
return fmt.Errorf("plugin is shutting down")
case <-session.done:
return fmt.Errorf("worker %s session is closed", workerID)
case session.outgoing <- message:
return nil
case <-time.After(r.sendTimeout):
@@ -1425,7 +1431,7 @@ func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugi
func (s *streamSession) close() {
s.closeOnce.Do(func() {
close(s.outgoing)
close(s.done)
})
}

12
weed/admin/plugin/plugin_cancel_test.go
View File

@@ -26,7 +26,7 @@ func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
pluginSvc.putSession(session)
ctx, cancel := context.WithCancel(context.Background())
@@ -77,7 +77,7 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
{JobType: jobType, CanExecute: true, MaxExecutionConcurrency: 1},
},
})
session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4)}
session := &streamSession{workerID: workerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 4), done: make(chan struct{})}
pluginSvc.putSession(session)
job := &plugin_pb.JobSpec{JobId: "job-1", JobType: jobType}
@@ -135,8 +135,8 @@ func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)
@@ -214,8 +214,8 @@ func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8), done: make(chan struct{})}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)

8
weed/admin/plugin/plugin_detection_test.go
View File

@@ -22,7 +22,7 @@ func TestRunDetectionIncludesLatestSuccessfulRun(t *testing.T) {
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
pluginSvc.putSession(session)
oldSuccess := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -80,7 +80,7 @@ func TestRunDetectionOmitsLastSuccessfulRunWhenNoSuccessHistory(t *testing.T) {
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
pluginSvc.putSession(session)
if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
@@ -130,7 +130,7 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
session := &streamSession{workerID: "worker-a", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
pluginSvc.putSession(session)
reportCh := make(chan *DetectionReport, 1)
@@ -210,7 +210,7 @@ func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1), done: make(chan struct{})}
pluginSvc.putSession(session)
successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)

86
weed/admin/plugin/plugin_scheduler.go
View File

@@ -95,16 +95,6 @@ func (r *Plugin) laneSchedulerLoop(ls *schedulerLaneState) {
}
}
// schedulerLoop is kept for backward compatibility; it delegates to
// laneSchedulerLoop with the default lane. New code should not call this.
func (r *Plugin) schedulerLoop() {
ls := r.lanes[LaneDefault]
if ls == nil {
ls = newLaneState(LaneDefault)
}
r.laneSchedulerLoop(ls)
}
// runLaneSchedulerIteration runs one scheduling pass for a single lane,
// processing only the job types assigned to that lane.
//
@@ -229,82 +219,6 @@ func (r *Plugin) runLaneSchedulerIterationConcurrent(ls *schedulerLaneState, job
return hadJobs.Load()
}
// runSchedulerIteration is kept for backward compatibility. It runs a
// single iteration across ALL job types (equivalent to the old single-loop
// behavior). It is only used by the legacy schedulerLoop() fallback.
func (r *Plugin) runSchedulerIteration() bool {
ls := r.lanes[LaneDefault]
if ls == nil {
ls = newLaneState(LaneDefault)
}
// For backward compat, the old function processes all job types.
r.expireStaleJobs(time.Now().UTC())
jobTypes := r.registry.DetectableJobTypes()
if len(jobTypes) == 0 {
r.setSchedulerLoopState("", "idle")
return false
}
r.setSchedulerLoopState("", "waiting_for_lock")
releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
if err != nil {
glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
r.setSchedulerLoopState("", "idle")
return false
}
if releaseLock != nil {
defer releaseLock()
}
active := make(map[string]struct{}, len(jobTypes))
hadJobs := false
for _, jobType := range jobTypes {
active[jobType] = struct{}{}
policy, enabled, err := r.loadSchedulerPolicy(jobType)
if err != nil {
glog.Warningf("Plugin scheduler failed to load policy for %s: %v", jobType, err)
continue
}
if !enabled {
r.clearSchedulerJobType(jobType)
continue
}
initialDelay := time.Duration(0)
if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
initialDelay = 5 * time.Second
}
if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
continue
}
detected := r.runJobTypeIteration(jobType, policy)
if detected {
hadJobs = true
}
}
r.pruneSchedulerState(active)
r.pruneDetectorLeases(active)
r.setSchedulerLoopState("", "idle")
return hadJobs
}
// wakeLane wakes the scheduler goroutine for a specific lane.
func (r *Plugin) wakeLane(lane SchedulerLane) {
if r == nil {
return
}
if ls, ok := r.lanes[lane]; ok {
select {
case ls.wakeCh <- struct{}{}:
default:
}
}
}
// wakeAllLanes wakes all lane scheduler goroutines.
func (r *Plugin) wakeAllLanes() {
if r == nil {

10
weed/admin/plugin/scheduler_status.go
View File

@@ -210,16 +210,6 @@ func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
}
}
func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
if r == nil {
return
}
r.schedulerLoopMu.Lock()
r.schedulerLoopState.lastIterationHadJobs = hadJobs
r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
r.schedulerLoopMu.Unlock()
}
func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
if r == nil {
return schedulerLoopState{}

14
weed/admin/view/app/template_helpers.go
View File

@@ -6,20 +6,6 @@ import (
"strings"
)
// getStatusColor returns Bootstrap color class for status
func getStatusColor(status string) string {
switch status {
case "active", "healthy":
return "success"
case "warning":
return "warning"
case "critical", "unreachable":
return "danger"
default:
return "secondary"
}
}
// formatBytes converts bytes to human readable format
func formatBytes(bytes int64) string {
if bytes == 0 {