Files
seaweedFS/weed/admin/plugin/config_store_test.go
Chris Lu 8ec9ff4a12 Refactor plugin system and migrate worker runtime (#8369)
* admin: add plugin runtime UI page and route wiring

* pb: add plugin gRPC contract and generated bindings

* admin/plugin: implement worker registry, runtime, monitoring, and config store

* admin/dash: wire plugin runtime and expose plugin workflow APIs

* command: add flags to enable plugin runtime

* admin: rename remaining plugin v2 wording to plugin

* admin/plugin: add detectable job type registry helper

* admin/plugin: add scheduled detection and dispatch orchestration

* admin/plugin: prefetch job type descriptors when workers connect

* admin/plugin: add known job type discovery API and UI

* admin/plugin: refresh design doc to match current implementation

* admin/plugin: enforce per-worker scheduler concurrency limits

* admin/plugin: use descriptor runtime defaults for scheduler policy

* admin/ui: auto-load first known plugin job type on page open

* admin/plugin: bootstrap persisted config from descriptor defaults

* admin/plugin: dedupe scheduled proposals by dedupe key

* admin/ui: add job type and state filters for plugin monitoring

* admin/ui: add per-job-type plugin activity summary

* admin/plugin: split descriptor read API from schema refresh

* admin/ui: keep plugin summary metrics global while tables are filtered

* admin/plugin: retry executor reservation before timing out

* admin/plugin: expose scheduler states for monitoring

* admin/ui: show per-job-type scheduler states in plugin monitor

* pb/plugin: rename protobuf package to plugin

* admin/plugin: rename pluginRuntime wiring to plugin

* admin/plugin: remove runtime naming from plugin APIs and UI

* admin/plugin: rename runtime files to plugin naming

* admin/plugin: persist jobs and activities for monitor recovery

* admin/plugin: lease one detector worker per job type

* admin/ui: show worker load from plugin heartbeats

* admin/plugin: skip stale workers for detector and executor picks

* plugin/worker: add plugin worker command and stream runtime scaffold

* plugin/worker: implement vacuum detect and execute handlers

* admin/plugin: document external vacuum plugin worker starter

* command: update plugin.worker help to reflect implemented flow

* command/admin: drop legacy Plugin V2 label

* plugin/worker: validate vacuum job type and respect min interval

* plugin/worker: test no-op detect when min interval not elapsed

* command/admin: document plugin.worker external process

* plugin/worker: advertise configured concurrency in hello

* command/plugin.worker: add jobType handler selection

* command/plugin.worker: test handler selection by job type

* command/plugin.worker: persist worker id in workingDir

* admin/plugin: document plugin.worker jobType and workingDir flags

* plugin/worker: support cancel request for in-flight work

* plugin/worker: test cancel request acknowledgements

* command/plugin.worker: document workingDir and jobType behavior

* plugin/worker: emit executor activity events for monitor

* plugin/worker: test executor activity builder

* admin/plugin: send last successful run in detection request

* admin/plugin: send cancel request when detect or execute context ends

* admin/plugin: document worker cancel request responsibility

* admin/handlers: expose plugin scheduler states API in no-auth mode

* admin/handlers: test plugin scheduler states route registration

* admin/plugin: keep worker id on worker-generated activity records

* admin/plugin: test worker id propagation in monitor activities

* admin/dash: always initialize plugin service

* command/admin: remove plugin enable flags and default to enabled

* admin/dash: drop pluginEnabled constructor parameter

* admin/plugin UI: stop checking plugin enabled state

* admin/plugin: remove docs for plugin enable flags

* admin/dash: remove unused plugin enabled check method

* admin/dash: fallback to in-memory plugin init when dataDir fails

* admin/plugin API: expose worker gRPC port in status

* command/plugin.worker: resolve admin gRPC port via plugin status

* split plugin UI into overview/configuration/monitoring pages

* Update layout_templ.go

* add volume_balance plugin worker handler

* wire plugin.worker CLI for volume_balance job type

* add erasure_coding plugin worker handler

* wire plugin.worker CLI for erasure_coding job type

* support multi-job handlers in plugin worker runtime

* allow plugin.worker jobType as comma-separated list

* admin/plugin UI: rename to Workers and simplify config view

* plugin worker: queue detection requests instead of capacity reject

* Update plugin_worker.go

* plugin volume_balance: remove force_move/timeout from worker config UI

* plugin erasure_coding: enforce local working dir and cleanup

* admin/plugin UI: rename admin settings to job scheduling

* admin/plugin UI: persist and robustly render detection results

* admin/plugin: record and return detection trace metadata

* admin/plugin UI: show detection process and decision trace

* plugin: surface detector decision trace as activities

* mini: start a plugin worker by default

* admin/plugin UI: split monitoring into detection and execution tabs

* plugin worker: emit detection decision trace for EC and balance

* admin workers UI: split monitoring into detection and execution pages

* plugin scheduler: skip proposals for active assigned/running jobs

* admin workers UI: add job queue tab

* plugin worker: add dummy stress detector and executor job type

* admin workers UI: reorder tabs to detection queue execution

* admin workers UI: regenerate plugin template

* plugin defaults: include dummy stress and add stress tests

* plugin dummy stress: rotate detection selections across runs

* plugin scheduler: remove cross-run proposal dedupe

* plugin queue: track pending scheduled jobs

* plugin scheduler: wait for executor capacity before dispatch

* plugin scheduler: skip detection when waiting backlog is high

* plugin: add disk-backed job detail API and persistence

* admin ui: show plugin job detail modal from job id links

* plugin: generate unique job ids instead of reusing proposal ids

* plugin worker: emit heartbeats on work state changes

* plugin registry: round-robin tied executor and detector picks

* add temporary EC overnight stress runner

* plugin job details: persist and render EC execution plans

* ec volume details: color data and parity shard badges

* shard labels: keep parity ids numeric and color-only distinction

* admin: remove legacy maintenance UI routes and templates

* admin: remove dead maintenance endpoint helpers

* Update layout_templ.go

* remove dummy_stress worker and command support

* refactor plugin UI to job-type top tabs and sub-tabs

* migrate weed worker command to plugin runtime

* remove plugin.worker command and keep worker runtime with metrics

* update helm worker args for jobType and execution flags

* set plugin scheduling defaults to global 16 and per-worker 4

* stress: fix RPC context reuse and remove redundant variables in ec_stress_runner

* admin/plugin: fix lifecycle races, safe channel operations, and terminal state constants

* admin/dash: randomize job IDs and fix priority zero-value overwrite in plugin API

* admin/handlers: implement buffered rendering to prevent response corruption

* admin/plugin: implement debounced persistence flusher and optimize BuildJobDetail memory lookups

* admin/plugin: fix priority overwrite and implement bounded wait in scheduler reserve

* admin/plugin: implement atomic file writes and fix run record side effects

* admin/plugin: use P prefix for parity shard labels in execution plans

* admin/plugin: enable parallel execution for cancellation tests

* admin: refactor time.Time fields to pointers for better JSON omitempty support

* admin/plugin: implement pointer-safe time assignments and comparisons in plugin core

* admin/plugin: fix time assignment and sorting logic in plugin monitor after pointer refactor

* admin/plugin: update scheduler activity tracking to use time pointers

* admin/plugin: fix time-based run history trimming after pointer refactor

* admin/dash: fix JobSpec struct literal in plugin API after pointer refactor

* admin/view: add D/P prefixes to EC shard badges for UI consistency

* admin/plugin: use lifecycle-aware context for schema prefetching

* Update ec_volume_details_templ.go

* admin/stress: fix proposal sorting and log volume cleanup errors

* stress: refine ec stress runner with math/rand and collection name

- Added Collection field to VolumeEcShardsDeleteRequest for correct filename construction.
- Replaced crypto/rand with seeded math/rand PRNG for bulk payloads.
- Added documentation for EcMinAge zero-value behavior.
- Added logging for ignored errors in volume/shard deletion.

* admin: return internal server error for plugin store failures

Changed error status code from 400 Bad Request to 500 Internal Server Error for failures in GetPluginJobDetail to correctly reflect server-side errors.

* admin: implement safe channel sends and graceful shutdown sync

- Added sync.WaitGroup to Plugin struct to manage background goroutines.
- Implemented safeSendCh helper using recover() to prevent panics on closed channels.
- Ensured Shutdown() waits for all background operations to complete.

* admin: robustify plugin monitor with nil-safe time and record init

- Standardized nil-safe assignment for *time.Time pointers (CreatedAt, UpdatedAt, CompletedAt).
- Ensured persistJobDetailSnapshot initializes new records correctly if they don't exist on disk.
- Fixed debounced persistence to trigger immediate write on job completion.

* admin: improve scheduler shutdown behavior and logic guards

- Replaced brittle error string matching with explicit r.shutdownCh selection for shutdown detection.
- Removed redundant nil guard in buildScheduledJobSpec.
- Standardized WaitGroup usage for schedulerLoop.

* admin: implement deep copy for job parameters and atomic write fixes

- Implemented deepCopyGenericValue and used it in cloneTrackedJob to prevent shared state.
- Ensured atomicWriteFile creates parent directories before writing.

* admin: remove unreachable branch in shard classification

Removed an unreachable 'totalShards <= 0' check in classifyShardID as dataShards and parityShards are already guarded.

* admin: secure UI links and use canonical shard constants

- Added rel="noopener noreferrer" to external links for security.
- Replaced magic number 14 with erasure_coding.TotalShardsCount.
- Used renderEcShardBadge for missing shard list consistency.

* admin: stabilize plugin tests and fix regressions

- Composed a robust plugin_monitor_test.go to handle asynchronous persistence.
- Updated all time.Time literals to use timeToPtr helper.
- Added explicit Shutdown() calls in tests to synchronize with debounced writes.
- Fixed syntax errors and orphaned struct literals in tests.

* Potential fix for code scanning alert no. 278: Slice memory allocation with excessive size value

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* Potential fix for code scanning alert no. 283: Uncontrolled data used in path expression

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* admin: finalize refinements for error handling, scheduler, and race fixes

- Standardized HTTP 500 status codes for store failures in plugin_api.go.
- Tracked scheduled detection goroutines with sync.WaitGroup for safe shutdown.
- Fixed race condition in safeSendDetectionComplete by extracting channel under lock.
- Implemented deep copy for JobActivity details.
- Used defaultDirPerm constant in atomicWriteFile.

* test(ec): migrate admin dockertest to plugin APIs

* admin/plugin_api: fix RunPluginJobTypeAPI to return 500 for server-side detection/filter errors

* admin/plugin_api: fix ExecutePluginJobAPI to return 500 for job execution failures

* admin/plugin_api: limit parseProtoJSONBody request body to 1MB to prevent unbounded memory usage

* admin/plugin: consolidate regex to package-level validJobTypePattern; add char validation to sanitizeJobID

* admin/plugin: fix racy Shutdown channel close with sync.Once

* admin/plugin: track sendLoop and recv goroutines in WorkerStream with r.wg

* admin/plugin: document writeProtoFiles atomicity — .pb is source of truth, .json is human-readable only

* admin/plugin: extract activityLess helper to deduplicate nil-safe OccurredAt sort comparators

* test/ec: check http.NewRequest errors to prevent nil req panics

* test/ec: replace deprecated ioutil/math/rand, fix stale step comment 5.1→3.1

* plugin(ec): raise default detection and scheduling throughput limits

* topology: include empty disks in volume list and EC capacity fallback

* topology: remove hard 10-task cap for detection planning

* Update ec_volume_details_templ.go

* adjust default

* fix tests

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
2026-02-18 13:42:41 -08:00

258 lines
6.7 KiB
Go

package plugin
import (
"reflect"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestConfigStoreDescriptorRoundTrip(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
store, err := NewConfigStore(tempDir)
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "vacuum",
DisplayName: "Vacuum",
Description: "Vacuum volumes",
DescriptorVersion: 1,
}
if err := store.SaveDescriptor("vacuum", descriptor); err != nil {
t.Fatalf("SaveDescriptor: %v", err)
}
got, err := store.LoadDescriptor("vacuum")
if err != nil {
t.Fatalf("LoadDescriptor: %v", err)
}
if got == nil {
t.Fatalf("LoadDescriptor: nil descriptor")
}
if got.DisplayName != descriptor.DisplayName {
t.Fatalf("unexpected display name: got %q want %q", got.DisplayName, descriptor.DisplayName)
}
}
func TestConfigStoreRunHistoryRetention(t *testing.T) {
t.Parallel()
store, err := NewConfigStore(t.TempDir())
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
base := time.Now().UTC().Add(-24 * time.Hour)
for i := 0; i < 15; i++ {
err := store.AppendRunRecord("balance", &JobRunRecord{
RunID: "s" + time.Duration(i).String(),
JobID: "job-success",
JobType: "balance",
WorkerID: "worker-a",
Outcome: RunOutcomeSuccess,
CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)),
})
if err != nil {
t.Fatalf("AppendRunRecord success[%d]: %v", i, err)
}
}
for i := 0; i < 12; i++ {
err := store.AppendRunRecord("balance", &JobRunRecord{
RunID: "e" + time.Duration(i).String(),
JobID: "job-error",
JobType: "balance",
WorkerID: "worker-b",
Outcome: RunOutcomeError,
CompletedAt: timeToPtr(base.Add(time.Duration(i) * time.Minute)),
})
if err != nil {
t.Fatalf("AppendRunRecord error[%d]: %v", i, err)
}
}
history, err := store.LoadRunHistory("balance")
if err != nil {
t.Fatalf("LoadRunHistory: %v", err)
}
if len(history.SuccessfulRuns) != MaxSuccessfulRunHistory {
t.Fatalf("successful retention mismatch: got %d want %d", len(history.SuccessfulRuns), MaxSuccessfulRunHistory)
}
if len(history.ErrorRuns) != MaxErrorRunHistory {
t.Fatalf("error retention mismatch: got %d want %d", len(history.ErrorRuns), MaxErrorRunHistory)
}
for i := 1; i < len(history.SuccessfulRuns); i++ {
t1 := time.Time{}
if history.SuccessfulRuns[i-1].CompletedAt != nil {
t1 = *history.SuccessfulRuns[i-1].CompletedAt
}
t2 := time.Time{}
if history.SuccessfulRuns[i].CompletedAt != nil {
t2 = *history.SuccessfulRuns[i].CompletedAt
}
if t1.Before(t2) {
t.Fatalf("successful run order not descending at %d", i)
}
}
for i := 1; i < len(history.ErrorRuns); i++ {
t1 := time.Time{}
if history.ErrorRuns[i-1].CompletedAt != nil {
t1 = *history.ErrorRuns[i-1].CompletedAt
}
t2 := time.Time{}
if history.ErrorRuns[i].CompletedAt != nil {
t2 = *history.ErrorRuns[i].CompletedAt
}
if t1.Before(t2) {
t.Fatalf("error run order not descending at %d", i)
}
}
}
func TestConfigStoreListJobTypes(t *testing.T) {
t.Parallel()
store, err := NewConfigStore("")
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
if err := store.SaveDescriptor("vacuum", &plugin_pb.JobTypeDescriptor{JobType: "vacuum"}); err != nil {
t.Fatalf("SaveDescriptor: %v", err)
}
if err := store.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
JobType: "balance",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
}); err != nil {
t.Fatalf("SaveJobTypeConfig: %v", err)
}
if err := store.AppendRunRecord("ec", &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: timeToPtr(time.Now().UTC())}); err != nil {
t.Fatalf("AppendRunRecord: %v", err)
}
got, err := store.ListJobTypes()
if err != nil {
t.Fatalf("ListJobTypes: %v", err)
}
want := []string{"balance", "ec", "vacuum"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected job types: got=%v want=%v", got, want)
}
}
func TestConfigStoreMonitorStateRoundTrip(t *testing.T) {
t.Parallel()
store, err := NewConfigStore(t.TempDir())
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
tracked := []TrackedJob{
{
JobID: "job-1",
JobType: "vacuum",
State: "running",
Progress: 55,
WorkerID: "worker-a",
CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)),
UpdatedAt: timeToPtr(time.Now().UTC().Add(-1 * time.Minute)),
},
}
activities := []JobActivity{
{
JobID: "job-1",
JobType: "vacuum",
Source: "worker_progress",
Message: "processing",
Stage: "running",
OccurredAt: timeToPtr(time.Now().UTC()),
Details: map[string]interface{}{
"step": "scan",
},
},
}
if err := store.SaveTrackedJobs(tracked); err != nil {
t.Fatalf("SaveTrackedJobs: %v", err)
}
if err := store.SaveActivities(activities); err != nil {
t.Fatalf("SaveActivities: %v", err)
}
gotTracked, err := store.LoadTrackedJobs()
if err != nil {
t.Fatalf("LoadTrackedJobs: %v", err)
}
if len(gotTracked) != 1 || gotTracked[0].JobID != tracked[0].JobID {
t.Fatalf("unexpected tracked jobs: %+v", gotTracked)
}
gotActivities, err := store.LoadActivities()
if err != nil {
t.Fatalf("LoadActivities: %v", err)
}
if len(gotActivities) != 1 || gotActivities[0].Message != activities[0].Message {
t.Fatalf("unexpected activities: %+v", gotActivities)
}
if gotActivities[0].Details["step"] != "scan" {
t.Fatalf("unexpected activity details: %+v", gotActivities[0].Details)
}
}
func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
t.Parallel()
store, err := NewConfigStore(t.TempDir())
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
input := TrackedJob{
JobID: "job-detail-1",
JobType: "vacuum",
Summary: "detail summary",
Detail: "detail payload",
CreatedAt: timeToPtr(time.Now().UTC().Add(-2 * time.Minute)),
UpdatedAt: timeToPtr(time.Now().UTC()),
Parameters: map[string]interface{}{
"volume_id": map[string]interface{}{"int64_value": "3"},
},
Labels: map[string]string{
"source": "detector",
},
ResultOutputValues: map[string]interface{}{
"moved": map[string]interface{}{"bool_value": true},
},
}
if err := store.SaveJobDetail(input); err != nil {
t.Fatalf("SaveJobDetail: %v", err)
}
got, err := store.LoadJobDetail(input.JobID)
if err != nil {
t.Fatalf("LoadJobDetail: %v", err)
}
if got == nil {
t.Fatalf("LoadJobDetail returned nil")
}
if got.Detail != input.Detail {
t.Fatalf("unexpected detail: got=%q want=%q", got.Detail, input.Detail)
}
if got.Labels["source"] != "detector" {
t.Fatalf("unexpected labels: %+v", got.Labels)
}
if got.ResultOutputValues == nil {
t.Fatalf("expected result output values")
}
}