Expire stuck plugin jobs (#8492)

* Add stale job expiry and expire API

* Add expire job button

* Add test hook and coverage for ExpirePluginJobAPI

* Document scheduler filtering side effect and reuse helper

* Restore job spec proposal test

* Regenerate plugin template output

---------

Co-authored-by: Copilot <copilot@github.com>

weed/admin/plugin/plugin_scheduler.go

@@ -61,6 +61,8 @@ func (r *Plugin) schedulerLoop() {
 }
 
 func (r *Plugin) runSchedulerTick() {
+	r.expireStaleJobs(time.Now().UTC())
+
 	jobTypes := r.registry.DetectableJobTypes()
 	if len(jobTypes) == 0 {
 		return
@@ -839,11 +841,16 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
 	}
 }
 
+// filterProposalsWithActiveJobs removes proposals whose dedupe keys already have active jobs.
+// It first expires stale tracked jobs via expireStaleJobs, which can mutate scheduler state,
+// so callers should treat this method as a stateful operation.
 func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) {
 	if len(proposals) == 0 {
 		return proposals, 0
 	}
 
+	r.expireStaleJobs(time.Now().UTC())
+
 	activeKeys := make(map[string]struct{})
 	r.jobsMu.RLock()
 	for _, job := range r.jobs {