Plugin scheduler: sequential iterations with max runtime (#8496)
* pb: add job type max runtime setting * plugin: default job type max runtime * plugin: redesign scheduler loop * admin ui: update scheduler settings * plugin: fix scheduler loop state name * plugin scheduler: restore backlog skip * plugin scheduler: drop legacy detection helper * admin api: require scheduler config body * admin ui: preserve detection interval on save * plugin scheduler: use job context and drain cancels * plugin scheduler: respect detection intervals * plugin scheduler: gate runs and drain queue * ec test: reuse req/resp vars * ec test: add scheduler debug logs * Adjust scheduler idle sleep and initial run delay * Clear pending job queue before scheduler runs * Log next detection time in EC integration test * Improve plugin scheduler debug logging in EC test * Expose scheduler next detection time * Log scheduler next detection time in EC test * Wake scheduler on config or worker updates * Expose scheduler sleep interval in UI * Fix scheduler sleep save value selection * Set scheduler idle sleep default to 613s * Show scheduler next run time in plugin UI --------- Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
Chris Lu
@@ -120,7 +120,7 @@ templ Plugin(page string) {
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
|
||||
<h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
|
||||
<small class="text-muted">Per job type detection schedule and execution limits</small>
|
||||
<small class="text-muted">Sequential scheduler with per-job runtime limits</small>
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
<div class="table-responsive">
|
||||
@@ -131,12 +131,12 @@ templ Plugin(page string) {
|
||||
<th>Enabled</th>
|
||||
<th>Detector</th>
|
||||
<th>In Flight</th>
|
||||
<th>Next Detection</th>
|
||||
<th>Interval</th>
|
||||
<th>Max Runtime</th>
|
||||
<th>Exec Global</th>
|
||||
<th>Exec/Worker</th>
|
||||
<th>Executor Workers</th>
|
||||
<th>Effective Exec</th>
|
||||
<th>Last Run</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="plugin-scheduler-table-body">
|
||||
@@ -148,6 +148,38 @@ templ Plugin(page string) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row mb-4">
|
||||
<div class="col-lg-6 mb-3">
|
||||
<div class="card shadow-sm h-100">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="mb-0"><i class="fas fa-sliders-h me-2"></i>Scheduler Settings</h5>
|
||||
<small class="text-muted">Global</small>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="mb-2">
|
||||
<label class="form-label" for="plugin-scheduler-idle-sleep-overview">Sleep Between Iterations (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-scheduler-idle-sleep-overview" min="0"/>
|
||||
<div class="form-text">Used when no jobs are detected.</div>
|
||||
</div>
|
||||
<button type="button" class="btn btn-outline-primary" id="plugin-save-scheduler-btn-overview">
|
||||
<i class="fas fa-save me-1"></i>Save Scheduler Settings
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6 mb-3">
|
||||
<div class="card shadow-sm h-100">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
|
||||
<small class="text-muted">Scheduler</small>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="h5 mb-1 plugin-scheduler-next-run">-</div>
|
||||
<div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row mb-4">
|
||||
<div class="col-12">
|
||||
<div class="card shadow-sm">
|
||||
@@ -242,14 +274,14 @@ templ Plugin(page string) {
|
||||
<input class="form-check-input" type="checkbox" id="plugin-admin-enabled"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label" for="plugin-admin-detection-interval">Detection Interval (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-admin-detection-interval" min="0"/>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label" for="plugin-admin-detection-timeout">Detection Timeout (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-admin-detection-timeout" min="0"/>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label" for="plugin-admin-max-runtime">Job Type Max Runtime (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-admin-max-runtime" min="0"/>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label" for="plugin-admin-max-results">Max Jobs / Detection</label>
|
||||
<input type="number" class="form-control" id="plugin-admin-max-results" min="0"/>
|
||||
@@ -273,6 +305,33 @@ templ Plugin(page string) {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm mt-3">
|
||||
<div class="card-header">
|
||||
<h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler Settings</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="mb-3">
|
||||
<label class="form-label" for="plugin-scheduler-idle-sleep">Sleep Between Iterations (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-scheduler-idle-sleep" min="0"/>
|
||||
<div class="form-text">Used when no jobs are detected.</div>
|
||||
</div>
|
||||
<button type="button" class="btn btn-outline-primary" id="plugin-save-scheduler-btn">
|
||||
<i class="fas fa-save me-1"></i>Save Scheduler Settings
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm mt-3">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
|
||||
<small class="text-muted">Scheduler</small>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="h5 mb-1 plugin-scheduler-next-run">-</div>
|
||||
<div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -572,6 +631,9 @@ templ Plugin(page string) {
|
||||
jobs: [],
|
||||
activities: [],
|
||||
schedulerStates: [],
|
||||
schedulerStatus: null,
|
||||
schedulerConfig: null,
|
||||
schedulerConfigLoaded: false,
|
||||
allJobs: [],
|
||||
allActivities: [],
|
||||
loadedJobType: '',
|
||||
@@ -1442,8 +1504,8 @@ templ Plugin(page string) {
|
||||
var enabled = !!item.enabled;
|
||||
var inFlight = !!item.detection_in_flight;
|
||||
var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
|
||||
var intervalSeconds = Number(item.detection_interval_seconds || 0);
|
||||
var intervalText = intervalSeconds > 0 ? (String(intervalSeconds) + 's') : '-';
|
||||
var maxRuntimeSeconds = Number(item.job_type_max_runtime_seconds || 0);
|
||||
var maxRuntimeText = maxRuntimeSeconds > 0 ? (String(maxRuntimeSeconds) + 's') : '-';
|
||||
var globalExec = Number(item.global_execution_concurrency || 0);
|
||||
var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
|
||||
var executorWorkers = Number(item.executor_worker_count || 0);
|
||||
@@ -1452,6 +1514,13 @@ templ Plugin(page string) {
|
||||
var perWorkerExecText = enabled ? String(perWorkerExec) : '-';
|
||||
var executorWorkersText = enabled ? String(executorWorkers) : '-';
|
||||
var effectiveExecText = enabled ? String(effectiveExec) : '-';
|
||||
var lastRunStatus = textOrDash(item.last_run_status);
|
||||
var lastRunTime = parseTime(item.last_run_completed_at);
|
||||
var lastRunText = '-';
|
||||
if (lastRunStatus !== '-' || lastRunTime) {
|
||||
var statusLabel = lastRunStatus !== '-' ? lastRunStatus : 'run';
|
||||
lastRunText = lastRunTime ? (statusLabel + ' @ ' + lastRunTime) : statusLabel;
|
||||
}
|
||||
|
||||
var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
|
||||
var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
|
||||
@@ -1465,18 +1534,38 @@ templ Plugin(page string) {
|
||||
'<td>' + enabledBadge + '</td>' +
|
||||
'<td><small>' + escapeHtml(detector) + '</small></td>' +
|
||||
'<td>' + inFlightBadge + '</td>' +
|
||||
'<td><small>' + escapeHtml(parseTime(item.next_detection_at) || '-') + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(intervalText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(maxRuntimeText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(effectiveExecText) + '</small></td>' +
|
||||
'<td><small>' + escapeHtml(lastRunText) + '</small></td>' +
|
||||
'</tr>';
|
||||
}
|
||||
|
||||
tbody.innerHTML = rows;
|
||||
}
|
||||
|
||||
function renderSchedulerStatus() {
|
||||
var valueNodes = document.querySelectorAll('.plugin-scheduler-next-run');
|
||||
if (!valueNodes.length) {
|
||||
return;
|
||||
}
|
||||
var metaNodes = document.querySelectorAll('.plugin-scheduler-next-run-meta');
|
||||
var status = state.schedulerStatus || {};
|
||||
var nextRun = parseTime(status.next_detection_at);
|
||||
var display = nextRun || '-';
|
||||
valueNodes.forEach(function(node) {
|
||||
node.textContent = display;
|
||||
});
|
||||
var metaText = nextRun ? 'Local time' : 'Not scheduled';
|
||||
if (metaNodes.length) {
|
||||
metaNodes.forEach(function(node) {
|
||||
node.textContent = metaText;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function renderWorkers() {
|
||||
var tbody = document.getElementById('plugin-workers-table-body');
|
||||
if (!state.workers.length) {
|
||||
@@ -2372,8 +2461,8 @@ templ Plugin(page string) {
|
||||
}
|
||||
|
||||
document.getElementById('plugin-admin-enabled').checked = pickBool('enabled');
|
||||
document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_seconds'));
|
||||
document.getElementById('plugin-admin-detection-timeout').value = String(pickNumber('detection_timeout_seconds'));
|
||||
document.getElementById('plugin-admin-max-runtime').value = String(pickNumber('job_type_max_runtime_seconds'));
|
||||
document.getElementById('plugin-admin-max-results').value = String(pickNumber('max_jobs_per_detection'));
|
||||
document.getElementById('plugin-admin-global-exec').value = String(pickNumber('global_execution_concurrency'));
|
||||
document.getElementById('plugin-admin-per-worker-exec').value = String(pickNumber('per_worker_execution_concurrency'));
|
||||
@@ -2382,6 +2471,9 @@ templ Plugin(page string) {
|
||||
}
|
||||
|
||||
function collectAdminSettings() {
|
||||
var existingRuntime = (state.config && state.config.admin_runtime) ? state.config.admin_runtime : {};
|
||||
var existingDetectionInterval = Number(existingRuntime.detection_interval_seconds || 0);
|
||||
|
||||
function getInt(id) {
|
||||
var raw = String(document.getElementById(id).value || '').trim();
|
||||
if (!raw) {
|
||||
@@ -2396,8 +2488,9 @@ templ Plugin(page string) {
|
||||
|
||||
return {
|
||||
enabled: !!document.getElementById('plugin-admin-enabled').checked,
|
||||
detection_interval_seconds: getInt('plugin-admin-detection-interval'),
|
||||
detection_interval_seconds: existingDetectionInterval,
|
||||
detection_timeout_seconds: getInt('plugin-admin-detection-timeout'),
|
||||
job_type_max_runtime_seconds: getInt('plugin-admin-max-runtime'),
|
||||
max_jobs_per_detection: getInt('plugin-admin-max-results'),
|
||||
global_execution_concurrency: getInt('plugin-admin-global-exec'),
|
||||
per_worker_execution_concurrency: getInt('plugin-admin-per-worker-exec'),
|
||||
@@ -2713,6 +2806,75 @@ templ Plugin(page string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSchedulerConfig(forceRefresh) {
|
||||
if (state.schedulerConfigLoaded && !forceRefresh) {
|
||||
return;
|
||||
}
|
||||
var idleInputs = [
|
||||
document.getElementById('plugin-scheduler-idle-sleep'),
|
||||
document.getElementById('plugin-scheduler-idle-sleep-overview'),
|
||||
].filter(Boolean);
|
||||
if (idleInputs.length === 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
var cfg = await pluginRequest('GET', '/api/plugin/scheduler-config');
|
||||
state.schedulerConfig = cfg || {};
|
||||
state.schedulerConfigLoaded = true;
|
||||
var idleSeconds = Number((cfg && cfg.idle_sleep_seconds) || 0);
|
||||
idleInputs.forEach(function(input) {
|
||||
input.value = idleSeconds > 0 ? String(idleSeconds) : '';
|
||||
});
|
||||
} catch (e) {
|
||||
notify('Failed to load scheduler config: ' + e.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async function saveSchedulerConfig(sourceInput) {
|
||||
var idleInputs = [
|
||||
document.getElementById('plugin-scheduler-idle-sleep'),
|
||||
document.getElementById('plugin-scheduler-idle-sleep-overview'),
|
||||
].filter(Boolean);
|
||||
if (idleInputs.length === 0) {
|
||||
return;
|
||||
}
|
||||
var raw = '';
|
||||
if (sourceInput) {
|
||||
raw = String(sourceInput.value || '').trim();
|
||||
}
|
||||
if (!raw) {
|
||||
for (var i = 0; i < idleInputs.length; i++) {
|
||||
if (idleInputs[i] === sourceInput) {
|
||||
continue;
|
||||
}
|
||||
var candidate = String(idleInputs[i].value || '').trim();
|
||||
if (candidate) {
|
||||
raw = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
var parsed = raw ? parseInt(raw, 10) : 0;
|
||||
if (Number.isNaN(parsed) || parsed < 0) {
|
||||
notify('Invalid idle sleep value', 'error');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
var updated = await pluginRequest('PUT', '/api/plugin/scheduler-config', {
|
||||
idle_sleep_seconds: parsed,
|
||||
});
|
||||
state.schedulerConfig = updated || {};
|
||||
state.schedulerConfigLoaded = true;
|
||||
var idleSeconds = Number((updated && updated.idle_sleep_seconds) || 0);
|
||||
idleInputs.forEach(function(input) {
|
||||
input.value = idleSeconds > 0 ? String(idleSeconds) : '';
|
||||
});
|
||||
notify('Scheduler settings saved', 'success');
|
||||
} catch (e) {
|
||||
notify('Failed to save scheduler config: ' + e.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
function getMaxResults() {
|
||||
var raw = String(document.getElementById('plugin-admin-max-results').value || '').trim();
|
||||
if (!raw) {
|
||||
@@ -2788,21 +2950,30 @@ templ Plugin(page string) {
|
||||
var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
|
||||
var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
|
||||
var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
|
||||
var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status');
|
||||
|
||||
var allJobs = await allJobsPromise;
|
||||
var allActivities = await allActivitiesPromise;
|
||||
var schedulerStates = await schedulerPromise;
|
||||
var schedulerStatus = null;
|
||||
try {
|
||||
schedulerStatus = await schedulerStatusPromise;
|
||||
} catch (e) {
|
||||
schedulerStatus = null;
|
||||
}
|
||||
|
||||
state.jobs = Array.isArray(allJobs) ? allJobs : [];
|
||||
state.activities = Array.isArray(allActivities) ? allActivities : [];
|
||||
state.allJobs = state.jobs;
|
||||
state.allActivities = state.activities;
|
||||
state.schedulerStates = Array.isArray(schedulerStates) ? schedulerStates : [];
|
||||
state.schedulerStatus = schedulerStatus && schedulerStatus.scheduler ? schedulerStatus.scheduler : null;
|
||||
renderQueueJobs();
|
||||
renderDetectionJobs();
|
||||
renderExecutionJobs();
|
||||
renderExecutionActivities();
|
||||
renderSchedulerStates();
|
||||
renderSchedulerStatus();
|
||||
renderStatus();
|
||||
renderJobTypeSummary();
|
||||
}
|
||||
@@ -2880,6 +3051,19 @@ templ Plugin(page string) {
|
||||
saveConfig();
|
||||
});
|
||||
|
||||
var saveSchedulerBtn = document.getElementById('plugin-save-scheduler-btn');
|
||||
if (saveSchedulerBtn) {
|
||||
saveSchedulerBtn.addEventListener('click', function() {
|
||||
saveSchedulerConfig(document.getElementById('plugin-scheduler-idle-sleep'));
|
||||
});
|
||||
}
|
||||
var saveSchedulerBtnOverview = document.getElementById('plugin-save-scheduler-btn-overview');
|
||||
if (saveSchedulerBtnOverview) {
|
||||
saveSchedulerBtnOverview.addEventListener('click', function() {
|
||||
saveSchedulerConfig(document.getElementById('plugin-scheduler-idle-sleep-overview'));
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('plugin-trigger-detection-btn').addEventListener('click', function() {
|
||||
runDetection();
|
||||
});
|
||||
@@ -2964,6 +3148,7 @@ templ Plugin(page string) {
|
||||
ensureActiveNavigation();
|
||||
renderNavigationState();
|
||||
await refreshAll();
|
||||
await loadSchedulerConfig(false);
|
||||
|
||||
state.refreshTimer = setInterval(function() {
|
||||
refreshAll();
|
||||
|
||||
Reference in New Issue
Block a user