fix(worker): add metrics HTTP server and health checks for Kubernetes (#7860)
* feat(worker): add metrics HTTP server and debug profiling support - Add -metricsPort flag to enable Prometheus metrics endpoint - Add -metricsIp flag to configure metrics server bind address - Implement /metrics endpoint for Prometheus-compatible metrics - Implement /health endpoint for Kubernetes readiness/liveness probes - Add -debug flag to enable pprof debugging server - Add -debug.port flag to configure debug server port - Fix stats package import naming conflict by using alias - Update usage examples to show new flags Fixes #7843 * feat(helm): add worker metrics and health check support - Update worker readiness probe to use httpGet on /health endpoint - Update worker liveness probe to use httpGet on /health endpoint - Add metricsPort flag to worker command in deployment template - Support both httpGet and tcpSocket probe types for backward compatibility - Update values.yaml with health check configuration This enables Kubernetes pod lifecycle management for worker components through proper health checks on the new metrics HTTP endpoint. * feat(mini): align all services to share single debug and metrics servers - Disable S3's separate debug server in mini mode (port 6060 now shared by all) - Add metrics server startup to embedded worker for health monitoring - All services now share the single metrics port (9327) and single debug port (6060) - Consistent pattern with master, filer, volume, webdav services * fix(worker): fix variable shadowing in health check handler - Rename http.ResponseWriter parameter from 'w' to 'rw' to avoid shadowing the outer 'w *worker.Worker' parameter - Prevents potential bugs if future code tries to use worker state in handler - Improves code clarity and follows Go best practices * fix(worker): remove unused worker parameter in metrics server - Change 'w *worker.Worker' parameter to '_' as it's not used - Clarifies intent that parameter is intentionally unused - Follows Go best practices and improves code clarity * fix(helm): fix trailing backslash syntax errors in worker command - Fix conditional backslash placement to prevent shell syntax errors - Only add backslash when metricsPort OR extraArgs are present - Prevents worker pod startup failures due to malformed command arguments - Ensures proper shell command parsing regardless of configuration state * refactor(worker): use standard stats.StartMetricsServer for consistency - Replace custom metrics server implementation with stats.StartMetricsServer to match pattern used in master, volume, s3, filer_sync components - Simplifies code and improves maintainability - Uses glog.Fatal for errors (consistent with other SeaweedFS components) - Remove unused net/http and prometheus/promhttp imports - Automatically provides /metrics and /health endpoints via standard implementation
This commit is contained in:
@@ -138,7 +138,10 @@ spec:
|
||||
{{- end }}
|
||||
-capabilities={{ .Values.worker.capabilities }} \
|
||||
-maxConcurrent={{ .Values.worker.maxConcurrent }} \
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- if .Values.worker.metricsPort }}
|
||||
-metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- end }}
|
||||
{{- range $index, $arg := .Values.worker.extraArgs }}
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
|
||||
{{- end }}
|
||||
@@ -187,9 +190,13 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.worker.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
{{- with .Values.worker.livenessProbe.tcpSocket }}
|
||||
{{- if .Values.worker.livenessProbe.httpGet }}
|
||||
httpGet:
|
||||
path: {{ .Values.worker.livenessProbe.httpGet.path }}
|
||||
port: {{ .Values.worker.livenessProbe.httpGet.port }}
|
||||
{{- else if .Values.worker.livenessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ .port }}
|
||||
port: {{ .Values.worker.livenessProbe.tcpSocket.port }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
|
||||
@@ -199,9 +206,13 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.worker.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
{{- with .Values.worker.readinessProbe.tcpSocket }}
|
||||
{{- if .Values.worker.readinessProbe.httpGet }}
|
||||
httpGet:
|
||||
path: {{ .Values.worker.readinessProbe.httpGet.path }}
|
||||
port: {{ .Values.worker.readinessProbe.httpGet.port }}
|
||||
{{- else if .Values.worker.readinessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ .port }}
|
||||
port: {{ .Values.worker.readinessProbe.tcpSocket.port }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
|
||||
|
||||
@@ -1302,10 +1302,11 @@ worker:
|
||||
extraEnvironmentVars: {}
|
||||
|
||||
# Health checks for worker pods
|
||||
# Since workers do not have an HTTP endpoint, a tcpSocket probe on the metrics port is recommended.
|
||||
# Workers expose metrics on the metricsPort with a /health endpoint for readiness checks.
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
tcpSocket:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 60
|
||||
@@ -1315,7 +1316,8 @@ worker:
|
||||
|
||||
readinessProbe:
|
||||
enabled: true
|
||||
tcpSocket:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 15
|
||||
|
||||
Reference in New Issue
Block a user