gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prometheus metric to count upload errors (#8788)

Browse Source 
Add Prometheus metric to count upload errors (#8775)

Add SeaweedFS_upload_error_total counter labeled by HTTP status code,
so operators can alert on write/replication failures. Code "0" indicates
a transport error (no HTTP response received).

Also add an "Upload Errors" panel to the Grafana dashboard.
This commit is contained in:
Chris Lu
2026-03-26 16:58:05 -07:00
committed by GitHub
parent 17028fbf59
commit 5fa5507234
3 changed files with 143 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json
View File

@@ -1168,6 +1168,108 @@
"title": "Filer QPS", "title": "Filer QPS",
"type": "timeseries" "type": "timeseries"
}, },
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 28
},
"id": 91,
"links": [],
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true,
"width": 250
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "10.3.1",
"targets": [
{
"exemplar": true,
"expr": "sum by (code) (rate(SeaweedFS_upload_error_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{code}}",
"refId": "A",
"step": 30
}
],
"title": "Upload Errors",
"type": "timeseries"
},
{ {
"collapsed": false, "collapsed": false,
"datasource": { "datasource": {
@@ -1178,7 +1280,7 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 28 "y": 35
}, },
"id": 61, "id": 61,
"panels": [], "panels": [],
@@ -1251,7 +1353,7 @@
"h": 7, "h": 7,
"w": 8, "w": 8,
"x": 0, "x": 0,
"y": 29 "y": 36
}, },
"id": 65, "id": 65,
"links": [], "links": [],
@@ -1357,7 +1459,7 @@
"h": 7, "h": 7,
"w": 8, "w": 8,
"x": 8, "x": 8,
"y": 29 "y": 36
}, },
"id": 56, "id": 56,
"links": [], "links": [],
@@ -1463,7 +1565,7 @@
"h": 7, "h": 7,
"w": 8, "w": 8,
"x": 16, "x": 16,
"y": 29 "y": 36
}, },
"id": 58, "id": 58,
"links": [], "links": [],
@@ -1520,7 +1622,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 36 "y": 43
}, },
"id": 84, "id": 84,
"links": [], "links": [],
@@ -1565,7 +1667,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 36 "y": 43
}, },
"id": 85, "id": 85,
"links": [], "links": [],
@@ -1661,7 +1763,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 41 "y": 48
}, },
"id": 86, "id": 86,
"options": { "options": {
@@ -1756,7 +1858,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 41 "y": 48
}, },
"id": 72, "id": 72,
"links": [], "links": [],
@@ -1874,7 +1976,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 50 "y": 57
}, },
"id": 73, "id": 73,
"links": [], "links": [],
@@ -2030,7 +2132,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 57 "y": 64
}, },
"id": 55, "id": 55,
"links": [], "links": [],
@@ -2187,7 +2289,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 64 "y": 71
}, },
"hideTimeOverride": false, "hideTimeOverride": false,
"id": 59, "id": 59,
@@ -2259,7 +2361,7 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 71 "y": 78
}, },
"id": 62, "id": 62,
"panels": [], "panels": [],
@@ -2331,7 +2433,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 72 "y": 79
}, },
"id": 47, "id": 47,
"links": [], "links": [],
@@ -2474,7 +2576,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 72 "y": 79
}, },
"id": 40, "id": 40,
"links": [], "links": [],
@@ -2571,7 +2673,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 79 "y": 86
}, },
"id": 48, "id": 48,
"links": [], "links": [],
@@ -2681,7 +2783,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 86 "y": 93
}, },
"id": 50, "id": 50,
"links": [], "links": [],
@@ -2783,7 +2885,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 93 "y": 100
}, },
"id": 51, "id": 51,
"links": [], "links": [],
@@ -2823,7 +2925,7 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 93 "y": 100
}, },
"id": 63, "id": 63,
"panels": [], "panels": [],
@@ -2896,7 +2998,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 101 "y": 108
}, },
"id": 12, "id": 12,
"links": [], "links": [],
@@ -2991,7 +3093,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 101 "y": 108
}, },
"id": 14, "id": 14,
"links": [], "links": [],
@@ -3033,7 +3135,7 @@
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 108 "y": 115
}, },
"id": 64, "id": 64,
"panels": [], "panels": [],
@@ -3106,7 +3208,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 109 "y": 116
}, },
"id": 52, "id": 52,
"links": [], "links": [],
@@ -3234,7 +3336,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 109 "y": 116
}, },
"id": 54, "id": 54,
"links": [], "links": [],
@@ -3331,7 +3433,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 116 "y": 123
}, },
"id": 53, "id": 53,
"links": [], "links": [],
@@ -3426,7 +3528,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 48 "y": 55
}, },
"id": 89, "id": 89,
"options": { "options": {
@@ -3533,7 +3635,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 48 "y": 55
}, },
"id": 90, "id": 90,
"options": { "options": {
@@ -3654,4 +3756,4 @@
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304", "uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
"version": 1, "version": 1,
"weekStart": "" "weekStart": ""
} }

5
weed/operation/upload_content.go
View File

@@ -11,6 +11,7 @@ import (
"net/http" "net/http"
"net/textproto" "net/textproto"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -415,6 +416,7 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
} }
} }
if post_err != nil { if post_err != nil {
stats.UploadErrorCounter.WithLabelValues("0").Inc()
return nil, fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err) return nil, fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)
} }
// print("-") // print("-")
@@ -428,15 +430,18 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction
resp_body, ra_err := io.ReadAll(resp.Body) resp_body, ra_err := io.ReadAll(resp.Body)
if ra_err != nil { if ra_err != nil {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
return nil, fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err) return nil, fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)
} }
unmarshal_err := json.Unmarshal(resp_body, &ret) unmarshal_err := json.Unmarshal(resp_body, &ret)
if unmarshal_err != nil { if unmarshal_err != nil {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body)) glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body))
return nil, fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err) return nil, fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)
} }
if ret.Error != "" { if ret.Error != "" {
stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
return nil, fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error) return nil, fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)
} }
ret.ETag = etag ret.ETag = etag

9
weed/stats/metrics.go
View File

@@ -458,6 +458,13 @@ var (
Name: "bucket_object_count", Name: "bucket_object_count",
Help: "Current number of objects in each S3 bucket (logical count, deduplicated across replicas).", Help: "Current number of objects in each S3 bucket (logical count, deduplicated across replicas).",
}, []string{"bucket"}) }, []string{"bucket"})
UploadErrorCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
Name: "upload_error_total",
Help: "Counter of upload errors by HTTP status code. Code 0 means transport error (no response received).",
}, []string{"code"})
) )
func init() { func init() {
@@ -519,6 +526,8 @@ func init() {
Gather.MustRegister(S3BucketPhysicalSizeBytesGauge) Gather.MustRegister(S3BucketPhysicalSizeBytesGauge)
Gather.MustRegister(S3BucketObjectCountGauge) Gather.MustRegister(S3BucketObjectCountGauge)
Gather.MustRegister(UploadErrorCounter)
go bucketMetricTTLControl() go bucketMetricTTLControl()
} }