gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce mutex scope in bucket metrics TTL sweep

Browse Source 
Collect expired bucket names under the lock, then release before
calling DeletePartialMatch on Prometheus metrics. This prevents
RecordBucketActiveTime from blocking during the expensive cleanup.
This commit is contained in:
Chris Lu
2026-03-10 18:43:35 -07:00
parent 737116e83c
commit 0a2dac1e56
1 changed files with 21 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
weed/stats/metrics.go
View File

@@ -590,27 +590,34 @@ func bucketMetricTTLControl() {
for {
now := time.Now().UnixNano()
// Collect expired buckets under the lock, then release before
// doing the expensive Prometheus DeletePartialMatch calls.
// This prevents blocking RecordBucketActiveTime during cleanup.
bucketLastActiveLock.Lock()
var expiredBuckets []string
for bucket, ts := range bucketLastActiveTsNs {
if (now - ts) > ttlNs {
expiredBuckets = append(expiredBuckets, bucket)
delete(bucketLastActiveTsNs, bucket)
labels := prometheus.Labels{"bucket": bucket}
c := S3RequestCounter.DeletePartialMatch(labels)
c += S3RequestHistogram.DeletePartialMatch(labels)
c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels)
c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels)
c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels)
c += S3DeletedObjectsCounter.DeletePartialMatch(labels)
c += S3UploadedObjectsCounter.DeletePartialMatch(labels)
c += S3BucketSizeBytesGauge.DeletePartialMatch(labels)
c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels)
c += S3BucketObjectCountGauge.DeletePartialMatch(labels)
glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c)
}
}
bucketLastActiveLock.Unlock()
for _, bucket := range expiredBuckets {
labels := prometheus.Labels{"bucket": bucket}
c := S3RequestCounter.DeletePartialMatch(labels)
c += S3RequestHistogram.DeletePartialMatch(labels)
c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels)
c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels)
c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels)
c += S3DeletedObjectsCounter.DeletePartialMatch(labels)
c += S3UploadedObjectsCounter.DeletePartialMatch(labels)
c += S3BucketSizeBytesGauge.DeletePartialMatch(labels)
c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels)
c += S3BucketObjectCountGauge.DeletePartialMatch(labels)
glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c)
}
time.Sleep(bucketAtiveTTL)
}