From 0a2dac1e56631a739f9cccc5e048b81a6be1bd10 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 10 Mar 2026 18:43:35 -0700 Subject: [PATCH] Reduce mutex scope in bucket metrics TTL sweep Collect expired bucket names under the lock, then release before calling DeletePartialMatch on Prometheus metrics. This prevents RecordBucketActiveTime from blocking during the expensive cleanup. --- weed/stats/metrics.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 448f0e484..9c3b902a3 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -590,27 +590,34 @@ func bucketMetricTTLControl() { for { now := time.Now().UnixNano() + // Collect expired buckets under the lock, then release before + // doing the expensive Prometheus DeletePartialMatch calls. + // This prevents blocking RecordBucketActiveTime during cleanup. bucketLastActiveLock.Lock() + var expiredBuckets []string for bucket, ts := range bucketLastActiveTsNs { if (now - ts) > ttlNs { + expiredBuckets = append(expiredBuckets, bucket) delete(bucketLastActiveTsNs, bucket) - - labels := prometheus.Labels{"bucket": bucket} - c := S3RequestCounter.DeletePartialMatch(labels) - c += S3RequestHistogram.DeletePartialMatch(labels) - c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) - c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) - c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) - c += S3DeletedObjectsCounter.DeletePartialMatch(labels) - c += S3UploadedObjectsCounter.DeletePartialMatch(labels) - c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) - c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) - c += S3BucketObjectCountGauge.DeletePartialMatch(labels) - glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c) } } - bucketLastActiveLock.Unlock() + + for _, bucket := range expiredBuckets { + labels := prometheus.Labels{"bucket": bucket} + c := S3RequestCounter.DeletePartialMatch(labels) + c += S3RequestHistogram.DeletePartialMatch(labels) + c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) + c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) + c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) + c += S3DeletedObjectsCounter.DeletePartialMatch(labels) + c += S3UploadedObjectsCounter.DeletePartialMatch(labels) + c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketObjectCountGauge.DeletePartialMatch(labels) + glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c) + } + time.Sleep(bucketAtiveTTL) }