diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 448f0e484..9c3b902a3 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -590,27 +590,34 @@ func bucketMetricTTLControl() { for { now := time.Now().UnixNano() + // Collect expired buckets under the lock, then release before + // doing the expensive Prometheus DeletePartialMatch calls. + // This prevents blocking RecordBucketActiveTime during cleanup. bucketLastActiveLock.Lock() + var expiredBuckets []string for bucket, ts := range bucketLastActiveTsNs { if (now - ts) > ttlNs { + expiredBuckets = append(expiredBuckets, bucket) delete(bucketLastActiveTsNs, bucket) - - labels := prometheus.Labels{"bucket": bucket} - c := S3RequestCounter.DeletePartialMatch(labels) - c += S3RequestHistogram.DeletePartialMatch(labels) - c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) - c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) - c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) - c += S3DeletedObjectsCounter.DeletePartialMatch(labels) - c += S3UploadedObjectsCounter.DeletePartialMatch(labels) - c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) - c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) - c += S3BucketObjectCountGauge.DeletePartialMatch(labels) - glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c) } } - bucketLastActiveLock.Unlock() + + for _, bucket := range expiredBuckets { + labels := prometheus.Labels{"bucket": bucket} + c := S3RequestCounter.DeletePartialMatch(labels) + c += S3RequestHistogram.DeletePartialMatch(labels) + c += S3TimeToFirstByteHistogram.DeletePartialMatch(labels) + c += S3BucketTrafficReceivedBytesCounter.DeletePartialMatch(labels) + c += S3BucketTrafficSentBytesCounter.DeletePartialMatch(labels) + c += S3DeletedObjectsCounter.DeletePartialMatch(labels) + c += S3UploadedObjectsCounter.DeletePartialMatch(labels) + c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) + c += S3BucketObjectCountGauge.DeletePartialMatch(labels) + glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c) + } + time.Sleep(bucketAtiveTTL) }