From 4705d8b82b085f47886813f93252f8db318c2b55 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 30 Mar 2026 18:51:38 -0700 Subject: [PATCH] Fix stale admin lock metric when lock expires and is reacquired (#8859) * Fix stale admin lock metric when lock expires and is reacquired (#8857) When a lock expired without an explicit unlock and a different client acquired it, the old client's metric was never cleared, causing multiple clients to appear as simultaneously holding the lock. * Use DeleteLabelValues instead of Set(0) to remove stale metric series Avoids cardinality explosion from accumulated stale series when client names are dynamic. --- weed/server/master_grpc_server_admin.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/server/master_grpc_server_admin.go b/weed/server/master_grpc_server_admin.go index eb6cb452b..ce90f5cac 100644 --- a/weed/server/master_grpc_server_admin.go +++ b/weed/server/master_grpc_server_admin.go @@ -106,6 +106,9 @@ func (locks *AdminLocks) isValidToken(lockName string, ts time.Time, token int64 func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts time.Time, token int64) { locks.Lock() defer locks.Unlock() + if existing, found := locks.locks[lockName]; found && existing.lastClient != clientName { + stats.MasterAdminLock.DeleteLabelValues(existing.lastClient) + } lock := &AdminLock{ accessSecret: rand.Int64(), accessLockTime: time.Now(), @@ -118,7 +121,7 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t func (locks *AdminLocks) deleteLock(lockName string) { locks.Lock() - stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0) + stats.MasterAdminLock.DeleteLabelValues(locks.locks[lockName].lastClient) defer locks.Unlock() delete(locks.locks, lockName) }