Fix stale admin lock metric when lock expires and is reacquired (#8859)
* Fix stale admin lock metric when lock expires and is reacquired (#8857) When a lock expired without an explicit unlock and a different client acquired it, the old client's metric was never cleared, causing multiple clients to appear as simultaneously holding the lock. * Use DeleteLabelValues instead of Set(0) to remove stale metric series Avoids cardinality explosion from accumulated stale series when client names are dynamic.
This commit is contained in:
@@ -106,6 +106,9 @@ func (locks *AdminLocks) isValidToken(lockName string, ts time.Time, token int64
|
|||||||
func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts time.Time, token int64) {
|
func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts time.Time, token int64) {
|
||||||
locks.Lock()
|
locks.Lock()
|
||||||
defer locks.Unlock()
|
defer locks.Unlock()
|
||||||
|
if existing, found := locks.locks[lockName]; found && existing.lastClient != clientName {
|
||||||
|
stats.MasterAdminLock.DeleteLabelValues(existing.lastClient)
|
||||||
|
}
|
||||||
lock := &AdminLock{
|
lock := &AdminLock{
|
||||||
accessSecret: rand.Int64(),
|
accessSecret: rand.Int64(),
|
||||||
accessLockTime: time.Now(),
|
accessLockTime: time.Now(),
|
||||||
@@ -118,7 +121,7 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t
|
|||||||
|
|
||||||
func (locks *AdminLocks) deleteLock(lockName string) {
|
func (locks *AdminLocks) deleteLock(lockName string) {
|
||||||
locks.Lock()
|
locks.Lock()
|
||||||
stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0)
|
stats.MasterAdminLock.DeleteLabelValues(locks.locks[lockName].lastClient)
|
||||||
defer locks.Unlock()
|
defer locks.Unlock()
|
||||||
delete(locks.locks, lockName)
|
delete(locks.locks, lockName)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user