Healthz check for deadlocks (#4558)

weed/server/raft_server_handlers.go

@@ -1,9 +1,12 @@
 package weed_server
 
 import (
+	"github.com/cenkalti/backoff/v4"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"net/http"
+	"time"
 )
 
 type ClusterStatusResult struct {
@@ -27,12 +30,24 @@ func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *RaftServer) HealthzHandler(w http.ResponseWriter, r *http.Request) {
-	_, err := s.topo.Leader()
+	leader, err := s.topo.Leader()
 	if err != nil {
 		w.WriteHeader(http.StatusServiceUnavailable)
-	} else {
-		w.WriteHeader(http.StatusOK)
+		return
 	}
+	if s.serverAddr == leader {
+		expBackoff := backoff.NewExponentialBackOff()
+		expBackoff.InitialInterval = 20 * time.Millisecond
+		expBackoff.MaxInterval = 1 * time.Second
+		expBackoff.MaxElapsedTime = 5 * time.Second
+		isLocked, err := backoff.RetryWithData(s.topo.IsChildLocked, expBackoff)
+		glog.Errorf("HealthzHandler: %+v", err)
+		if isLocked {
+			w.WriteHeader(http.StatusLocked)
+			return
+		}
+	}
+	w.WriteHeader(http.StatusOK)
 }
 
 func (s *RaftServer) StatsRaftHandler(w http.ResponseWriter, r *http.Request) {