fix: improve raft leader election reliability and failover speed (#8692)

* fix: clear raft vote state file on non-resume startup

The seaweedfs/raft library v1.1.7 added a persistent `state` file for
currentTerm and votedFor. When RaftResumeState=false (the default), the
log, conf, and snapshot directories are cleared but this state file was
not. On repeated restarts, different masters accumulate divergent terms,
causing AppendEntries rejections and preventing leader election.

Fixes #8690

* fix: recover TopologyId from snapshot before clearing raft state

When RaftResumeState=false clears log/conf/snapshot, the TopologyId
(used for license validation) was lost. Now extract it from the latest
snapshot before cleanup and restore it on the topology.

Both seaweedfs/raft and hashicorp/raft paths are handled, with a shared
recoverTopologyIdFromState helper in raft_common.go.

* fix: stagger multi-master bootstrap delay by peer index

Previously all masters used a fixed 1500ms delay before the bootstrap
check. Now the delay is proportional to the peer's sorted index with
randomization (matching the hashicorp raft path), giving the designated
bootstrap node (peer 0) a head start while later peers wait for gRPC
servers to be ready.

Also adds diagnostic logging showing why DoJoinCommand was or wasn't
called, making leader election issues easier to diagnose from logs.

* fix: skip unreachable masters during leader reconnection

When a master leader goes down, non-leader masters still redirect
clients to the stale leader address. The masterClient would follow
these redirects, fail, and retry — wasting round-trips each cycle.

Now tryAllMasters tracks which masters failed within a cycle and skips
redirects pointing to them, reducing log spam and connection overhead
during leader failover.

* fix: take snapshot after TopologyId generation for recovery

After generating a new TopologyId on the leader, immediately take a raft
snapshot so the ID can be recovered from the snapshot on future restarts
with RaftResumeState=false. Without this, short-lived clusters would
lose the TopologyId on restart since no automatic snapshot had been
taken yet.

* test: add multi-master raft failover integration tests

Integration test framework and 5 test scenarios for 3-node master
clusters:

- TestLeaderConsistencyAcrossNodes: all nodes agree on leader and
  TopologyId
- TestLeaderDownAndRecoverQuickly: leader stops, new leader elected,
  old leader rejoins as follower
- TestLeaderDownSlowRecover: leader gone for extended period, cluster
  continues with 2/3 quorum
- TestTwoMastersDownAndRestart: quorum lost (2/3 down), recovered
  when both restart
- TestAllMastersDownAndRestart: full cluster restart, leader elected,
  all nodes agree on TopologyId

* fix: address PR review comments

- peerIndex: return -1 (not 0) when self not found, add warning log
- recoverTopologyIdFromSnapshot: defer dir.Close()
- tests: check GetTopologyId errors instead of discarding them

* fix: address review comments on failover tests

- Assert no leader after quorum loss (was only logging)
- Verify follower cs.Leader matches expected leader via
  ServerAddress.ToHttpAddress() comparison
- Check GetTopologyId error in TestTwoMastersDownAndRestart

weed/command/master.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"os"
@@ -257,12 +258,27 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	// For multi-master mode with non-Hashicorp raft, wait and check if we should join
 	if !*masterOption.raftHashicorp && !isSingleMaster {
 		go func() {
-			time.Sleep(raftJoinCheckDelay)
+			// Stagger bootstrap by peer index so masters don't all check
+			// simultaneously. Peer 0 waits ~1.5s, peer 1 ~3s, etc.
+			idx := peerIndex(myMasterAddress, peers)
+			delay := time.Duration(float64(raftJoinCheckDelay) * (rand.Float64()*0.25 + 1) * float64(idx+1))
+			glog.V(0).Infof("bootstrap check in %v (peer index %d of %d)", delay, idx, len(peers))
+			time.Sleep(delay)
 
 			ms.Topo.RaftServerAccessLock.RLock()
 			isEmptyMaster := ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty()
-			if isEmptyMaster && isTheFirstOne(myMasterAddress, peers) && ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
-				raftServer.DoJoinCommand()
+			isFirst := idx == 0
+			if isEmptyMaster && isFirst {
+				existingLeader := ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress)
+				if existingLeader == "" {
+					raftServer.DoJoinCommand()
+				} else {
+					glog.V(0).Infof("skip bootstrap: existing leader %s found from peers", existingLeader)
+				}
+			} else if !isEmptyMaster {
+				glog.V(0).Infof("skip bootstrap: leader=%q logEmpty=%v", ms.Topo.RaftServer.Leader(), ms.Topo.RaftServer.IsLogEmpty())
+			} else {
+				glog.V(0).Infof("skip bootstrap: %v is not the first master in peers (index %d)", myMasterAddress, idx)
 			}
 			ms.Topo.RaftServerAccessLock.RUnlock()
 		}()
@@ -385,14 +401,19 @@ func normalizeMasterPeerAddress(peer pb.ServerAddress, self pb.ServerAddress) pb
 	return pb.NewServerAddressWithGrpcPort(peer.ToHttpAddress(), grpcPortValue)
 }
 
-func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
+// peerIndex returns the 0-based position of self in the sorted peer list.
+// Peer 0 is the designated bootstrap node. Returns -1 if self is not found.
+func peerIndex(self pb.ServerAddress, peers []pb.ServerAddress) int {
 	slices.SortFunc(peers, func(a, b pb.ServerAddress) int {
 		return strings.Compare(a.ToHttpAddress(), b.ToHttpAddress())
 	})
-	if len(peers) <= 0 {
-		return true
+	for i, peer := range peers {
+		if peer.ToHttpAddress() == self.ToHttpAddress() {
+			return i
+		}
 	}
-	return self.ToHttpAddress() == peers[0].ToHttpAddress()
+	glog.Warningf("peerIndex: self %s not found in peers %v", self, peers)
+	return -1
 }
 
 func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {

weed/command/master_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 )
 
-func TestIsTheFirstOneIgnoresGrpcPort(t *testing.T) {
+func TestPeerIndexIgnoresGrpcPort(t *testing.T) {
 	self := pb.ServerAddress("127.0.0.1:9000.19000")
 	peers := []pb.ServerAddress{
 		"127.0.0.1:9000",
@@ -14,8 +14,8 @@ func TestIsTheFirstOneIgnoresGrpcPort(t *testing.T) {
 		"127.0.0.1:9003.19003",
 	}
 
-	if !isTheFirstOne(self, peers) {
-		t.Fatalf("expected first peer match by HTTP address between %q and %+v", self, peers)
+	if idx := peerIndex(self, peers); idx != 0 {
+		t.Fatalf("expected peer index 0 for %q among %+v, got %d", self, peers, idx)
 	}
 }