15f4a97029
* fix: clear raft vote state file on non-resume startup The seaweedfs/raft library v1.1.7 added a persistent `state` file for currentTerm and votedFor. When RaftResumeState=false (the default), the log, conf, and snapshot directories are cleared but this state file was not. On repeated restarts, different masters accumulate divergent terms, causing AppendEntries rejections and preventing leader election. Fixes #8690 * fix: recover TopologyId from snapshot before clearing raft state When RaftResumeState=false clears log/conf/snapshot, the TopologyId (used for license validation) was lost. Now extract it from the latest snapshot before cleanup and restore it on the topology. Both seaweedfs/raft and hashicorp/raft paths are handled, with a shared recoverTopologyIdFromState helper in raft_common.go. * fix: stagger multi-master bootstrap delay by peer index Previously all masters used a fixed 1500ms delay before the bootstrap check. Now the delay is proportional to the peer's sorted index with randomization (matching the hashicorp raft path), giving the designated bootstrap node (peer 0) a head start while later peers wait for gRPC servers to be ready. Also adds diagnostic logging showing why DoJoinCommand was or wasn't called, making leader election issues easier to diagnose from logs. * fix: skip unreachable masters during leader reconnection When a master leader goes down, non-leader masters still redirect clients to the stale leader address. The masterClient would follow these redirects, fail, and retry — wasting round-trips each cycle. Now tryAllMasters tracks which masters failed within a cycle and skips redirects pointing to them, reducing log spam and connection overhead during leader failover. * fix: take snapshot after TopologyId generation for recovery After generating a new TopologyId on the leader, immediately take a raft snapshot so the ID can be recovered from the snapshot on future restarts with RaftResumeState=false. Without this, short-lived clusters would lose the TopologyId on restart since no automatic snapshot had been taken yet. * test: add multi-master raft failover integration tests Integration test framework and 5 test scenarios for 3-node master clusters: - TestLeaderConsistencyAcrossNodes: all nodes agree on leader and TopologyId - TestLeaderDownAndRecoverQuickly: leader stops, new leader elected, old leader rejoins as follower - TestLeaderDownSlowRecover: leader gone for extended period, cluster continues with 2/3 quorum - TestTwoMastersDownAndRestart: quorum lost (2/3 down), recovered when both restart - TestAllMastersDownAndRestart: full cluster restart, leader elected, all nodes agree on TopologyId * fix: address PR review comments - peerIndex: return -1 (not 0) when self not found, add warning log - recoverTopologyIdFromSnapshot: defer dir.Close() - tests: check GetTopologyId errors instead of discarding them * fix: address review comments on failover tests - Assert no leader after quorum loss (was only logging) - Verify follower cs.Leader matches expected leader via ServerAddress.ToHttpAddress() comparison - Check GetTopologyId error in TestTwoMastersDownAndRestart
67 lines
1.7 KiB
Go
67 lines
1.7 KiB
Go
package command
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
)
|
|
|
|
func TestPeerIndexIgnoresGrpcPort(t *testing.T) {
|
|
self := pb.ServerAddress("127.0.0.1:9000.19000")
|
|
peers := []pb.ServerAddress{
|
|
"127.0.0.1:9000",
|
|
"127.0.0.1:9002.19002",
|
|
"127.0.0.1:9003.19003",
|
|
}
|
|
|
|
if idx := peerIndex(self, peers); idx != 0 {
|
|
t.Fatalf("expected peer index 0 for %q among %+v, got %d", self, peers, idx)
|
|
}
|
|
}
|
|
|
|
func TestCheckPeersAddsSelfWhenGrpcPortMismatches(t *testing.T) {
|
|
self, peers := checkPeers("127.0.0.1", 9000, 19000, "127.0.0.1:9002,127.0.0.1:9003")
|
|
|
|
found := false
|
|
for _, peer := range peers {
|
|
if peer.ToHttpAddress() == self.ToHttpAddress() {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("expected peers %+v to contain self %s by HTTP address", peers, self)
|
|
}
|
|
}
|
|
|
|
func TestCheckPeersCanonicalizesSelfEntry(t *testing.T) {
|
|
self, peers := checkPeers("127.0.0.1", 9000, 19000, "127.0.0.1:9000,127.0.0.1:9002,127.0.0.1:9003")
|
|
|
|
for _, peer := range peers {
|
|
if peer.ToHttpAddress() == self.ToHttpAddress() && peer != self {
|
|
t.Fatalf("expected self peer to be canonicalized to %q, got %q", self, peer)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCheckPeersDeduplicatesAliasPeers(t *testing.T) {
|
|
_, peers := checkPeers("127.0.0.1", 9000, 19000, "127.0.0.1:9002,127.0.0.1:9002.19002,127.0.0.1:9003")
|
|
|
|
if len(peers) != 3 {
|
|
t.Fatalf("expected 3 unique peers after normalization, got %d: %+v", len(peers), peers)
|
|
}
|
|
|
|
count9002 := 0
|
|
for _, peer := range peers {
|
|
if peer.ToHttpAddress() == "127.0.0.1:9002" {
|
|
count9002++
|
|
if string(peer) != "127.0.0.1:9002.19002" {
|
|
t.Fatalf("expected canonical peer 127.0.0.1:9002.19002, got %q", peer)
|
|
}
|
|
}
|
|
}
|
|
if count9002 != 1 {
|
|
t.Fatalf("expected one peer for 127.0.0.1:9002, got %d in %+v", count9002, peers)
|
|
}
|
|
}
|