gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(balance): replica placement validation for volume moves (#8622)

Browse Source 
* feat(balance): add replica placement validation for volume moves

When the volume balance detection proposes moving a volume, validate
that the move does not violate the volume's replication policy (e.g.,
ReplicaPlacement=010 requires replicas on different racks). If the
preferred destination violates the policy, fall back to score-based
planning; if that also violates, skip the volume entirely.

- Add ReplicaLocation type and VolumeReplicaMap to ClusterInfo
- Build replica map from all volumes before collection filtering
- Port placement validation logic from command_volume_fix_replication.go
- Thread replica map through collectVolumeMetrics call chain
- Add IsGoodMove check in createBalanceTask before destination use

* address PR review: extract validation closure, add defensive checks

- Extract validateMove closure to eliminate duplicated ReplicaLocation
  construction and IsGoodMove calls
- Add defensive check for empty replica map entries (len(replicas) == 0)
- Add bounds check for int-to-byte cast on ExpectedReplicas (0-255)

* address nitpick: rp test helper accepts *testing.T and fails on error

Prevents silent failures from typos in replica placement codes.

* address review: add composite replica placement tests (011, 110)

Test multi-constraint placement policies where both rack and DC
rules must be satisfied simultaneously.

* address review: use struct keys instead of string concatenation

Replace string-concatenated map keys with typed rackKey/nodeKey
structs to eliminate allocations and avoid ambiguity if IDs
contain spaces.

* address review: simplify bounds check, log fallback error, guard source

- Remove unreachable ExpectedReplicas < 0 branch (outer condition
  already guarantees > 0), fold bounds check into single condition
- Log error from planBalanceDestination in replica validation fallback
- Return false from IsGoodMove when sourceNodeID not found in
  existing replicas (inconsistent cluster state)

* address review: use slices.Contains instead of hand-rolled helpers

Replace isAmongDC and isAmongRack with slices.Contains from the
standard library, reducing boilerplate.
This commit is contained in:
Chris Lu
2026-03-13 17:39:25 -07:00
committed by GitHub
parent 47ddf05d95
commit 8056b702ba
9 changed files with 364 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
weed/plugin/worker/erasure_coding_handler.go
View File

@@ -601,7 +601,8 @@ func (h *ErasureCodingHandler) collectVolumeMetrics(
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
return collectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
metrics, activeTopology, _, err := collectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
return metrics, activeTopology, err
}
func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *erasureCodingWorkerConfig {

3
weed/plugin/worker/vacuum_handler.go
View File

@@ -505,7 +505,8 @@ func (h *VacuumHandler) collectVolumeMetrics(
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
return collectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
metrics, activeTopology, _, err := collectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
return metrics, activeTopology, err
}
func deriveVacuumConfig(values map[string]*plugin_pb.ConfigValue) *vacuumtask.Config {

13
weed/plugin/worker/volume_balance_handler.go
View File

@@ -314,7 +314,7 @@ func (h *VolumeBalanceHandler) Detect(
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
}
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
metrics, activeTopology, replicaMap, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
if err != nil {
return err
}
@@ -334,7 +334,10 @@ func (h *VolumeBalanceHandler) Detect(
workerConfig.TaskConfig.RackFilter = rackFilter
workerConfig.TaskConfig.NodeFilter = nodeFilter
clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
clusterInfo := &workertypes.ClusterInfo{
ActiveTopology: activeTopology,
VolumeReplicaMap: replicaMap,
}
maxResults := int(request.MaxResults)
var results []*workertypes.TaskDetectionResult
@@ -1072,10 +1075,8 @@ func (h *VolumeBalanceHandler) collectVolumeMetrics(
ctx context.Context,
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
// Reuse the same master topology fetch/build flow used by the vacuum handler.
helper := &VacuumHandler{grpcDialOption: h.grpcDialOption}
return helper.collectVolumeMetrics(ctx, masterAddresses, collectionFilter)
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, map[uint32][]workertypes.ReplicaLocation, error) {
return collectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
}
func deriveBalanceWorkerConfig(values map[string]*plugin_pb.ConfigValue) *volumeBalanceWorkerConfig {

33
weed/plugin/worker/volume_metrics.go
View File

@@ -22,12 +22,12 @@ func collectVolumeMetricsFromMasters(
masterAddresses []string,
collectionFilter string,
grpcDialOption grpc.DialOption,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, map[uint32][]workertypes.ReplicaLocation, error) {
if grpcDialOption == nil {
return nil, nil, fmt.Errorf("grpc dial option is not configured")
return nil, nil, nil, fmt.Errorf("grpc dial option is not configured")
}
if len(masterAddresses) == 0 {
return nil, nil, fmt.Errorf("no master addresses provided in cluster context")
return nil, nil, nil, fmt.Errorf("no master addresses provided in cluster context")
}
for _, masterAddress := range masterAddresses {
@@ -37,20 +37,20 @@ func collectVolumeMetricsFromMasters(
continue
}
metrics, activeTopology, buildErr := buildVolumeMetrics(response, collectionFilter)
metrics, activeTopology, replicaMap, buildErr := buildVolumeMetrics(response, collectionFilter)
if buildErr != nil {
// Configuration errors (e.g. invalid regex) will fail on every master,
// so return immediately instead of masking them with retries.
if isConfigError(buildErr) {
return nil, nil, buildErr
return nil, nil, nil, buildErr
}
glog.Warningf("Plugin worker failed to build metrics from master %s: %v", masterAddress, buildErr)
continue
}
return metrics, activeTopology, nil
return metrics, activeTopology, replicaMap, nil
}
return nil, nil, fmt.Errorf("failed to load topology from all provided masters")
return nil, nil, nil, fmt.Errorf("failed to load topology from all provided masters")
}
func fetchVolumeList(ctx context.Context, address string, grpcDialOption grpc.DialOption) (*master_pb.VolumeListResponse, error) {
@@ -89,14 +89,14 @@ func fetchVolumeList(ctx context.Context, address string, grpcDialOption grpc.Di
func buildVolumeMetrics(
response *master_pb.VolumeListResponse,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, map[uint32][]workertypes.ReplicaLocation, error) {
if response == nil || response.TopologyInfo == nil {
return nil, nil, fmt.Errorf("volume list response has no topology info")
return nil, nil, nil, fmt.Errorf("volume list response has no topology info")
}
activeTopology := topology.NewActiveTopology(10)
if err := activeTopology.UpdateTopology(response.TopologyInfo); err != nil {
return nil, nil, err
return nil, nil, nil, err
}
var collectionRegex *regexp.Regexp
@@ -105,19 +105,28 @@ func buildVolumeMetrics(
var err error
collectionRegex, err = regexp.Compile(trimmedFilter)
if err != nil {
return nil, nil, &configError{err: fmt.Errorf("invalid collection_filter regex %q: %w", trimmedFilter, err)}
return nil, nil, nil, &configError{err: fmt.Errorf("invalid collection_filter regex %q: %w", trimmedFilter, err)}
}
}
volumeSizeLimitBytes := uint64(response.VolumeSizeLimitMb) * 1024 * 1024
now := time.Now()
metrics := make([]*workertypes.VolumeHealthMetrics, 0, 256)
replicaMap := make(map[uint32][]workertypes.ReplicaLocation)
for _, dc := range response.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for diskType, diskInfo := range node.DiskInfos {
for _, volume := range diskInfo.VolumeInfos {
// Build replica map from ALL volumes BEFORE collection filtering,
// since replicas may span filtered/unfiltered nodes.
replicaMap[volume.Id] = append(replicaMap[volume.Id], workertypes.ReplicaLocation{
DataCenter: dc.Id,
Rack: rack.Id,
NodeID: node.Id,
})
if collectionRegex != nil && !collectionRegex.MatchString(volume.Collection) {
continue
}
@@ -160,7 +169,7 @@ func buildVolumeMetrics(
metric.ReplicaCount = replicaCounts[metric.VolumeID]
}
return metrics, activeTopology, nil
return metrics, activeTopology, replicaMap, nil
}
// configError wraps configuration errors that should not be retried across masters.

10
weed/plugin/worker/volume_metrics_test.go
View File

@@ -39,7 +39,7 @@ func TestBuildVolumeMetricsEmptyFilter(t *testing.T) {
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
)
metrics, _, err := buildVolumeMetrics(resp, "")
metrics, _, _, err := buildVolumeMetrics(resp, "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -53,7 +53,7 @@ func TestBuildVolumeMetricsAllCollections(t *testing.T) {
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
)
metrics, _, err := buildVolumeMetrics(resp, collectionFilterAll)
metrics, _, _, err := buildVolumeMetrics(resp, collectionFilterAll)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -68,7 +68,7 @@ func TestBuildVolumeMetricsEachCollection(t *testing.T) {
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
)
// EACH_COLLECTION passes all volumes through; filtering happens in the handler
metrics, _, err := buildVolumeMetrics(resp, collectionFilterEach)
metrics, _, _, err := buildVolumeMetrics(resp, collectionFilterEach)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -83,7 +83,7 @@ func TestBuildVolumeMetricsRegexFilter(t *testing.T) {
&master_pb.VolumeInformationMessage{Id: 2, Collection: "videos", Size: 200},
&master_pb.VolumeInformationMessage{Id: 3, Collection: "photos-backup", Size: 300},
)
metrics, _, err := buildVolumeMetrics(resp, "^photos$")
metrics, _, _, err := buildVolumeMetrics(resp, "^photos$")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -99,7 +99,7 @@ func TestBuildVolumeMetricsInvalidRegex(t *testing.T) {
resp := makeTestVolumeListResponse(
&master_pb.VolumeInformationMessage{Id: 1, Collection: "photos", Size: 100},
)
_, _, err := buildVolumeMetrics(resp, "[invalid")
_, _, _, err := buildVolumeMetrics(resp, "[invalid")
if err == nil {
t.Fatal("expected error for invalid regex")
}