chore: remove ~50k lines of unreachable dead code (#8913)
* chore: remove unreachable dead code across the codebase Remove ~50,000 lines of unreachable code identified by static analysis. Major removals: - weed/filer/redis_lua: entire unused Redis Lua filer store implementation - weed/wdclient/net2, resource_pool: unused connection/resource pool packages - weed/plugin/worker/lifecycle: unused lifecycle plugin worker - weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy, multipart IAM, key rotation, and various SSE helper functions - weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions - weed/mq/offset: unused SQL storage and migration code - weed/worker: unused registry, task, and monitoring functions - weed/query: unused SQL engine, parquet scanner, and type functions - weed/shell: unused EC proportional rebalance functions - weed/storage/erasure_coding/distribution: unused distribution analysis functions - Individual unreachable functions removed from 150+ files across admin, credential, filer, iam, kms, mount, mq, operation, pb, s3api, server, shell, storage, topology, and util packages * fix(s3): reset shared memory store in IAM test to prevent flaky failure TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because the MemoryStore credential backend is a singleton registered via init(). Earlier tests that create anonymous identities pollute the shared store, causing LookupAnonymous() to unexpectedly return true. Fix by calling Reset() on the memory store before the test runs. * style: run gofmt on changed files * fix: restore KMS functions used by integration tests * fix(plugin): prevent panic on send to closed worker session channel The Plugin.sendToWorker method could panic with "send on closed channel" when a worker disconnected while a message was being sent. The race was between streamSession.close() closing the outgoing channel and sendToWorker writing to it concurrently. Add a done channel to streamSession that is closed before the outgoing channel, and check it in sendToWorker's select to safely detect closed sessions without panicking.
This commit is contained in:
Chris Lu
@@ -406,30 +406,6 @@ func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) {
|
||||
})
|
||||
}
|
||||
|
||||
// if the index node changed the freeEcSlot, need to keep every EcNode still sorted
|
||||
func ensureSortedEcNodes(data []*CandidateEcNode, index int, lessThan func(i, j int) bool) {
|
||||
for i := index - 1; i >= 0; i-- {
|
||||
if lessThan(i+1, i) {
|
||||
swap(data, i, i+1)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
for i := index + 1; i < len(data); i++ {
|
||||
if lessThan(i, i-1) {
|
||||
swap(data, i, i-1)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func swap(data []*CandidateEcNode, i, j int) {
|
||||
t := data[i]
|
||||
data[i] = data[j]
|
||||
data[j] = t
|
||||
}
|
||||
|
||||
func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) {
|
||||
for _, eci := range ecShardInfos {
|
||||
count += erasure_coding.GetShardCount(eci)
|
||||
@@ -1135,48 +1111,6 @@ func (ecb *ecBalancer) pickRackForShardType(
|
||||
return selected.id, nil
|
||||
}
|
||||
|
||||
func (ecb *ecBalancer) pickRackToBalanceShardsInto(rackToEcNodes map[RackId]*EcRack, rackToShardCount map[string]int) (RackId, error) {
|
||||
targets := []RackId{}
|
||||
targetShards := -1
|
||||
for _, shards := range rackToShardCount {
|
||||
if shards > targetShards {
|
||||
targetShards = shards
|
||||
}
|
||||
}
|
||||
|
||||
details := ""
|
||||
for rackId, rack := range rackToEcNodes {
|
||||
shards := rackToShardCount[string(rackId)]
|
||||
|
||||
if rack.freeEcSlot <= 0 {
|
||||
details += fmt.Sprintf(" Skipped %s because it has no free slots\n", rackId)
|
||||
continue
|
||||
}
|
||||
// For EC shards, replica placement constraint only applies when DiffRackCount > 0.
|
||||
// When DiffRackCount = 0 (e.g., replica placement "000"), EC shards should be
|
||||
// distributed freely across racks for fault tolerance - the "000" means
|
||||
// "no volume replication needed" because erasure coding provides redundancy.
|
||||
if ecb.replicaPlacement != nil && ecb.replicaPlacement.DiffRackCount > 0 && shards > ecb.replicaPlacement.DiffRackCount {
|
||||
details += fmt.Sprintf(" Skipped %s because shards %d > replica placement limit for other racks (%d)\n", rackId, shards, ecb.replicaPlacement.DiffRackCount)
|
||||
continue
|
||||
}
|
||||
|
||||
if shards < targetShards {
|
||||
// Favor racks with less shards, to ensure an uniform distribution.
|
||||
targets = nil
|
||||
targetShards = shards
|
||||
}
|
||||
if shards == targetShards {
|
||||
targets = append(targets, rackId)
|
||||
}
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
return "", errors.New(details)
|
||||
}
|
||||
return targets[rand.IntN(len(targets))], nil
|
||||
}
|
||||
|
||||
func (ecb *ecBalancer) balanceEcShardsWithinRacks(collection string) error {
|
||||
// collect vid => []ecNode, since previous steps can change the locations
|
||||
vidLocations := ecb.collectVolumeIdToEcNodes(collection)
|
||||
@@ -1567,46 +1501,6 @@ func (ecb *ecBalancer) pickOneEcNodeAndMoveOneShard(existingLocation *EcNode, co
|
||||
return moveMountedShardToEcNode(ecb.commandEnv, existingLocation, collection, vid, shardId, destNode, destDiskId, ecb.applyBalancing, ecb.diskType)
|
||||
}
|
||||
|
||||
func pickNEcShardsToMoveFrom(ecNodes []*EcNode, vid needle.VolumeId, n int, diskType types.DiskType) map[erasure_coding.ShardId]*EcNode {
|
||||
picked := make(map[erasure_coding.ShardId]*EcNode)
|
||||
var candidateEcNodes []*CandidateEcNode
|
||||
for _, ecNode := range ecNodes {
|
||||
si := findEcVolumeShardsInfo(ecNode, vid, diskType)
|
||||
if si.Count() > 0 {
|
||||
candidateEcNodes = append(candidateEcNodes, &CandidateEcNode{
|
||||
ecNode: ecNode,
|
||||
shardCount: si.Count(),
|
||||
})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(candidateEcNodes, func(a, b *CandidateEcNode) int {
|
||||
return b.shardCount - a.shardCount
|
||||
})
|
||||
for i := 0; i < n; i++ {
|
||||
selectedEcNodeIndex := -1
|
||||
for i, candidateEcNode := range candidateEcNodes {
|
||||
si := findEcVolumeShardsInfo(candidateEcNode.ecNode, vid, diskType)
|
||||
if si.Count() > 0 {
|
||||
selectedEcNodeIndex = i
|
||||
for _, shardId := range si.Ids() {
|
||||
candidateEcNode.shardCount--
|
||||
picked[shardId] = candidateEcNode.ecNode
|
||||
candidateEcNode.ecNode.deleteEcVolumeShards(vid, []erasure_coding.ShardId{shardId}, diskType)
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if selectedEcNodeIndex >= 0 {
|
||||
ensureSortedEcNodes(candidateEcNodes, selectedEcNodeIndex, func(i, j int) bool {
|
||||
return candidateEcNodes[i].shardCount > candidateEcNodes[j].shardCount
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
return picked
|
||||
}
|
||||
|
||||
func (ecb *ecBalancer) collectVolumeIdToEcNodes(collection string) map[needle.VolumeId][]*EcNode {
|
||||
vidLocations := make(map[needle.VolumeId][]*EcNode)
|
||||
for _, ecNode := range ecb.ecNodes {
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func errorCheck(got error, want string) error {
|
||||
if got == nil && want == "" {
|
||||
return nil
|
||||
}
|
||||
if got != nil && want == "" {
|
||||
return fmt.Errorf("expected no error, got %q", got.Error())
|
||||
}
|
||||
if got == nil && want != "" {
|
||||
return fmt.Errorf("got no error, expected %q", want)
|
||||
}
|
||||
if !strings.Contains(got.Error(), want) {
|
||||
return fmt.Errorf("expected error %q, got %q", want, got.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestCollectCollectionsForVolumeIds(t *testing.T) {
|
||||
testCases := []struct {
|
||||
topology *master_pb.TopologyInfo
|
||||
vids []needle.VolumeId
|
||||
want []string
|
||||
}{
|
||||
// normal volumes
|
||||
{testTopology1, nil, nil},
|
||||
{testTopology1, []needle.VolumeId{}, nil},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(9999)}, nil},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(2)}, []string{""}},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272)}, []string{"", "collection2"}},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272), needle.VolumeId(299)}, []string{"", "collection2"}},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95)}, []string{"collection1", "collection2"}},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51)}, []string{"collection1", "collection2"}},
|
||||
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51), needle.VolumeId(15)}, []string{"collection0", "collection1", "collection2"}},
|
||||
// EC volumes
|
||||
{testTopology2, []needle.VolumeId{needle.VolumeId(9577)}, []string{"s3qldata"}},
|
||||
{testTopology2, []needle.VolumeId{needle.VolumeId(9577), needle.VolumeId(12549)}, []string{"s3qldata"}},
|
||||
// normal + EC volumes
|
||||
{testTopology2, []needle.VolumeId{needle.VolumeId(18111)}, []string{"s3qldata"}},
|
||||
{testTopology2, []needle.VolumeId{needle.VolumeId(8677)}, []string{"s3qldata"}},
|
||||
{testTopology2, []needle.VolumeId{needle.VolumeId(18111), needle.VolumeId(8677)}, []string{"s3qldata"}},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
got := collectCollectionsForVolumeIds(tc.topology, tc.vids)
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("for %v: got %v, want %v", tc.vids, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReplicaPlacementArg(t *testing.T) {
|
||||
getDefaultReplicaPlacementOrig := getDefaultReplicaPlacement
|
||||
getDefaultReplicaPlacement = func(commandEnv *CommandEnv) (*super_block.ReplicaPlacement, error) {
|
||||
return super_block.NewReplicaPlacementFromString("123")
|
||||
}
|
||||
defer func() {
|
||||
getDefaultReplicaPlacement = getDefaultReplicaPlacementOrig
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
argument string
|
||||
want string
|
||||
wantErr string
|
||||
}{
|
||||
{"lalala", "lal", "unexpected replication type"},
|
||||
{"", "123", ""},
|
||||
{"021", "021", ""},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
commandEnv := &CommandEnv{}
|
||||
got, gotErr := parseReplicaPlacementArg(commandEnv, tc.argument)
|
||||
|
||||
if err := errorCheck(gotErr, tc.wantErr); err != nil {
|
||||
t.Errorf("argument %q: %s", tc.argument, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
want, _ := super_block.NewReplicaPlacementFromString(tc.want)
|
||||
if !got.Equals(want) {
|
||||
t.Errorf("got replica placement %q, want %q", got.String(), want.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEcDistribution(t *testing.T) {
|
||||
|
||||
// find out all volume servers with one slot left.
|
||||
ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(testTopology1, "", types.HardDriveType)
|
||||
|
||||
sortEcNodesByFreeslotsDescending(ecNodes)
|
||||
|
||||
if totalFreeEcSlots < erasure_coding.TotalShardsCount {
|
||||
t.Errorf("not enough free ec shard slots: %d", totalFreeEcSlots)
|
||||
}
|
||||
allocatedDataNodes := ecNodes
|
||||
if len(allocatedDataNodes) > erasure_coding.TotalShardsCount {
|
||||
allocatedDataNodes = allocatedDataNodes[:erasure_coding.TotalShardsCount]
|
||||
}
|
||||
|
||||
for _, dn := range allocatedDataNodes {
|
||||
// fmt.Printf("info %+v %+v\n", dn.info, dn)
|
||||
fmt.Printf("=> %+v %+v\n", dn.info.Id, dn.freeEcSlot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickRackToBalanceShardsInto(t *testing.T) {
|
||||
testCases := []struct {
|
||||
topology *master_pb.TopologyInfo
|
||||
vid string
|
||||
replicaPlacement string
|
||||
wantOneOf []string
|
||||
wantErr string
|
||||
}{
|
||||
// Non-EC volumes. We don't care about these, but the function should return all racks as a safeguard.
|
||||
{testTopologyEc, "", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
|
||||
{testTopologyEc, "6225", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
|
||||
{testTopologyEc, "6226", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
|
||||
{testTopologyEc, "6241", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
|
||||
{testTopologyEc, "6242", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
|
||||
// EC volumes.
|
||||
// With replication "000" (DiffRackCount=0), EC shards should be distributed freely
|
||||
// because erasure coding provides its own redundancy. No replica placement error.
|
||||
{testTopologyEc, "9577", "", []string{"rack1", "rack2", "rack3"}, ""},
|
||||
{testTopologyEc, "9577", "111", []string{"rack1", "rack2", "rack3"}, ""},
|
||||
{testTopologyEc, "9577", "222", []string{"rack1", "rack2", "rack3"}, ""},
|
||||
{testTopologyEc, "10457", "222", []string{"rack1"}, ""},
|
||||
{testTopologyEc, "12737", "222", []string{"rack2"}, ""},
|
||||
{testTopologyEc, "14322", "222", []string{"rack3"}, ""},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
vid, _ := needle.NewVolumeId(tc.vid)
|
||||
ecNodes, _ := collectEcVolumeServersByDc(tc.topology, "", types.HardDriveType)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString(tc.replicaPlacement)
|
||||
|
||||
ecb := &ecBalancer{
|
||||
ecNodes: ecNodes,
|
||||
replicaPlacement: rp,
|
||||
diskType: types.HardDriveType,
|
||||
}
|
||||
|
||||
racks := ecb.racks()
|
||||
rackToShardCount := countShardsByRack(vid, ecNodes, types.HardDriveType)
|
||||
|
||||
got, gotErr := ecb.pickRackToBalanceShardsInto(racks, rackToShardCount)
|
||||
if err := errorCheck(gotErr, tc.wantErr); err != nil {
|
||||
t.Errorf("volume %q: %s", tc.vid, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if string(got) == "" && len(tc.wantOneOf) == 0 {
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, want := range tc.wantOneOf {
|
||||
if got := string(got); got == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !(found) {
|
||||
t.Errorf("expected one of %v for volume %q, got %q", tc.wantOneOf, tc.vid, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestPickEcNodeToBalanceShardsInto(t *testing.T) {
|
||||
testCases := []struct {
|
||||
topology *master_pb.TopologyInfo
|
||||
nodeId string
|
||||
vid string
|
||||
wantOneOf []string
|
||||
wantErr string
|
||||
}{
|
||||
{testTopologyEc, "", "", nil, "INTERNAL: missing source nodes"},
|
||||
{testTopologyEc, "idontexist", "12737", nil, "INTERNAL: missing source nodes"},
|
||||
// Non-EC nodes. We don't care about these, but the function should return all available target nodes as a safeguard.
|
||||
{
|
||||
testTopologyEc, "172.19.0.10:8702", "6225", []string{
|
||||
"172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704", "172.19.0.17:8703",
|
||||
"172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710", "172.19.0.3:8708",
|
||||
"172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713", "172.19.0.8:8709",
|
||||
"172.19.0.9:8712"},
|
||||
"",
|
||||
},
|
||||
{
|
||||
testTopologyEc, "172.19.0.8:8709", "6226", []string{
|
||||
"172.19.0.10:8702", "172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704",
|
||||
"172.19.0.17:8703", "172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710",
|
||||
"172.19.0.3:8708", "172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713",
|
||||
"172.19.0.9:8712"},
|
||||
"",
|
||||
},
|
||||
// EC volumes.
|
||||
{testTopologyEc, "172.19.0.10:8702", "14322", []string{
|
||||
"172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"},
|
||||
""},
|
||||
{testTopologyEc, "172.19.0.13:8701", "10457", []string{
|
||||
"172.19.0.10:8702", "172.19.0.6:8713"},
|
||||
""},
|
||||
{testTopologyEc, "172.19.0.17:8703", "12737", []string{
|
||||
"172.19.0.13:8701"},
|
||||
""},
|
||||
{testTopologyEc, "172.19.0.20:8706", "14322", []string{
|
||||
"172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"},
|
||||
""},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
vid, _ := needle.NewVolumeId(tc.vid)
|
||||
allEcNodes, _ := collectEcVolumeServersByDc(tc.topology, "", types.HardDriveType)
|
||||
|
||||
ecb := &ecBalancer{
|
||||
ecNodes: allEcNodes,
|
||||
diskType: types.HardDriveType,
|
||||
}
|
||||
|
||||
// Resolve target node by name
|
||||
var ecNode *EcNode
|
||||
for _, n := range allEcNodes {
|
||||
if n.info.Id == tc.nodeId {
|
||||
ecNode = n
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
got, gotErr := ecb.pickEcNodeToBalanceShardsInto(vid, ecNode, allEcNodes)
|
||||
if err := errorCheck(gotErr, tc.wantErr); err != nil {
|
||||
t.Errorf("node %q, volume %q: %s", tc.nodeId, tc.vid, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if got == nil {
|
||||
if len(tc.wantOneOf) == 0 {
|
||||
continue
|
||||
}
|
||||
t.Errorf("node %q, volume %q: got no node, want %q", tc.nodeId, tc.vid, tc.wantOneOf)
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, want := range tc.wantOneOf {
|
||||
if got := got.info.Id; got == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !(found) {
|
||||
t.Errorf("expected one of %v for volume %q, got %q", tc.wantOneOf, tc.vid, got.info.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountFreeShardSlots(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
topology *master_pb.TopologyInfo
|
||||
diskType types.DiskType
|
||||
want map[string]int
|
||||
}{
|
||||
{
|
||||
name: "topology #1, free HDD shards",
|
||||
topology: testTopology1,
|
||||
diskType: types.HardDriveType,
|
||||
want: map[string]int{
|
||||
"192.168.1.1:8080": 17330,
|
||||
"192.168.1.2:8080": 1540,
|
||||
"192.168.1.4:8080": 1900,
|
||||
"192.168.1.5:8080": 27010,
|
||||
"192.168.1.6:8080": 17420,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "topology #1, no free SSD shards available",
|
||||
topology: testTopology1,
|
||||
diskType: types.SsdType,
|
||||
want: map[string]int{
|
||||
"192.168.1.1:8080": 0,
|
||||
"192.168.1.2:8080": 0,
|
||||
"192.168.1.4:8080": 0,
|
||||
"192.168.1.5:8080": 0,
|
||||
"192.168.1.6:8080": 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "topology #2, no negative free HDD shards",
|
||||
topology: testTopology2,
|
||||
diskType: types.HardDriveType,
|
||||
want: map[string]int{
|
||||
"172.19.0.3:8708": 0,
|
||||
"172.19.0.4:8707": 8,
|
||||
"172.19.0.5:8705": 58,
|
||||
"172.19.0.6:8713": 39,
|
||||
"172.19.0.8:8709": 8,
|
||||
"172.19.0.9:8712": 0,
|
||||
"172.19.0.10:8702": 0,
|
||||
"172.19.0.13:8701": 0,
|
||||
"172.19.0.14:8711": 0,
|
||||
"172.19.0.16:8704": 89,
|
||||
"172.19.0.17:8703": 0,
|
||||
"172.19.0.19:8700": 9,
|
||||
"172.19.0.20:8706": 0,
|
||||
"172.19.0.21:8710": 9,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "topology #2, no free SSD shards available",
|
||||
topology: testTopology2,
|
||||
diskType: types.SsdType,
|
||||
want: map[string]int{
|
||||
"172.19.0.10:8702": 0,
|
||||
"172.19.0.13:8701": 0,
|
||||
"172.19.0.14:8711": 0,
|
||||
"172.19.0.16:8704": 0,
|
||||
"172.19.0.17:8703": 0,
|
||||
"172.19.0.19:8700": 0,
|
||||
"172.19.0.20:8706": 0,
|
||||
"172.19.0.21:8710": 0,
|
||||
"172.19.0.3:8708": 0,
|
||||
"172.19.0.4:8707": 0,
|
||||
"172.19.0.5:8705": 0,
|
||||
"172.19.0.6:8713": 0,
|
||||
"172.19.0.8:8709": 0,
|
||||
"172.19.0.9:8712": 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := map[string]int{}
|
||||
eachDataNode(tc.topology, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
|
||||
got[dn.Id] = countFreeShardSlots(dn, tc.diskType)
|
||||
})
|
||||
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("got %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
@@ -138,25 +136,6 @@ func (ce *CommandEnv) GetDataCenter() string {
|
||||
return ce.MasterClient.GetDataCenter()
|
||||
}
|
||||
|
||||
func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
|
||||
if strings.HasPrefix(entryPath, "http") {
|
||||
var u *url.URL
|
||||
u, err = url.Parse(entryPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
filerServer = u.Hostname()
|
||||
portString := u.Port()
|
||||
if portString != "" {
|
||||
filerPort, err = strconv.ParseInt(portString, 10, 32)
|
||||
}
|
||||
path = u.Path
|
||||
} else {
|
||||
err = fmt.Errorf("path should have full url /path/to/dirOrFile : %s", entryPath)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func findInputDirectory(args []string) (input string) {
|
||||
input = "."
|
||||
if len(args) > 0 {
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@@ -13,18 +11,6 @@ import (
|
||||
// ECDistribution is an alias to the distribution package type for backward compatibility
|
||||
type ECDistribution = distribution.ECDistribution
|
||||
|
||||
// CalculateECDistribution computes the target EC shard distribution based on replication policy.
|
||||
// This is a convenience wrapper that uses the default 10+4 EC configuration.
|
||||
// For custom EC ratios, use the distribution package directly.
|
||||
func CalculateECDistribution(totalShards, parityShards int, rp *super_block.ReplicaPlacement) *ECDistribution {
|
||||
ec := distribution.ECConfig{
|
||||
DataShards: totalShards - parityShards,
|
||||
ParityShards: parityShards,
|
||||
}
|
||||
rep := distribution.NewReplicationConfig(rp)
|
||||
return distribution.CalculateDistribution(ec, rep)
|
||||
}
|
||||
|
||||
// TopologyDistributionAnalysis holds the current shard distribution analysis
|
||||
// This wraps the distribution package's TopologyAnalysis with shell-specific EcNode handling
|
||||
type TopologyDistributionAnalysis struct {
|
||||
@@ -34,99 +20,6 @@ type TopologyDistributionAnalysis struct {
|
||||
nodeMap map[string]*EcNode // nodeID -> EcNode
|
||||
}
|
||||
|
||||
// NewTopologyDistributionAnalysis creates a new analysis structure
|
||||
func NewTopologyDistributionAnalysis() *TopologyDistributionAnalysis {
|
||||
return &TopologyDistributionAnalysis{
|
||||
inner: distribution.NewTopologyAnalysis(),
|
||||
nodeMap: make(map[string]*EcNode),
|
||||
}
|
||||
}
|
||||
|
||||
// AddNode adds a node and its shards to the analysis
|
||||
func (a *TopologyDistributionAnalysis) AddNode(node *EcNode, shardsInfo *erasure_coding.ShardsInfo) {
|
||||
nodeId := node.info.Id
|
||||
|
||||
// Create distribution.TopologyNode from EcNode
|
||||
topoNode := &distribution.TopologyNode{
|
||||
NodeID: nodeId,
|
||||
DataCenter: string(node.dc),
|
||||
Rack: string(node.rack),
|
||||
FreeSlots: node.freeEcSlot,
|
||||
TotalShards: shardsInfo.Count(),
|
||||
ShardIDs: shardsInfo.IdsInt(),
|
||||
}
|
||||
|
||||
a.inner.AddNode(topoNode)
|
||||
a.nodeMap[nodeId] = node
|
||||
|
||||
// Add shard locations
|
||||
for _, shardId := range shardsInfo.Ids() {
|
||||
a.inner.AddShardLocation(distribution.ShardLocation{
|
||||
ShardID: int(shardId),
|
||||
NodeID: nodeId,
|
||||
DataCenter: string(node.dc),
|
||||
Rack: string(node.rack),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize completes the analysis
|
||||
func (a *TopologyDistributionAnalysis) Finalize() {
|
||||
a.inner.Finalize()
|
||||
}
|
||||
|
||||
// String returns a summary
|
||||
func (a *TopologyDistributionAnalysis) String() string {
|
||||
return a.inner.String()
|
||||
}
|
||||
|
||||
// DetailedString returns detailed analysis
|
||||
func (a *TopologyDistributionAnalysis) DetailedString() string {
|
||||
return a.inner.DetailedString()
|
||||
}
|
||||
|
||||
// GetShardsByDC returns shard counts by DC
|
||||
func (a *TopologyDistributionAnalysis) GetShardsByDC() map[DataCenterId]int {
|
||||
result := make(map[DataCenterId]int)
|
||||
for dc, count := range a.inner.ShardsByDC {
|
||||
result[DataCenterId(dc)] = count
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetShardsByRack returns shard counts by rack
|
||||
func (a *TopologyDistributionAnalysis) GetShardsByRack() map[RackId]int {
|
||||
result := make(map[RackId]int)
|
||||
for rack, count := range a.inner.ShardsByRack {
|
||||
result[RackId(rack)] = count
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetShardsByNode returns shard counts by node
|
||||
func (a *TopologyDistributionAnalysis) GetShardsByNode() map[EcNodeId]int {
|
||||
result := make(map[EcNodeId]int)
|
||||
for nodeId, count := range a.inner.ShardsByNode {
|
||||
result[EcNodeId(nodeId)] = count
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// AnalyzeVolumeDistribution creates an analysis of current shard distribution for a volume
|
||||
func AnalyzeVolumeDistribution(volumeId needle.VolumeId, locations []*EcNode, diskType types.DiskType) *TopologyDistributionAnalysis {
|
||||
analysis := NewTopologyDistributionAnalysis()
|
||||
|
||||
for _, node := range locations {
|
||||
si := findEcVolumeShardsInfo(node, volumeId, diskType)
|
||||
if si.Count() > 0 {
|
||||
analysis.AddNode(node, si)
|
||||
}
|
||||
}
|
||||
|
||||
analysis.Finalize()
|
||||
return analysis
|
||||
}
|
||||
|
||||
// ECShardMove represents a planned shard move (shell-specific with EcNode references)
|
||||
type ECShardMove struct {
|
||||
VolumeId needle.VolumeId
|
||||
@@ -136,12 +29,6 @@ type ECShardMove struct {
|
||||
Reason string
|
||||
}
|
||||
|
||||
// String returns a human-readable description
|
||||
func (m ECShardMove) String() string {
|
||||
return fmt.Sprintf("volume %d shard %d: %s -> %s (%s)",
|
||||
m.VolumeId, m.ShardId, m.SourceNode.info.Id, m.DestNode.info.Id, m.Reason)
|
||||
}
|
||||
|
||||
// ProportionalECRebalancer implements proportional shard distribution for shell commands
|
||||
type ProportionalECRebalancer struct {
|
||||
ecNodes []*EcNode
|
||||
@@ -149,133 +36,3 @@ type ProportionalECRebalancer struct {
|
||||
diskType types.DiskType
|
||||
ecConfig distribution.ECConfig
|
||||
}
|
||||
|
||||
// NewProportionalECRebalancer creates a new proportional rebalancer with default EC config
|
||||
func NewProportionalECRebalancer(
|
||||
ecNodes []*EcNode,
|
||||
rp *super_block.ReplicaPlacement,
|
||||
diskType types.DiskType,
|
||||
) *ProportionalECRebalancer {
|
||||
return NewProportionalECRebalancerWithConfig(
|
||||
ecNodes,
|
||||
rp,
|
||||
diskType,
|
||||
distribution.DefaultECConfig(),
|
||||
)
|
||||
}
|
||||
|
||||
// NewProportionalECRebalancerWithConfig creates a rebalancer with custom EC configuration
|
||||
func NewProportionalECRebalancerWithConfig(
|
||||
ecNodes []*EcNode,
|
||||
rp *super_block.ReplicaPlacement,
|
||||
diskType types.DiskType,
|
||||
ecConfig distribution.ECConfig,
|
||||
) *ProportionalECRebalancer {
|
||||
return &ProportionalECRebalancer{
|
||||
ecNodes: ecNodes,
|
||||
replicaPlacement: rp,
|
||||
diskType: diskType,
|
||||
ecConfig: ecConfig,
|
||||
}
|
||||
}
|
||||
|
||||
// PlanMoves generates a plan for moving shards to achieve proportional distribution
|
||||
func (r *ProportionalECRebalancer) PlanMoves(
|
||||
volumeId needle.VolumeId,
|
||||
locations []*EcNode,
|
||||
) ([]ECShardMove, error) {
|
||||
// Build topology analysis
|
||||
analysis := distribution.NewTopologyAnalysis()
|
||||
nodeMap := make(map[string]*EcNode)
|
||||
|
||||
// Add all EC nodes to the analysis (even those without shards)
|
||||
for _, node := range r.ecNodes {
|
||||
nodeId := node.info.Id
|
||||
topoNode := &distribution.TopologyNode{
|
||||
NodeID: nodeId,
|
||||
DataCenter: string(node.dc),
|
||||
Rack: string(node.rack),
|
||||
FreeSlots: node.freeEcSlot,
|
||||
}
|
||||
analysis.AddNode(topoNode)
|
||||
nodeMap[nodeId] = node
|
||||
}
|
||||
|
||||
// Add shard locations from nodes that have shards
|
||||
for _, node := range locations {
|
||||
nodeId := node.info.Id
|
||||
si := findEcVolumeShardsInfo(node, volumeId, r.diskType)
|
||||
for _, shardId := range si.Ids() {
|
||||
analysis.AddShardLocation(distribution.ShardLocation{
|
||||
ShardID: int(shardId),
|
||||
NodeID: nodeId,
|
||||
DataCenter: string(node.dc),
|
||||
Rack: string(node.rack),
|
||||
})
|
||||
}
|
||||
if _, exists := nodeMap[nodeId]; !exists {
|
||||
nodeMap[nodeId] = node
|
||||
}
|
||||
}
|
||||
|
||||
analysis.Finalize()
|
||||
|
||||
// Create rebalancer and plan moves
|
||||
rep := distribution.NewReplicationConfig(r.replicaPlacement)
|
||||
rebalancer := distribution.NewRebalancer(r.ecConfig, rep)
|
||||
|
||||
plan, err := rebalancer.PlanRebalance(analysis)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert distribution moves to shell moves
|
||||
var moves []ECShardMove
|
||||
for _, move := range plan.Moves {
|
||||
srcNode := nodeMap[move.SourceNode.NodeID]
|
||||
destNode := nodeMap[move.DestNode.NodeID]
|
||||
if srcNode == nil || destNode == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
moves = append(moves, ECShardMove{
|
||||
VolumeId: volumeId,
|
||||
ShardId: erasure_coding.ShardId(move.ShardID),
|
||||
SourceNode: srcNode,
|
||||
DestNode: destNode,
|
||||
Reason: move.Reason,
|
||||
})
|
||||
}
|
||||
|
||||
return moves, nil
|
||||
}
|
||||
|
||||
// GetDistributionSummary returns a summary of the planned distribution
|
||||
func GetDistributionSummary(rp *super_block.ReplicaPlacement) string {
|
||||
ec := distribution.DefaultECConfig()
|
||||
rep := distribution.NewReplicationConfig(rp)
|
||||
dist := distribution.CalculateDistribution(ec, rep)
|
||||
return dist.Summary()
|
||||
}
|
||||
|
||||
// GetDistributionSummaryWithConfig returns a summary with custom EC configuration
|
||||
func GetDistributionSummaryWithConfig(rp *super_block.ReplicaPlacement, ecConfig distribution.ECConfig) string {
|
||||
rep := distribution.NewReplicationConfig(rp)
|
||||
dist := distribution.CalculateDistribution(ecConfig, rep)
|
||||
return dist.Summary()
|
||||
}
|
||||
|
||||
// GetFaultToleranceAnalysis returns fault tolerance analysis for the given configuration
|
||||
func GetFaultToleranceAnalysis(rp *super_block.ReplicaPlacement) string {
|
||||
ec := distribution.DefaultECConfig()
|
||||
rep := distribution.NewReplicationConfig(rp)
|
||||
dist := distribution.CalculateDistribution(ec, rep)
|
||||
return dist.FaultToleranceAnalysis()
|
||||
}
|
||||
|
||||
// GetFaultToleranceAnalysisWithConfig returns fault tolerance analysis with custom EC configuration
|
||||
func GetFaultToleranceAnalysisWithConfig(rp *super_block.ReplicaPlacement, ecConfig distribution.ECConfig) string {
|
||||
rep := distribution.NewReplicationConfig(rp)
|
||||
dist := distribution.CalculateDistribution(ecConfig, rep)
|
||||
return dist.FaultToleranceAnalysis()
|
||||
}
|
||||
|
||||
@@ -1,251 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func TestCalculateECDistributionShell(t *testing.T) {
|
||||
// Test the shell wrapper function
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("110")
|
||||
|
||||
dist := CalculateECDistribution(
|
||||
erasure_coding.TotalShardsCount,
|
||||
erasure_coding.ParityShardsCount,
|
||||
rp,
|
||||
)
|
||||
|
||||
if dist.ReplicationConfig.MinDataCenters != 2 {
|
||||
t.Errorf("Expected 2 DCs, got %d", dist.ReplicationConfig.MinDataCenters)
|
||||
}
|
||||
if dist.TargetShardsPerDC != 7 {
|
||||
t.Errorf("Expected 7 shards per DC, got %d", dist.TargetShardsPerDC)
|
||||
}
|
||||
|
||||
t.Log(dist.Summary())
|
||||
}
|
||||
|
||||
func TestAnalyzeVolumeDistributionShell(t *testing.T) {
|
||||
diskType := types.HardDriveType
|
||||
diskTypeKey := string(diskType)
|
||||
|
||||
// Build a topology with unbalanced distribution
|
||||
node1 := &EcNode{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "127.0.0.1:8080",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {
|
||||
Type: diskTypeKey,
|
||||
MaxVolumeCount: 10,
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{
|
||||
Id: 1,
|
||||
Collection: "test",
|
||||
EcIndexBits: 0x3FFF, // All 14 shards
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc1",
|
||||
rack: "rack1",
|
||||
freeEcSlot: 5,
|
||||
}
|
||||
|
||||
node2 := &EcNode{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "127.0.0.1:8081",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {
|
||||
Type: diskTypeKey,
|
||||
MaxVolumeCount: 10,
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{},
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc2",
|
||||
rack: "rack2",
|
||||
freeEcSlot: 10,
|
||||
}
|
||||
|
||||
locations := []*EcNode{node1, node2}
|
||||
volumeId := needle.VolumeId(1)
|
||||
|
||||
analysis := AnalyzeVolumeDistribution(volumeId, locations, diskType)
|
||||
|
||||
shardsByDC := analysis.GetShardsByDC()
|
||||
if shardsByDC["dc1"] != 14 {
|
||||
t.Errorf("Expected 14 shards in dc1, got %d", shardsByDC["dc1"])
|
||||
}
|
||||
|
||||
t.Log(analysis.DetailedString())
|
||||
}
|
||||
|
||||
func TestProportionalRebalancerShell(t *testing.T) {
|
||||
diskType := types.HardDriveType
|
||||
diskTypeKey := string(diskType)
|
||||
|
||||
// Build topology: 2 DCs, 2 racks each, all shards on one node
|
||||
nodes := []*EcNode{
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc1-rack1-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {
|
||||
Type: diskTypeKey,
|
||||
MaxVolumeCount: 10,
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 1, Collection: "test", EcIndexBits: 0x3FFF},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc1", rack: "dc1-rack1", freeEcSlot: 0,
|
||||
},
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc1-rack2-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {Type: diskTypeKey, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
dc: "dc1", rack: "dc1-rack2", freeEcSlot: 10,
|
||||
},
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc2-rack1-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {Type: diskTypeKey, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
dc: "dc2", rack: "dc2-rack1", freeEcSlot: 10,
|
||||
},
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc2-rack2-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {Type: diskTypeKey, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
dc: "dc2", rack: "dc2-rack2", freeEcSlot: 10,
|
||||
},
|
||||
}
|
||||
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("110")
|
||||
rebalancer := NewProportionalECRebalancer(nodes, rp, diskType)
|
||||
|
||||
volumeId := needle.VolumeId(1)
|
||||
moves, err := rebalancer.PlanMoves(volumeId, []*EcNode{nodes[0]})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("PlanMoves failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Planned %d moves", len(moves))
|
||||
for i, move := range moves {
|
||||
t.Logf(" %d. %s", i+1, move.String())
|
||||
}
|
||||
|
||||
// Verify moves to dc2
|
||||
movedToDC2 := 0
|
||||
for _, move := range moves {
|
||||
if move.DestNode.dc == "dc2" {
|
||||
movedToDC2++
|
||||
}
|
||||
}
|
||||
|
||||
if movedToDC2 == 0 {
|
||||
t.Error("Expected some moves to dc2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomECConfigRebalancer(t *testing.T) {
|
||||
diskType := types.HardDriveType
|
||||
diskTypeKey := string(diskType)
|
||||
|
||||
// Test with custom 8+4 EC configuration
|
||||
ecConfig, err := distribution.NewECConfig(8, 4)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create EC config: %v", err)
|
||||
}
|
||||
|
||||
// Build topology for 12 shards (8+4)
|
||||
nodes := []*EcNode{
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc1-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {
|
||||
Type: diskTypeKey,
|
||||
MaxVolumeCount: 10,
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: 1, Collection: "test", EcIndexBits: 0x0FFF}, // 12 shards (bits 0-11)
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dc: "dc1", rack: "dc1-rack1", freeEcSlot: 0,
|
||||
},
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc2-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {Type: diskTypeKey, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
dc: "dc2", rack: "dc2-rack1", freeEcSlot: 10,
|
||||
},
|
||||
{
|
||||
info: &master_pb.DataNodeInfo{
|
||||
Id: "dc3-node1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
diskTypeKey: {Type: diskTypeKey, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
dc: "dc3", rack: "dc3-rack1", freeEcSlot: 10,
|
||||
},
|
||||
}
|
||||
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("200") // 3 DCs
|
||||
rebalancer := NewProportionalECRebalancerWithConfig(nodes, rp, diskType, ecConfig)
|
||||
|
||||
volumeId := needle.VolumeId(1)
|
||||
moves, err := rebalancer.PlanMoves(volumeId, []*EcNode{nodes[0]})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("PlanMoves failed: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Custom 8+4 EC with 200 replication: planned %d moves", len(moves))
|
||||
|
||||
// Get the distribution summary
|
||||
summary := GetDistributionSummaryWithConfig(rp, ecConfig)
|
||||
t.Log(summary)
|
||||
|
||||
analysis := GetFaultToleranceAnalysisWithConfig(rp, ecConfig)
|
||||
t.Log(analysis)
|
||||
}
|
||||
|
||||
func TestGetDistributionSummaryShell(t *testing.T) {
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("110")
|
||||
|
||||
summary := GetDistributionSummary(rp)
|
||||
t.Log(summary)
|
||||
|
||||
if len(summary) == 0 {
|
||||
t.Error("Summary should not be empty")
|
||||
}
|
||||
|
||||
analysis := GetFaultToleranceAnalysis(rp)
|
||||
t.Log(analysis)
|
||||
|
||||
if len(analysis) == 0 {
|
||||
t.Error("Analysis should not be empty")
|
||||
}
|
||||
}
|
||||
@@ -126,17 +126,6 @@ func processEachCmd(cmd string, commandEnv *CommandEnv) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func stripQuotes(s string) string {
|
||||
tokens, unbalanced := parseShellInput(s, false)
|
||||
if unbalanced {
|
||||
return s
|
||||
}
|
||||
if len(tokens) > 0 {
|
||||
return tokens[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func splitCommandLine(line string) []string {
|
||||
tokens, _ := parseShellInput(line, true)
|
||||
return tokens
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitCommandLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
input: `s3.configure -user=test`,
|
||||
expected: []string{`s3.configure`, `-user=test`},
|
||||
},
|
||||
{
|
||||
input: `s3.configure -user=Test_number_004 -account_display_name="Test number 004" -actions=write -apply`,
|
||||
expected: []string{`s3.configure`, `-user=Test_number_004`, `-account_display_name=Test number 004`, `-actions=write`, `-apply`},
|
||||
},
|
||||
{
|
||||
input: `s3.configure -user=Test_number_004 -account_display_name='Test number 004' -actions=write -apply`,
|
||||
expected: []string{`s3.configure`, `-user=Test_number_004`, `-account_display_name=Test number 004`, `-actions=write`, `-apply`},
|
||||
},
|
||||
{
|
||||
input: `s3.configure -flag="a b"c'd e'`,
|
||||
expected: []string{`s3.configure`, `-flag=a bcd e`},
|
||||
},
|
||||
{
|
||||
input: `s3.configure -name="a\"b"`,
|
||||
expected: []string{`s3.configure`, `-name=a"b`},
|
||||
},
|
||||
{
|
||||
input: `s3.configure -path='a\ b'`,
|
||||
expected: []string{`s3.configure`, `-path=a\ b`},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := splitCommandLine(tt.input)
|
||||
if !reflect.DeepEqual(got, tt.expected) {
|
||||
t.Errorf("input: %s\ngot: %v\nwant: %v", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripQuotes(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{input: `"Test number 004"`, expected: `Test number 004`},
|
||||
{input: `'Test number 004'`, expected: `Test number 004`},
|
||||
{input: `-account_display_name="Test number 004"`, expected: `-account_display_name=Test number 004`},
|
||||
{input: `-flag="a"b'c'`, expected: `-flag=abc`},
|
||||
{input: `-name="a\"b"`, expected: `-name=a"b`},
|
||||
{input: `-path='a\ b'`, expected: `-path=a\ b`},
|
||||
{input: `"unbalanced`, expected: `"unbalanced`},
|
||||
{input: `'unbalanced`, expected: `'unbalanced`},
|
||||
{input: `-name="a\"b`, expected: `-name="a\"b`},
|
||||
{input: `trailing\`, expected: `trailing\`},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := stripQuotes(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("input: %s, got: %s, want: %s", tt.input, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlagParsing(t *testing.T) {
|
||||
fs := flag.NewFlagSet("test", flag.ContinueOnError)
|
||||
displayName := fs.String("account_display_name", "", "display name")
|
||||
|
||||
rawArg := `-account_display_name="Test number 004"`
|
||||
args := []string{stripQuotes(rawArg)}
|
||||
err := fs.Parse(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expected := "Test number 004"
|
||||
if *displayName != expected {
|
||||
t.Errorf("got: [%s], want: [%s]", *displayName, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEscapedFlagParsing(t *testing.T) {
|
||||
fs := flag.NewFlagSet("test", flag.ContinueOnError)
|
||||
name := fs.String("name", "", "name")
|
||||
|
||||
rawArg := `-name="a\"b"`
|
||||
args := []string{stripQuotes(rawArg)}
|
||||
err := fs.Parse(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expected := `a"b`
|
||||
if *name != expected {
|
||||
t.Errorf("got: [%s], want: [%s]", *name, expected)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user