gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Humanize output for weed.server by default (#7758)

Browse Source 
* Implement a `weed shell` command to return a status overview of the cluster.

Detailed file information will be implemented in a follow-up MR. Note also
that masters are currently not reporting back EC shard sizes correctly, via
`master_pb.VolumeEcShardInformationMessage.shard_sizes`.

F.ex:

```
> status

cluster:
	id:       topo
	status:   LOCKED
	nodes:    10
	topology: 1 DC(s)s, 1 disk(s) on 1 rack(s)

volumes:
	total:    3 volumes on 1 collections
	max size: 31457280000 bytes
	regular:  2/80 volumes on 6 replicas, 6 writable (100.00%), 0 read-only (0.00%)
	EC:       1 EC volumes on 14 shards (14.00 shards/volume)

storage:
	total:           186024424 bytes
	regular volumes: 186024424 bytes
	EC volumes:      0 bytes
	raw:             558073152 bytes on volume replicas, 0 bytes on EC shard files
```

* Humanize output for `weed.server` by default.

Makes things more readable :)

```
> cluster.status

cluster:
	id:       topo
	status:   LOCKED
	nodes:    10
	topology: 1 DC, 10 disks on 1 rack

volumes:
	total:    3 volumes, 1 collection
	max size: 32 GB
	regular:  2/80 volumes on 6 replicas, 6 writable (100%), 0 read-only (0%)
	EC:       1 EC volume on 14 shards (14 shards/volume)

storage:
	total:           172 MB
	regular volumes: 172 MB
	EC volumes:      0 B
	raw:             516 MB on volume replicas, 0 B on EC shards
```

```
> cluster.status --humanize=false

cluster:
	id:       topo
	status:   LOCKED
	nodes:    10
	topology: 1 DC(s), 10 disk(s) on 1 rack(s)

volumes:
	total:    3 volume(s), 1 collection(s)
	max size: 31457280000 byte(s)
	regular:  2/80 volume(s) on 6 replica(s), 5 writable (83.33%), 1 read-only (16.67%)
	EC:       1 EC volume(s) on 14 shard(s) (14.00 shards/volume)

storage:
	total:           172128072 byte(s)
	regular volumes: 172128072 byte(s)
	EC volumes:      0 byte(s)
	raw:             516384216 byte(s) on volume replicas, 0 byte(s) on EC shards
```

Also adds unit tests, and reshuffles test files handling for clarity.
This commit is contained in:
Lisandro Pin
2025-12-15 20:18:45 +01:00
committed by GitHub
parent d1435ead8d
commit 187ef65e8f
8 changed files with 306 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
weed/shell/command_ec_common_test.go
View File

@@ -13,12 +13,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
var (
topology1 = parseOutput(topoData)
topology2 = parseOutput(topoData2)
topologyEc = parseOutput(topoDataEc)
)
func errorCheck(got error, want string) error {
if got == nil && want == "" {
return nil
@@ -42,22 +36,22 @@ func TestCollectCollectionsForVolumeIds(t *testing.T) {
want []string
}{
// normal volumes
{topology1, nil, nil},
{topology1, []needle.VolumeId{}, nil},
{topology1, []needle.VolumeId{needle.VolumeId(9999)}, nil},
{topology1, []needle.VolumeId{needle.VolumeId(2)}, []string{""}},
{topology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272)}, []string{"", "collection2"}},
{topology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272), needle.VolumeId(299)}, []string{"", "collection2"}},
{topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95)}, []string{"collection1", "collection2"}},
{topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51)}, []string{"collection1", "collection2"}},
{topology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51), needle.VolumeId(15)}, []string{"collection0", "collection1", "collection2"}},
{testTopology1, nil, nil},
{testTopology1, []needle.VolumeId{}, nil},
{testTopology1, []needle.VolumeId{needle.VolumeId(9999)}, nil},
{testTopology1, []needle.VolumeId{needle.VolumeId(2)}, []string{""}},
{testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272)}, []string{"", "collection2"}},
{testTopology1, []needle.VolumeId{needle.VolumeId(2), needle.VolumeId(272), needle.VolumeId(299)}, []string{"", "collection2"}},
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95)}, []string{"collection1", "collection2"}},
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51)}, []string{"collection1", "collection2"}},
{testTopology1, []needle.VolumeId{needle.VolumeId(272), needle.VolumeId(299), needle.VolumeId(95), needle.VolumeId(51), needle.VolumeId(15)}, []string{"collection0", "collection1", "collection2"}},
// EC volumes
{topology2, []needle.VolumeId{needle.VolumeId(9577)}, []string{"s3qldata"}},
{topology2, []needle.VolumeId{needle.VolumeId(9577), needle.VolumeId(12549)}, []string{"s3qldata"}},
{testTopology2, []needle.VolumeId{needle.VolumeId(9577)}, []string{"s3qldata"}},
{testTopology2, []needle.VolumeId{needle.VolumeId(9577), needle.VolumeId(12549)}, []string{"s3qldata"}},
// normal + EC volumes
{topology2, []needle.VolumeId{needle.VolumeId(18111)}, []string{"s3qldata"}},
{topology2, []needle.VolumeId{needle.VolumeId(8677)}, []string{"s3qldata"}},
{topology2, []needle.VolumeId{needle.VolumeId(18111), needle.VolumeId(8677)}, []string{"s3qldata"}},
{testTopology2, []needle.VolumeId{needle.VolumeId(18111)}, []string{"s3qldata"}},
{testTopology2, []needle.VolumeId{needle.VolumeId(8677)}, []string{"s3qldata"}},
{testTopology2, []needle.VolumeId{needle.VolumeId(18111), needle.VolumeId(8677)}, []string{"s3qldata"}},
}
for _, tc := range testCases {
@@ -106,7 +100,7 @@ func TestParseReplicaPlacementArg(t *testing.T) {
func TestEcDistribution(t *testing.T) {
// find out all volume servers with one slot left.
ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology1, "", types.HardDriveType)
ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(testTopology1, "", types.HardDriveType)
sortEcNodesByFreeslotsDescending(ecNodes)
@@ -133,18 +127,18 @@ func TestPickRackToBalanceShardsInto(t *testing.T) {
wantErr string
}{
// Non-EC volumes. We don't care about these, but the function should return all racks as a safeguard.
{topologyEc, "", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{topologyEc, "6225", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{topologyEc, "6226", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{topologyEc, "6241", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{topologyEc, "6242", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{testTopologyEc, "", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{testTopologyEc, "6225", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{testTopologyEc, "6226", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{testTopologyEc, "6241", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
{testTopologyEc, "6242", "123", []string{"rack1", "rack2", "rack3", "rack4", "rack5", "rack6"}, ""},
// EC volumes.
{topologyEc, "9577", "", nil, "shards 1 > replica placement limit for other racks (0)"},
{topologyEc, "9577", "111", []string{"rack1", "rack2", "rack3"}, ""},
{topologyEc, "9577", "222", []string{"rack1", "rack2", "rack3"}, ""},
{topologyEc, "10457", "222", []string{"rack1"}, ""},
{topologyEc, "12737", "222", []string{"rack2"}, ""},
{topologyEc, "14322", "222", []string{"rack3"}, ""},
{testTopologyEc, "9577", "", nil, "shards 1 > replica placement limit for other racks (0)"},
{testTopologyEc, "9577", "111", []string{"rack1", "rack2", "rack3"}, ""},
{testTopologyEc, "9577", "222", []string{"rack1", "rack2", "rack3"}, ""},
{testTopologyEc, "10457", "222", []string{"rack1"}, ""},
{testTopologyEc, "12737", "222", []string{"rack2"}, ""},
{testTopologyEc, "14322", "222", []string{"rack3"}, ""},
}
for _, tc := range testCases {
@@ -190,11 +184,11 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) {
wantOneOf []string
wantErr string
}{
{topologyEc, "", "", nil, "INTERNAL: missing source nodes"},
{topologyEc, "idontexist", "12737", nil, "INTERNAL: missing source nodes"},
{testTopologyEc, "", "", nil, "INTERNAL: missing source nodes"},
{testTopologyEc, "idontexist", "12737", nil, "INTERNAL: missing source nodes"},
// Non-EC nodes. We don't care about these, but the function should return all available target nodes as a safeguard.
{
topologyEc, "172.19.0.10:8702", "6225", []string{
testTopologyEc, "172.19.0.10:8702", "6225", []string{
"172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704", "172.19.0.17:8703",
"172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710", "172.19.0.3:8708",
"172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713", "172.19.0.8:8709",
@@ -202,7 +196,7 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) {
"",
},
{
topologyEc, "172.19.0.8:8709", "6226", []string{
testTopologyEc, "172.19.0.8:8709", "6226", []string{
"172.19.0.10:8702", "172.19.0.13:8701", "172.19.0.14:8711", "172.19.0.16:8704",
"172.19.0.17:8703", "172.19.0.19:8700", "172.19.0.20:8706", "172.19.0.21:8710",
"172.19.0.3:8708", "172.19.0.4:8707", "172.19.0.5:8705", "172.19.0.6:8713",
@@ -210,16 +204,16 @@ func TestPickEcNodeToBalanceShardsInto(t *testing.T) {
"",
},
// EC volumes.
{topologyEc, "172.19.0.10:8702", "14322", []string{
{testTopologyEc, "172.19.0.10:8702", "14322", []string{
"172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"},
""},
{topologyEc, "172.19.0.13:8701", "10457", []string{
{testTopologyEc, "172.19.0.13:8701", "10457", []string{
"172.19.0.10:8702", "172.19.0.6:8713"},
""},
{topologyEc, "172.19.0.17:8703", "12737", []string{
{testTopologyEc, "172.19.0.17:8703", "12737", []string{
"172.19.0.13:8701"},
""},
{topologyEc, "172.19.0.20:8706", "14322", []string{
{testTopologyEc, "172.19.0.20:8706", "14322", []string{
"172.19.0.14:8711", "172.19.0.5:8705", "172.19.0.6:8713"},
""},
}
@@ -277,7 +271,7 @@ func TestCountFreeShardSlots(t *testing.T) {
}{
{
name: "topology #1, free HDD shards",
topology: topology1,
topology: testTopology1,
diskType: types.HardDriveType,
want: map[string]int{
"192.168.1.1:8080": 17330,
@@ -289,7 +283,7 @@ func TestCountFreeShardSlots(t *testing.T) {
},
{
name: "topology #1, no free SSD shards available",
topology: topology1,
topology: testTopology1,
diskType: types.SsdType,
want: map[string]int{
"192.168.1.1:8080": 0,
@@ -301,7 +295,7 @@ func TestCountFreeShardSlots(t *testing.T) {
},
{
name: "topology #2, no negative free HDD shards",
topology: topology2,
topology: testTopology2,
diskType: types.HardDriveType,
want: map[string]int{
"172.19.0.3:8708": 0,
@@ -322,7 +316,7 @@ func TestCountFreeShardSlots(t *testing.T) {
},
{
name: "topology #2, no free SSD shards available",
topology: topology2,
topology: testTopology2,
diskType: types.SsdType,
want: map[string]int{
"172.19.0.10:8702": 0,