Admin UI: include ec shard sizes into volume server info (#7071)
* show ec shards on dashboard, show max in its own column * master collect shard size info * master send shard size via VolumeList * change to more efficient shard sizes slice * include ec shard sizes into volume server info * Eliminated Redundant gRPC Calls * much more efficient * Efficient Counting: bits.OnesCount32() uses CPU-optimized instructions to count set bits in O(1) * avoid extra volume list call * simplify * preserve existing shard sizes * avoid hard coded value * Update weed/storage/erasure_coding/ec_volume_info.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update weed/admin/dash/volume_management.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update ec_volume_info.go * address comments * avoid duplicated functions * Update weed/admin/dash/volume_management.go Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * simplify * refactoring * fix compilation --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Chris Lu
@@ -18,6 +18,7 @@ const (
|
||||
DataShardsCount = 10
|
||||
ParityShardsCount = 4
|
||||
TotalShardsCount = DataShardsCount + ParityShardsCount
|
||||
MinTotalDisks = TotalShardsCount/ParityShardsCount + 1
|
||||
ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
|
||||
ErasureCodingSmallBlockSize = 1024 * 1024 // 1MB
|
||||
)
|
||||
|
||||
68
weed/storage/erasure_coding/ec_shard_size_helper.go
Normal file
68
weed/storage/erasure_coding/ec_shard_size_helper.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// GetShardSize returns the size of a specific shard from VolumeEcShardInformationMessage
|
||||
// Returns the size and true if the shard exists, 0 and false if not present
|
||||
func GetShardSize(msg *master_pb.VolumeEcShardInformationMessage, shardId ShardId) (size int64, found bool) {
|
||||
if msg == nil || msg.ShardSizes == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
shardBits := ShardBits(msg.EcIndexBits)
|
||||
index, found := shardBits.ShardIdToIndex(shardId)
|
||||
if !found || index >= len(msg.ShardSizes) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return msg.ShardSizes[index], true
|
||||
}
|
||||
|
||||
// SetShardSize sets the size of a specific shard in VolumeEcShardInformationMessage
|
||||
// Returns true if successful, false if the shard is not present in EcIndexBits
|
||||
func SetShardSize(msg *master_pb.VolumeEcShardInformationMessage, shardId ShardId, size int64) bool {
|
||||
if msg == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
shardBits := ShardBits(msg.EcIndexBits)
|
||||
index, found := shardBits.ShardIdToIndex(shardId)
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
|
||||
// Initialize ShardSizes slice if needed
|
||||
expectedLength := shardBits.ShardIdCount()
|
||||
if msg.ShardSizes == nil {
|
||||
msg.ShardSizes = make([]int64, expectedLength)
|
||||
} else if len(msg.ShardSizes) != expectedLength {
|
||||
// Resize the slice to match the expected length
|
||||
newSizes := make([]int64, expectedLength)
|
||||
copy(newSizes, msg.ShardSizes)
|
||||
msg.ShardSizes = newSizes
|
||||
}
|
||||
|
||||
if index >= len(msg.ShardSizes) {
|
||||
return false
|
||||
}
|
||||
|
||||
msg.ShardSizes[index] = size
|
||||
return true
|
||||
}
|
||||
|
||||
// InitializeShardSizes initializes the ShardSizes slice based on EcIndexBits
|
||||
// This ensures the slice has the correct length for all present shards
|
||||
func InitializeShardSizes(msg *master_pb.VolumeEcShardInformationMessage) {
|
||||
if msg == nil {
|
||||
return
|
||||
}
|
||||
|
||||
shardBits := ShardBits(msg.EcIndexBits)
|
||||
expectedLength := shardBits.ShardIdCount()
|
||||
|
||||
if msg.ShardSizes == nil || len(msg.ShardSizes) != expectedLength {
|
||||
msg.ShardSizes = make([]int64, expectedLength)
|
||||
}
|
||||
}
|
||||
117
weed/storage/erasure_coding/ec_shard_size_helper_test.go
Normal file
117
weed/storage/erasure_coding/ec_shard_size_helper_test.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
func TestShardSizeHelpers(t *testing.T) {
|
||||
// Create a message with shards 0, 2, and 5 present (EcIndexBits = 0b100101 = 37)
|
||||
msg := &master_pb.VolumeEcShardInformationMessage{
|
||||
Id: 123,
|
||||
EcIndexBits: 37, // Binary: 100101, shards 0, 2, 5 are present
|
||||
}
|
||||
|
||||
// Test SetShardSize
|
||||
if !SetShardSize(msg, 0, 1000) {
|
||||
t.Error("Failed to set size for shard 0")
|
||||
}
|
||||
if !SetShardSize(msg, 2, 2000) {
|
||||
t.Error("Failed to set size for shard 2")
|
||||
}
|
||||
if !SetShardSize(msg, 5, 5000) {
|
||||
t.Error("Failed to set size for shard 5")
|
||||
}
|
||||
|
||||
// Test setting size for non-present shard should fail
|
||||
if SetShardSize(msg, 1, 1500) {
|
||||
t.Error("Should not be able to set size for non-present shard 1")
|
||||
}
|
||||
|
||||
// Verify ShardSizes slice has correct length (3 shards)
|
||||
if len(msg.ShardSizes) != 3 {
|
||||
t.Errorf("Expected ShardSizes length 3, got %d", len(msg.ShardSizes))
|
||||
}
|
||||
|
||||
// Test GetShardSize
|
||||
if size, found := GetShardSize(msg, 0); !found || size != 1000 {
|
||||
t.Errorf("Expected shard 0 size 1000, got %d (found: %v)", size, found)
|
||||
}
|
||||
if size, found := GetShardSize(msg, 2); !found || size != 2000 {
|
||||
t.Errorf("Expected shard 2 size 2000, got %d (found: %v)", size, found)
|
||||
}
|
||||
if size, found := GetShardSize(msg, 5); !found || size != 5000 {
|
||||
t.Errorf("Expected shard 5 size 5000, got %d (found: %v)", size, found)
|
||||
}
|
||||
|
||||
// Test getting size for non-present shard
|
||||
if size, found := GetShardSize(msg, 1); found {
|
||||
t.Errorf("Should not find shard 1, but got size %d", size)
|
||||
}
|
||||
|
||||
// Test direct slice access
|
||||
if len(msg.ShardSizes) != 3 {
|
||||
t.Errorf("Expected 3 shard sizes in slice, got %d", len(msg.ShardSizes))
|
||||
}
|
||||
|
||||
expectedSizes := []int64{1000, 2000, 5000} // Ordered by shard ID: 0, 2, 5
|
||||
for i, expectedSize := range expectedSizes {
|
||||
if i < len(msg.ShardSizes) && msg.ShardSizes[i] != expectedSize {
|
||||
t.Errorf("Expected ShardSizes[%d] = %d, got %d", i, expectedSize, msg.ShardSizes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestShardBitsHelpers(t *testing.T) {
|
||||
// Test with EcIndexBits = 37 (binary: 100101, shards 0, 2, 5)
|
||||
shardBits := ShardBits(37)
|
||||
|
||||
// Test ShardIdToIndex
|
||||
if index, found := shardBits.ShardIdToIndex(0); !found || index != 0 {
|
||||
t.Errorf("Expected shard 0 at index 0, got %d (found: %v)", index, found)
|
||||
}
|
||||
if index, found := shardBits.ShardIdToIndex(2); !found || index != 1 {
|
||||
t.Errorf("Expected shard 2 at index 1, got %d (found: %v)", index, found)
|
||||
}
|
||||
if index, found := shardBits.ShardIdToIndex(5); !found || index != 2 {
|
||||
t.Errorf("Expected shard 5 at index 2, got %d (found: %v)", index, found)
|
||||
}
|
||||
|
||||
// Test for non-present shard
|
||||
if index, found := shardBits.ShardIdToIndex(1); found {
|
||||
t.Errorf("Should not find shard 1, but got index %d", index)
|
||||
}
|
||||
|
||||
// Test IndexToShardId
|
||||
if shardId, found := shardBits.IndexToShardId(0); !found || shardId != 0 {
|
||||
t.Errorf("Expected index 0 to be shard 0, got %d (found: %v)", shardId, found)
|
||||
}
|
||||
if shardId, found := shardBits.IndexToShardId(1); !found || shardId != 2 {
|
||||
t.Errorf("Expected index 1 to be shard 2, got %d (found: %v)", shardId, found)
|
||||
}
|
||||
if shardId, found := shardBits.IndexToShardId(2); !found || shardId != 5 {
|
||||
t.Errorf("Expected index 2 to be shard 5, got %d (found: %v)", shardId, found)
|
||||
}
|
||||
|
||||
// Test for invalid index
|
||||
if shardId, found := shardBits.IndexToShardId(3); found {
|
||||
t.Errorf("Should not find shard for index 3, but got shard %d", shardId)
|
||||
}
|
||||
|
||||
// Test EachSetIndex
|
||||
var collectedShards []ShardId
|
||||
shardBits.EachSetIndex(func(shardId ShardId) {
|
||||
collectedShards = append(collectedShards, shardId)
|
||||
})
|
||||
expectedShards := []ShardId{0, 2, 5}
|
||||
if len(collectedShards) != len(expectedShards) {
|
||||
t.Errorf("Expected EachSetIndex to collect %v, got %v", expectedShards, collectedShards)
|
||||
}
|
||||
for i, expected := range expectedShards {
|
||||
if i >= len(collectedShards) || collectedShards[i] != expected {
|
||||
t.Errorf("Expected EachSetIndex to collect %v, got %v", expectedShards, collectedShards)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -227,6 +227,9 @@ func (ev *EcVolume) ToVolumeEcShardInformationMessage(diskId uint32) (messages [
|
||||
}
|
||||
prevVolumeId = s.VolumeId
|
||||
m.EcIndexBits = uint32(ShardBits(m.EcIndexBits).AddShardId(s.ShardId))
|
||||
|
||||
// Add shard size information using the optimized format
|
||||
SetShardSize(m, s.ShardId, s.Size())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
@@ -11,27 +13,51 @@ type EcVolumeInfo struct {
|
||||
Collection string
|
||||
ShardBits ShardBits
|
||||
DiskType string
|
||||
DiskId uint32 // ID of the disk this EC volume is on
|
||||
ExpireAtSec uint64 // ec volume destroy time, calculated from the ec volume was created
|
||||
}
|
||||
|
||||
func NewEcVolumeInfo(diskType string, collection string, vid needle.VolumeId, shardBits ShardBits, expireAtSec uint64, diskId uint32) *EcVolumeInfo {
|
||||
return &EcVolumeInfo{
|
||||
Collection: collection,
|
||||
VolumeId: vid,
|
||||
ShardBits: shardBits,
|
||||
DiskType: diskType,
|
||||
DiskId: diskId,
|
||||
ExpireAtSec: expireAtSec,
|
||||
}
|
||||
DiskId uint32 // ID of the disk this EC volume is on
|
||||
ExpireAtSec uint64 // ec volume destroy time, calculated from the ec volume was created
|
||||
ShardSizes []int64 // optimized: sizes for shards in order of set bits in ShardBits
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) AddShardId(id ShardId) {
|
||||
oldBits := ecInfo.ShardBits
|
||||
ecInfo.ShardBits = ecInfo.ShardBits.AddShardId(id)
|
||||
|
||||
// If shard was actually added, resize ShardSizes array
|
||||
if oldBits != ecInfo.ShardBits {
|
||||
ecInfo.resizeShardSizes(oldBits)
|
||||
}
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) RemoveShardId(id ShardId) {
|
||||
oldBits := ecInfo.ShardBits
|
||||
ecInfo.ShardBits = ecInfo.ShardBits.RemoveShardId(id)
|
||||
|
||||
// If shard was actually removed, resize ShardSizes array
|
||||
if oldBits != ecInfo.ShardBits {
|
||||
ecInfo.resizeShardSizes(oldBits)
|
||||
}
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) SetShardSize(id ShardId, size int64) {
|
||||
ecInfo.ensureShardSizesInitialized()
|
||||
if index, found := ecInfo.ShardBits.ShardIdToIndex(id); found && index < len(ecInfo.ShardSizes) {
|
||||
ecInfo.ShardSizes[index] = size
|
||||
}
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) GetShardSize(id ShardId) (int64, bool) {
|
||||
if index, found := ecInfo.ShardBits.ShardIdToIndex(id); found && index < len(ecInfo.ShardSizes) {
|
||||
return ecInfo.ShardSizes[index], true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) GetTotalSize() int64 {
|
||||
var total int64
|
||||
for _, size := range ecInfo.ShardSizes {
|
||||
total += size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) HasShardId(id ShardId) bool {
|
||||
@@ -48,17 +74,33 @@ func (ecInfo *EcVolumeInfo) ShardIdCount() (count int) {
|
||||
|
||||
func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo {
|
||||
ret := &EcVolumeInfo{
|
||||
VolumeId: ecInfo.VolumeId,
|
||||
Collection: ecInfo.Collection,
|
||||
ShardBits: ecInfo.ShardBits.Minus(other.ShardBits),
|
||||
DiskType: ecInfo.DiskType,
|
||||
VolumeId: ecInfo.VolumeId,
|
||||
Collection: ecInfo.Collection,
|
||||
ShardBits: ecInfo.ShardBits.Minus(other.ShardBits),
|
||||
DiskType: ecInfo.DiskType,
|
||||
DiskId: ecInfo.DiskId,
|
||||
ExpireAtSec: ecInfo.ExpireAtSec,
|
||||
}
|
||||
|
||||
// Initialize optimized ShardSizes for the result
|
||||
ret.ensureShardSizesInitialized()
|
||||
|
||||
// Copy shard sizes for remaining shards
|
||||
retIndex := 0
|
||||
for shardId := ShardId(0); shardId < TotalShardsCount && retIndex < len(ret.ShardSizes); shardId++ {
|
||||
if ret.ShardBits.HasShardId(shardId) {
|
||||
if size, exists := ecInfo.GetShardSize(shardId); exists {
|
||||
ret.ShardSizes[retIndex] = size
|
||||
}
|
||||
retIndex++
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage) {
|
||||
return &master_pb.VolumeEcShardInformationMessage{
|
||||
t := &master_pb.VolumeEcShardInformationMessage{
|
||||
Id: uint32(ecInfo.VolumeId),
|
||||
EcIndexBits: uint32(ecInfo.ShardBits),
|
||||
Collection: ecInfo.Collection,
|
||||
@@ -66,6 +108,12 @@ func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.
|
||||
ExpireAtSec: ecInfo.ExpireAtSec,
|
||||
DiskId: ecInfo.DiskId,
|
||||
}
|
||||
|
||||
// Directly set the optimized ShardSizes
|
||||
t.ShardSizes = make([]int64, len(ecInfo.ShardSizes))
|
||||
copy(t.ShardSizes, ecInfo.ShardSizes)
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
type ShardBits uint32 // use bits to indicate the shard id, use 32 bits just for possible future extension
|
||||
@@ -121,3 +169,81 @@ func (b ShardBits) MinusParityShards() ShardBits {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// ShardIdToIndex converts a shard ID to its index position in the ShardSizes slice
|
||||
// Returns the index and true if the shard is present, -1 and false if not present
|
||||
func (b ShardBits) ShardIdToIndex(shardId ShardId) (index int, found bool) {
|
||||
if !b.HasShardId(shardId) {
|
||||
return -1, false
|
||||
}
|
||||
|
||||
// Create a mask for bits before the shardId
|
||||
mask := uint32((1 << shardId) - 1)
|
||||
// Count set bits before the shardId using efficient bit manipulation
|
||||
index = bits.OnesCount32(uint32(b) & mask)
|
||||
return index, true
|
||||
}
|
||||
|
||||
// EachSetIndex iterates over all set shard IDs and calls the provided function for each
|
||||
// This is highly efficient using bit manipulation - only iterates over actual set bits
|
||||
func (b ShardBits) EachSetIndex(fn func(shardId ShardId)) {
|
||||
bitsValue := uint32(b)
|
||||
for bitsValue != 0 {
|
||||
// Find the position of the least significant set bit
|
||||
shardId := ShardId(bits.TrailingZeros32(bitsValue))
|
||||
fn(shardId)
|
||||
// Clear the least significant set bit
|
||||
bitsValue &= bitsValue - 1
|
||||
}
|
||||
}
|
||||
|
||||
// IndexToShardId converts an index position in ShardSizes slice to the corresponding shard ID
|
||||
// Returns the shard ID and true if valid index, -1 and false if invalid index
|
||||
func (b ShardBits) IndexToShardId(index int) (shardId ShardId, found bool) {
|
||||
if index < 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
currentIndex := 0
|
||||
for i := ShardId(0); i < TotalShardsCount; i++ {
|
||||
if b.HasShardId(i) {
|
||||
if currentIndex == index {
|
||||
return i, true
|
||||
}
|
||||
currentIndex++
|
||||
}
|
||||
}
|
||||
return 0, false // index out of range
|
||||
}
|
||||
|
||||
// Helper methods for EcVolumeInfo to manage the optimized ShardSizes slice
|
||||
func (ecInfo *EcVolumeInfo) ensureShardSizesInitialized() {
|
||||
expectedLength := ecInfo.ShardBits.ShardIdCount()
|
||||
if ecInfo.ShardSizes == nil {
|
||||
ecInfo.ShardSizes = make([]int64, expectedLength)
|
||||
} else if len(ecInfo.ShardSizes) != expectedLength {
|
||||
// Resize and preserve existing data
|
||||
ecInfo.resizeShardSizes(ecInfo.ShardBits)
|
||||
}
|
||||
}
|
||||
|
||||
func (ecInfo *EcVolumeInfo) resizeShardSizes(prevShardBits ShardBits) {
|
||||
expectedLength := ecInfo.ShardBits.ShardIdCount()
|
||||
newSizes := make([]int64, expectedLength)
|
||||
|
||||
// Copy existing sizes to new positions based on current ShardBits
|
||||
if len(ecInfo.ShardSizes) > 0 {
|
||||
newIndex := 0
|
||||
for shardId := ShardId(0); shardId < TotalShardsCount && newIndex < expectedLength; shardId++ {
|
||||
if ecInfo.ShardBits.HasShardId(shardId) {
|
||||
// Try to find the size for this shard in the old array using previous ShardBits
|
||||
if oldIndex, found := prevShardBits.ShardIdToIndex(shardId); found && oldIndex < len(ecInfo.ShardSizes) {
|
||||
newSizes[newIndex] = ecInfo.ShardSizes[oldIndex]
|
||||
}
|
||||
newIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ecInfo.ShardSizes = newSizes
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user