opt: reduce ShardsInfo memory usage with bitmap and sorted slice (#7974)
* opt: reduce ShardsInfo memory usage with bitmap and sorted slice - Replace map[ShardId]*ShardInfo with sorted []ShardInfo slice - Add ShardBits (uint32) bitmap for O(1) existence checks - Use binary search for O(log n) lookups by shard ID - Maintain sorted order for efficient iteration - Add comprehensive unit tests and benchmarks Memory savings: - Map overhead: ~48 bytes per entry eliminated - Pointers: 8 bytes per entry eliminated - Total: ~56 bytes per shard saved Performance improvements: - Has(): O(1) using bitmap - Size(): O(log n) using binary search (was O(1), acceptable tradeoff) - Count(): O(1) using popcount on bitmap - Iteration: Faster due to cache locality * refactor: add methods to ShardBits type - Add Has(), Set(), Clear(), and Count() methods to ShardBits - Simplify ShardsInfo methods by using ShardBits methods - Improves code readability and encapsulation * opt: use ShardBits directly in ShardsCountFromVolumeEcShardInformationMessage Avoid creating a full ShardsInfo object just to count shards. Directly cast vi.EcIndexBits to ShardBits and use Count() method. * opt: use strings.Builder in ShardsInfo.String() for efficiency * refactor: change AsSlice to return []ShardInfo (values instead of pointers) This completes the memory optimization by avoiding unnecessary pointer slices and potential allocations. * refactor: rename ShardsCountFromVolumeEcShardInformationMessage to GetShardCount * fix: prevent deadlock in Add and Subtract methods Copy shards data from 'other' before releasing its lock to avoid potential deadlock when a.Add(b) and b.Add(a) are called concurrently. The previous implementation held other's lock while calling si.Set/Delete, which acquires si's lock. This could deadlock if two goroutines tried to add/subtract each other concurrently. * opt: avoid unnecessary locking in constructor functions ShardsInfoFromVolume and ShardsInfoFromVolumeEcShardInformationMessage now build shards slice and bitmap directly without calling Set(), which acquires a lock on every call. Since the object is local and not yet shared, locking is unnecessary and adds overhead. This improves performance during object construction. * fix: rename 'copy' variable to avoid shadowing built-in function The variable name 'copy' in TestShardsInfo_Copy shadowed the built-in copy() function, which is confusing and bad practice. Renamed to 'siCopy'. * opt: use math/bits.OnesCount32 and reorganize types 1. Replace manual popcount loop with math/bits.OnesCount32 for better performance and idiomatic Go code 2. Move ShardSize type definition to ec_shards_info.go for better code organization since it's primarily used there * refactor: Set() now accepts ShardInfo for future extensibility Changed Set(id ShardId, size ShardSize) to Set(shard ShardInfo) to support future additions to ShardInfo without changing the API. This makes the code more extensible as new fields can be added to ShardInfo (e.g., checksum, location, etc.) without breaking the Set API. * refactor: move ShardInfo and ShardSize to separate file Created ec_shard_info.go to hold the basic shard types (ShardInfo and ShardSize) for better code organization and separation of concerns. * refactor: add ShardInfo constructor and helper functions Added NewShardInfo() constructor and IsValid() method to better encapsulate ShardInfo creation and validation. Updated code to use the constructor for cleaner, more maintainable code. * fix: update remaining Set() calls to use NewShardInfo constructor Fixed compilation errors in storage and shell packages where Set() calls were not updated to use the new NewShardInfo() constructor. * fix: remove unreachable code in filer backup commands Removed unreachable return statements after infinite loops in filer_backup.go and filer_meta_backup.go to fix compilation errors. * fix: rename 'new' variable to avoid shadowing built-in Renamed 'new' to 'result' in MinusParityShards, Plus, and Minus methods to avoid shadowing Go's built-in new() function. * fix: update remaining test files to use NewShardInfo constructor Fixed Set() calls in command_volume_list_test.go and ec_rebalance_slots_test.go to use NewShardInfo() constructor.
This commit is contained in:
Chris Lu
@@ -1,265 +1,10 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
// ShardsInfo encapsulates information for EC shards
|
||||
type ShardSize int64
|
||||
type ShardInfo struct {
|
||||
Id ShardId
|
||||
Size ShardSize
|
||||
}
|
||||
type ShardsInfo struct {
|
||||
shards map[ShardId]*ShardInfo
|
||||
}
|
||||
|
||||
func NewShardsInfo() *ShardsInfo {
|
||||
return &ShardsInfo{
|
||||
shards: map[ShardId]*ShardInfo{},
|
||||
}
|
||||
}
|
||||
|
||||
// Initializes a ShardsInfo from a ECVolume.
|
||||
func ShardsInfoFromVolume(ev *EcVolume) *ShardsInfo {
|
||||
res := &ShardsInfo{
|
||||
shards: map[ShardId]*ShardInfo{},
|
||||
}
|
||||
for _, s := range ev.Shards {
|
||||
res.Set(s.ShardId, ShardSize(s.Size()))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Initializes a ShardsInfo from a VolumeEcShardInformationMessage proto.
|
||||
func ShardsInfoFromVolumeEcShardInformationMessage(vi *master_pb.VolumeEcShardInformationMessage) *ShardsInfo {
|
||||
res := NewShardsInfo()
|
||||
if vi == nil {
|
||||
return res
|
||||
}
|
||||
|
||||
var id ShardId
|
||||
var j int
|
||||
for bitmap := vi.EcIndexBits; bitmap != 0; bitmap >>= 1 {
|
||||
if bitmap&1 != 0 {
|
||||
var size ShardSize
|
||||
if j < len(vi.ShardSizes) {
|
||||
size = ShardSize(vi.ShardSizes[j])
|
||||
}
|
||||
j++
|
||||
res.shards[id] = &ShardInfo{
|
||||
Id: id,
|
||||
Size: size,
|
||||
}
|
||||
}
|
||||
id++
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// Returns a count of shards from a VolumeEcShardInformationMessage proto.
|
||||
func ShardsCountFromVolumeEcShardInformationMessage(vi *master_pb.VolumeEcShardInformationMessage) int {
|
||||
if vi == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return ShardsInfoFromVolumeEcShardInformationMessage(vi).Count()
|
||||
}
|
||||
|
||||
// Returns a string representation for a ShardsInfo.
|
||||
func (sp *ShardsInfo) String() string {
|
||||
var res string
|
||||
ids := sp.Ids()
|
||||
for i, id := range sp.Ids() {
|
||||
res += fmt.Sprintf("%d:%s", id, humanize.Bytes(uint64(sp.shards[id].Size)))
|
||||
if i < len(ids)-1 {
|
||||
res += " "
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// AsSlice converts a ShardsInfo to a slice of ShardInfo structs, ordered by shard ID.
|
||||
func (si *ShardsInfo) AsSlice() []*ShardInfo {
|
||||
res := make([]*ShardInfo, len(si.shards))
|
||||
i := 0
|
||||
for _, id := range si.Ids() {
|
||||
res[i] = si.shards[id]
|
||||
i++
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// Count returns the number of EC shards.
|
||||
func (si *ShardsInfo) Count() int {
|
||||
return len(si.shards)
|
||||
}
|
||||
|
||||
// Has verifies if a shard ID is present.
|
||||
func (si *ShardsInfo) Has(id ShardId) bool {
|
||||
_, ok := si.shards[id]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Ids returns a list of shard IDs, in ascending order.
|
||||
func (si *ShardsInfo) Ids() []ShardId {
|
||||
ids := []ShardId{}
|
||||
for id := range si.shards {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
// IdsInt returns a list of shards ID as int, in ascending order.
|
||||
func (si *ShardsInfo) IdsInt() []int {
|
||||
ids := si.Ids()
|
||||
res := make([]int, len(ids))
|
||||
for i, id := range ids {
|
||||
res[i] = int(id)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// Ids returns a list of shards ID as uint32, in ascending order.
|
||||
func (si *ShardsInfo) IdsUint32() []uint32 {
|
||||
return ShardIdsToUint32(si.Ids())
|
||||
}
|
||||
|
||||
// Set sets the size for a given shard ID.
|
||||
func (si *ShardsInfo) Set(id ShardId, size ShardSize) {
|
||||
if id >= MaxShardCount {
|
||||
return
|
||||
}
|
||||
si.shards[id] = &ShardInfo{
|
||||
Id: id,
|
||||
Size: size,
|
||||
}
|
||||
}
|
||||
|
||||
// Delete deletes a shard by ID.
|
||||
func (si *ShardsInfo) Delete(id ShardId) {
|
||||
if id >= MaxShardCount {
|
||||
return
|
||||
}
|
||||
if _, ok := si.shards[id]; ok {
|
||||
delete(si.shards, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Bitmap returns a bitmap for all existing shard IDs (bit 0 = shard #0... bit 31 = shard #31), in little endian.
|
||||
func (si *ShardsInfo) Bitmap() uint32 {
|
||||
var bits uint32
|
||||
for id := range si.shards {
|
||||
bits |= (1 << id)
|
||||
}
|
||||
return bits
|
||||
}
|
||||
|
||||
// Size returns the size of a given shard ID, if present.
|
||||
func (si *ShardsInfo) Size(id ShardId) ShardSize {
|
||||
if s, ok := si.shards[id]; ok {
|
||||
return s.Size
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// TotalSize returns the size for all shards.
|
||||
func (si *ShardsInfo) TotalSize() ShardSize {
|
||||
var total ShardSize
|
||||
for _, s := range si.shards {
|
||||
total += s.Size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Sizes returns a compact slice of present shard sizes, from first to last.
|
||||
func (si *ShardsInfo) Sizes() []ShardSize {
|
||||
ids := si.Ids()
|
||||
|
||||
res := make([]ShardSize, len(ids))
|
||||
if len(res) != 0 {
|
||||
var i int
|
||||
for _, id := range ids {
|
||||
res[i] = si.shards[id].Size
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// SizesInt64 returns a compact slice of present shard sizes, from first to last, as int64.
|
||||
func (si *ShardsInfo) SizesInt64() []int64 {
|
||||
res := make([]int64, si.Count())
|
||||
|
||||
for i, s := range si.Sizes() {
|
||||
res[i] = int64(s)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Copy creates a copy of a ShardInfo.
|
||||
func (si *ShardsInfo) Copy() *ShardsInfo {
|
||||
new := NewShardsInfo()
|
||||
for _, s := range si.shards {
|
||||
new.Set(s.Id, s.Size)
|
||||
}
|
||||
return new
|
||||
}
|
||||
|
||||
// DeleteParityShards removes party shards from a ShardInfo.
|
||||
// Assumes default 10+4 EC layout where parity shards are IDs 10-13.
|
||||
func (si *ShardsInfo) DeleteParityShards() {
|
||||
for id := DataShardsCount; id < TotalShardsCount; id++ {
|
||||
si.Delete(ShardId(id))
|
||||
}
|
||||
}
|
||||
|
||||
// MinusParityShards creates a ShardInfo copy, but with parity shards removed.
|
||||
func (si *ShardsInfo) MinusParityShards() *ShardsInfo {
|
||||
new := si.Copy()
|
||||
new.DeleteParityShards()
|
||||
return new
|
||||
}
|
||||
|
||||
// Add merges all shards from another ShardInfo into this one.
|
||||
func (si *ShardsInfo) Add(other *ShardsInfo) {
|
||||
for _, s := range other.shards {
|
||||
si.Set(s.Id, s.Size)
|
||||
}
|
||||
}
|
||||
|
||||
// Subtract removes all shards present on another ShardInfo.
|
||||
func (si *ShardsInfo) Subtract(other *ShardsInfo) {
|
||||
for _, s := range other.shards {
|
||||
si.Delete(s.Id)
|
||||
}
|
||||
}
|
||||
|
||||
// Plus returns a new ShardInfo consisting of (this + other).
|
||||
func (si *ShardsInfo) Plus(other *ShardsInfo) *ShardsInfo {
|
||||
new := si.Copy()
|
||||
new.Add(other)
|
||||
return new
|
||||
}
|
||||
|
||||
// Minus returns a new ShardInfo consisting of (this - other).
|
||||
func (si *ShardsInfo) Minus(other *ShardsInfo) *ShardsInfo {
|
||||
new := si.Copy()
|
||||
new.Subtract(other)
|
||||
return new
|
||||
}
|
||||
|
||||
// data structure used in master
|
||||
type EcVolumeInfo struct {
|
||||
VolumeId needle.VolumeId
|
||||
|
||||
Reference in New Issue
Block a user