gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite needle_map.CompactMap() for more efficient memory usage (#6813)

Browse Source
This commit is contained in:
Lisandro Pin
2025-05-23 16:05:08 +02:00
committed by GitHub
parent f1181f1121
commit 2e1506c31e
2 changed files with 85 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
weed/storage/needle_map/compact_map.go
View File

@@ -16,118 +16,114 @@ type SectionalNeedleId uint32
const SectionalNeedleIdLimit = 1<<32 - 1 const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct { type SectionalNeedleValue struct {
Key SectionalNeedleId Key SectionalNeedleId
OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size Size `comment:"Size of the data portion"` Size Size `comment:"Size of the data portion"`
}
type SectionalNeedleValueExtra struct {
OffsetHigher OffsetHigher OffsetHigher OffsetHigher
} }
type CompactSection struct { type CompactSection struct {
sync.RWMutex sync.RWMutex
values []SectionalNeedleValue values []SectionalNeedleValue
valuesExtra []SectionalNeedleValueExtra overflow Overflow
overflow Overflow start NeedleId
overflowExtra OverflowExtra end NeedleId
start NeedleId counter int
end NeedleId
counter int
} }
type Overflow []SectionalNeedleValue type Overflow []SectionalNeedleValue
type OverflowExtra []SectionalNeedleValueExtra
func NewCompactSection(start NeedleId) *CompactSection { func NewCompactSection(start NeedleId) *CompactSection {
return &CompactSection{ return &CompactSection{
values: make([]SectionalNeedleValue, batch), values: []SectionalNeedleValue{},
valuesExtra: make([]SectionalNeedleValueExtra, batch), overflow: Overflow([]SectionalNeedleValue{}),
overflow: Overflow(make([]SectionalNeedleValue, 0)), start: start,
overflowExtra: OverflowExtra(make([]SectionalNeedleValueExtra, 0)),
start: start,
} }
} }
// return old entry size // return old entry size
func (cs *CompactSection) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) { func (cs *CompactSection) Set(key NeedleId, offset Offset, size Size) (oldOffset Offset, oldSize Size) {
cs.Lock() cs.Lock()
defer cs.Unlock()
if key > cs.end { if key > cs.end {
cs.end = key cs.end = key
} }
skey := SectionalNeedleId(key - cs.start) skey := SectionalNeedleId(key - cs.start)
if i := cs.binarySearchValues(skey); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size // update
//println("key", key, "old size", ret) oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size cs.values[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
} else { return
needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
if needOverflow {
lookBackIndex := cs.counter - 128
if lookBackIndex < 0 {
lookBackIndex = 0
}
if cs.counter < batch && cs.values[lookBackIndex].Key < skey {
// still has capacity and only partially out of order
p := &cs.values[cs.counter]
p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
for x := cs.counter - 1; x >= lookBackIndex; x-- {
if cs.values[x].Key > cs.values[x+1].Key {
cs.values[x], cs.values[x+1] = cs.values[x+1], cs.values[x]
cs.valuesExtra[x], cs.valuesExtra[x+1] = cs.valuesExtra[x+1], cs.valuesExtra[x]
} else {
break
}
}
cs.counter++
} else {
//println("start", cs.start, "counter", cs.counter, "key", key)
if oldValueExtra, oldValue, found := cs.findOverflowEntry(skey); found {
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValueExtra.OffsetHigher, oldValue.OffsetLower, oldValue.Size
}
cs.setOverflowEntry(skey, offset, size)
}
} else {
p := &cs.values[cs.counter]
p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
cs.counter++
}
} }
cs.Unlock()
needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
if !needOverflow {
// non-overflow insert
cs.values = append(cs.values, SectionalNeedleValue{
Key: skey,
OffsetLower: offset.OffsetLower,
Size: size,
OffsetHigher: offset.OffsetHigher,
})
cs.counter++
return
}
lookBackIndex := cs.counter - 128
if lookBackIndex < 0 {
lookBackIndex = 0
}
if cs.counter < batch && cs.values[lookBackIndex].Key < skey {
// still has capacity and only partially out of order
for ; lookBackIndex < cs.counter; lookBackIndex++ {
if cs.values[lookBackIndex].Key >= skey {
break
}
}
cs.values = append(cs.values, SectionalNeedleValue{})
copy(cs.values[lookBackIndex+1:], cs.values[lookBackIndex:])
cs.values[lookBackIndex].Key, cs.values[lookBackIndex].Size = skey, size
cs.values[lookBackIndex].OffsetLower, cs.values[lookBackIndex].OffsetHigher = offset.OffsetLower, offset.OffsetHigher
cs.counter++
return
}
// overflow insert
//println("start", cs.start, "counter", cs.counter, "key", key)
if oldValue, found := cs.findOverflowEntry(skey); found {
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValue.OffsetHigher, oldValue.OffsetLower, oldValue.Size
}
cs.setOverflowEntry(skey, offset, size)
return return
} }
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size Size) { func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size Size) {
needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size} needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size, OffsetHigher: offset.OffsetHigher}
needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: offset.OffsetHigher}
insertCandidate := sort.Search(len(cs.overflow), func(i int) bool { insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= needleValue.Key return cs.overflow[i].Key >= needleValue.Key
}) })
if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key { if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
cs.overflow[insertCandidate] = needleValue cs.overflow[insertCandidate] = needleValue
} else { return
cs.overflow = append(cs.overflow, needleValue)
cs.overflowExtra = append(cs.overflowExtra, needleValueExtra)
for i := len(cs.overflow) - 1; i > insertCandidate; i-- {
cs.overflow[i] = cs.overflow[i-1]
cs.overflowExtra[i] = cs.overflowExtra[i-1]
}
cs.overflow[insertCandidate] = needleValue
cs.overflowExtra[insertCandidate] = needleValueExtra
} }
cs.overflow = append(cs.overflow, SectionalNeedleValue{})
copy(cs.overflow[insertCandidate+1:], cs.overflow[insertCandidate:])
cs.overflow[insertCandidate] = needleValue
} }
func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nve SectionalNeedleValueExtra, nv SectionalNeedleValue, found bool) { func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
foundCandidate := sort.Search(len(cs.overflow), func(i int) bool { foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= key return cs.overflow[i].Key >= key
}) })
if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key { if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
return cs.overflowExtra[foundCandidate], cs.overflow[foundCandidate], true return cs.overflow[foundCandidate], true
} }
return nve, nv, false return nv, false
} }
func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) { func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
@@ -157,7 +153,7 @@ func (cs *CompactSection) Delete(key NeedleId) Size {
cs.values[i].Size = -cs.values[i].Size cs.values[i].Size = -cs.values[i].Size
} }
} }
if _, v, found := cs.findOverflowEntry(skey); found { if v, found := cs.findOverflowEntry(skey); found {
cs.deleteOverflowEntry(skey) cs.deleteOverflowEntry(skey)
ret = v.Size ret = v.Size
} }
@@ -170,12 +166,12 @@ func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
return nil, false return nil, false
} }
skey := SectionalNeedleId(key - cs.start) skey := SectionalNeedleId(key - cs.start)
if ve, v, ok := cs.findOverflowEntry(skey); ok { if v, ok := cs.findOverflowEntry(skey); ok {
nv := toNeedleValue(ve, v, cs) nv := toNeedleValue(v, cs)
return &nv, true return &nv, true
} }
if i := cs.binarySearchValues(skey); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs) nv := toNeedleValue(cs.values[i], cs)
return &nv, true return &nv, true
} }
return nil, false return nil, false
@@ -273,7 +269,7 @@ func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
var i, j int var i, j int
for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values) && j < cs.counter; { for i, j = 0, 0; i < len(cs.overflow) && j < len(cs.values) && j < cs.counter; {
if cs.overflow[i].Key < cs.values[j].Key { if cs.overflow[i].Key < cs.values[j].Key {
if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil { if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@@ -281,7 +277,7 @@ func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
} else if cs.overflow[i].Key == cs.values[j].Key { } else if cs.overflow[i].Key == cs.values[j].Key {
j++ j++
} else { } else {
if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil { if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@@ -289,13 +285,13 @@ func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
} }
} }
for ; i < len(cs.overflow); i++ { for ; i < len(cs.overflow); i++ {
if err := visit(toNeedleValue(cs.overflowExtra[i], cs.overflow[i], cs)); err != nil { if err := visit(toNeedleValue(cs.overflow[i], cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
} }
for ; j < len(cs.values) && j < cs.counter; j++ { for ; j < len(cs.values) && j < cs.counter; j++ {
if err := visit(toNeedleValue(cs.valuesExtra[j], cs.values[j], cs)); err != nil { if err := visit(toNeedleValue(cs.values[j], cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@@ -305,20 +301,19 @@ func (cm *CompactMap) AscendingVisit(visit func(NeedleValue) error) error {
return nil return nil
} }
func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs *CompactSection) NeedleValue { func toNeedleValue(snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
offset := Offset{ offset := Offset{
OffsetHigher: snve.OffsetHigher, OffsetHigher: snv.OffsetHigher,
OffsetLower: snv.OffsetLower, OffsetLower: snv.OffsetLower,
} }
return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size} return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
} }
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) { func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) SectionalNeedleValue {
return SectionalNeedleValue{ return SectionalNeedleValue{
SectionalNeedleId(nv.Key - cs.start), Key: SectionalNeedleId(nv.Key - cs.start),
nv.Offset.OffsetLower, OffsetLower: nv.Offset.OffsetLower,
nv.Size, Size: nv.Size,
}, SectionalNeedleValueExtra{ OffsetHigher: nv.Offset.OffsetHigher,
nv.Offset.OffsetHigher, }
}
} }

2
weed/storage/needle_map/compact_map_test.go
View File

@@ -150,7 +150,7 @@ func TestOverflow(t *testing.T) {
t.Fatalf("expecting 5 entries now: %+v", cs.overflow) t.Fatalf("expecting 5 entries now: %+v", cs.overflow)
} }
_, x, _ := cs.findOverflowEntry(5) x, _ := cs.findOverflowEntry(5)
if x.Key != 5 { if x.Key != 5 {
t.Fatalf("expecting entry 5 now: %+v", x) t.Fatalf("expecting entry 5 now: %+v", x)
} }