refactoring

weed/storage/needle_map/btree_map.go

@@ -0,0 +1,53 @@
+package needle_map
+
+import (
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/google/btree"
+)
+
+//This map assumes mostly inserting increasing keys
+type BtreeMap struct {
+	tree *btree.BTree
+}
+
+func NewBtreeMap() *BtreeMap {
+	return &BtreeMap{
+		tree: btree.New(32),
+	}
+}
+
+func (cm *BtreeMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
+	found := cm.tree.ReplaceOrInsert(NeedleValue{key, offset, size})
+	if found != nil {
+		old := found.(NeedleValue)
+		return old.Offset, old.Size
+	}
+	return
+}
+
+func (cm *BtreeMap) Delete(key NeedleId) (oldSize uint32) {
+	found := cm.tree.Delete(NeedleValue{key, Offset{}, 0})
+	if found != nil {
+		old := found.(NeedleValue)
+		return old.Size
+	}
+	return
+}
+func (cm *BtreeMap) Get(key NeedleId) (*NeedleValue, bool) {
+	found := cm.tree.Get(NeedleValue{key, Offset{}, 0})
+	if found != nil {
+		old := found.(NeedleValue)
+		return &old, true
+	}
+	return nil, false
+}
+
+// Visit visits all entries or stop if any error when visiting
+func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) {
+	cm.tree.Ascend(func(item btree.Item) bool {
+		needle := item.(NeedleValue)
+		ret = visit(needle)
+		return ret == nil
+	})
+	return ret
+}

weed/storage/needle_map/compact_map.go

@@ -0,0 +1,288 @@
+package needle_map
+
+import (
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"sort"
+	"sync"
+)
+
+const (
+	batch = 100000
+)
+
+type SectionalNeedleId uint32
+
+const SectionalNeedleIdLimit = 1<<32 - 1
+
+type SectionalNeedleValue struct {
+	Key         SectionalNeedleId
+	OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
+	Size        uint32      `comment:"Size of the data portion"`
+}
+
+type SectionalNeedleValueExtra struct {
+	OffsetHigher OffsetHigher
+}
+
+type CompactSection struct {
+	sync.RWMutex
+	values        []SectionalNeedleValue
+	valuesExtra   []SectionalNeedleValueExtra
+	overflow      Overflow
+	overflowExtra OverflowExtra
+	start         NeedleId
+	end           NeedleId
+	counter       int
+}
+
+type Overflow []SectionalNeedleValue
+type OverflowExtra []SectionalNeedleValueExtra
+
+func NewCompactSection(start NeedleId) *CompactSection {
+	return &CompactSection{
+		values:        make([]SectionalNeedleValue, batch),
+		valuesExtra:   make([]SectionalNeedleValueExtra, batch),
+		overflow:      Overflow(make([]SectionalNeedleValue, 0)),
+		overflowExtra: OverflowExtra(make([]SectionalNeedleValueExtra, 0)),
+		start:         start,
+	}
+}
+
+//return old entry size
+func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
+	cs.Lock()
+	if key > cs.end {
+		cs.end = key
+	}
+	skey := SectionalNeedleId(key - cs.start)
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
+		//println("key", key, "old size", ret)
+		cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
+	} else {
+		needOverflow := cs.counter >= batch
+		needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
+		if needOverflow {
+			//println("start", cs.start, "counter", cs.counter, "key", key)
+			if oldValueExtra, oldValue, found := cs.findOverflowEntry(skey); found {
+				oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValueExtra.OffsetHigher, oldValue.OffsetLower, oldValue.Size
+			}
+			cs.setOverflowEntry(skey, offset, size)
+		} else {
+			p := &cs.values[cs.counter]
+			p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
+			//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
+			cs.counter++
+		}
+	}
+	cs.Unlock()
+	return
+}
+
+func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size uint32) {
+	needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size}
+	needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: OffsetHigher{}}
+	insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
+		return cs.overflow[i].Key >= needleValue.Key
+	})
+	if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
+		cs.overflow[insertCandidate] = needleValue
+	} else {
+		cs.overflow = append(cs.overflow, needleValue)
+		cs.overflowExtra = append(cs.overflowExtra, needleValueExtra)
+		for i := len(cs.overflow) - 1; i > insertCandidate; i-- {
+			cs.overflow[i] = cs.overflow[i-1]
+			cs.overflowExtra[i] = cs.overflowExtra[i-1]
+		}
+		cs.overflow[insertCandidate] = needleValue
+	}
+}
+
+func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nve SectionalNeedleValueExtra, nv SectionalNeedleValue, found bool) {
+	foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
+		return cs.overflow[i].Key >= key
+	})
+	if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
+		return cs.overflowExtra[foundCandidate], cs.overflow[foundCandidate], true
+	}
+	return nve, nv, false
+}
+
+func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
+	length := len(cs.overflow)
+	deleteCandidate := sort.Search(length, func(i int) bool {
+		return cs.overflow[i].Key >= key
+	})
+	if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
+		for i := deleteCandidate; i < length-1; i++ {
+			cs.overflow[i] = cs.overflow[i+1]
+			cs.overflowExtra[i] = cs.overflowExtra[i+1]
+		}
+		cs.overflow = cs.overflow[0 : length-1]
+		cs.overflowExtra = cs.overflowExtra[0 : length-1]
+	}
+}
+
+//return old entry size
+func (cs *CompactSection) Delete(key NeedleId) uint32 {
+	skey := SectionalNeedleId(key - cs.start)
+	cs.Lock()
+	ret := uint32(0)
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize {
+			ret = cs.values[i].Size
+			cs.values[i].Size = TombstoneFileSize
+		}
+	}
+	if _, v, found := cs.findOverflowEntry(skey); found {
+		cs.deleteOverflowEntry(skey)
+		ret = v.Size
+	}
+	cs.Unlock()
+	return ret
+}
+func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
+	cs.RLock()
+	skey := SectionalNeedleId(key - cs.start)
+	if ve, v, ok := cs.findOverflowEntry(skey); ok {
+		cs.RUnlock()
+		nv := toNeedleValue(ve, v, cs)
+		return &nv, true
+	}
+	if i := cs.binarySearchValues(skey); i >= 0 {
+		cs.RUnlock()
+		nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs)
+		return &nv, true
+	}
+	cs.RUnlock()
+	return nil, false
+}
+func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
+	x := sort.Search(cs.counter, func(i int) bool {
+		return cs.values[i].Key >= key
+	})
+	if x == cs.counter {
+		return -1
+	}
+	if cs.values[x].Key > key {
+		return -2
+	}
+	return x
+}
+
+//This map assumes mostly inserting increasing keys
+//This map assumes mostly inserting increasing keys
+type CompactMap struct {
+	list []*CompactSection
+}
+
+func NewCompactMap() *CompactMap {
+	return &CompactMap{}
+}
+
+func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
+		// println(x, "adding to existing", len(cm.list), "sections, starting", key)
+		cs := NewCompactSection(key)
+		cm.list = append(cm.list, cs)
+		x = len(cm.list) - 1
+		//keep compact section sorted by start
+		for x >= 0 {
+			if x > 0 && cm.list[x-1].start > key {
+				cm.list[x] = cm.list[x-1]
+				// println("shift", x, "start", cs.start, "to", x-1)
+				x = x - 1
+			} else {
+				cm.list[x] = cs
+				// println("cs", x, "start", cs.start)
+				break
+			}
+		}
+	}
+	// println(key, "set to section[", x, "].start", cm.list[x].start)
+	return cm.list[x].Set(key, offset, size)
+}
+func (cm *CompactMap) Delete(key NeedleId) uint32 {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return uint32(0)
+	}
+	return cm.list[x].Delete(key)
+}
+func (cm *CompactMap) Get(key NeedleId) (*NeedleValue, bool) {
+	x := cm.binarySearchCompactSection(key)
+	if x < 0 {
+		return nil, false
+	}
+	return cm.list[x].Get(key)
+}
+func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
+	l, h := 0, len(cm.list)-1
+	if h < 0 {
+		return -5
+	}
+	if cm.list[h].start <= key {
+		if cm.list[h].counter < batch || key <= cm.list[h].end {
+			return h
+		}
+		return -4
+	}
+	for l <= h {
+		m := (l + h) / 2
+		if key < cm.list[m].start {
+			h = m - 1
+		} else { // cm.list[m].start <= key
+			if cm.list[m+1].start <= key {
+				l = m + 1
+			} else {
+				return m
+			}
+		}
+	}
+	return -3
+}
+
+// Visit visits all entries or stop if any error when visiting
+func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
+	for _, cs := range cm.list {
+		cs.RLock()
+		for i, v := range cs.overflow {
+			if err := visit(toNeedleValue(cs.overflowExtra[i], v, cs)); err != nil {
+				cs.RUnlock()
+				return err
+			}
+		}
+		for i, v := range cs.values {
+			if i >= cs.counter {
+				break
+			}
+			if _, _, found := cs.findOverflowEntry(v.Key); !found {
+				if err := visit(toNeedleValue(cs.valuesExtra[i], v, cs)); err != nil {
+					cs.RUnlock()
+					return err
+				}
+			}
+		}
+		cs.RUnlock()
+	}
+	return nil
+}
+
+func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
+	offset := Offset{
+		OffsetHigher: snve.OffsetHigher,
+		OffsetLower:  snv.OffsetLower,
+	}
+	return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
+}
+
+func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
+	return SectionalNeedleValue{
+			SectionalNeedleId(nv.Key - cs.start),
+			nv.Offset.OffsetLower,
+			nv.Size,
+		}, SectionalNeedleValueExtra{
+			nv.Offset.OffsetHigher,
+		}
+}

weed/storage/needle_map/compact_map_perf_test.go

@@ -0,0 +1,93 @@
+package needle_map
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+/*
+
+To see the memory usage:
+
+go test -run TestMemoryUsage
+The TotalAlloc section shows the memory increase for each iteration.
+
+go test -run TestMemoryUsage -memprofile=mem.out
+go tool pprof --alloc_space needle.test mem.out
+
+
+*/
+
+func TestMemoryUsage(t *testing.T) {
+
+	var maps []*CompactMap
+	totalRowCount := uint64(0)
+
+	startTime := time.Now()
+	for i := 0; i < 10; i++ {
+		indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
+		if ie != nil {
+			log.Fatalln(ie)
+		}
+		m, rowCount := loadNewNeedleMap(indexFile)
+		maps = append(maps, m)
+		totalRowCount += rowCount
+
+		indexFile.Close()
+
+		PrintMemUsage(totalRowCount)
+		now := time.Now()
+		fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
+		startTime = now
+	}
+
+}
+
+func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
+	m := NewCompactMap()
+	bytes := make([]byte, NeedleEntrySize)
+	rowCount := uint64(0)
+	count, e := file.Read(bytes)
+	for count > 0 && e == nil {
+		for i := 0; i < count; i += NeedleEntrySize {
+			rowCount++
+			key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
+			offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
+			size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
+
+			if !offset.IsZero() {
+				m.Set(NeedleId(key), offset, size)
+			} else {
+				m.Delete(key)
+			}
+		}
+
+		count, e = file.Read(bytes)
+	}
+
+	return m, rowCount
+
+}
+
+func PrintMemUsage(totalRowCount uint64) {
+
+	runtime.GC()
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
+	fmt.Printf("Each %.2f Bytes", float64(m.TotalAlloc)/float64(totalRowCount))
+	fmt.Printf("\tAlloc = %v MiB", bToMb(m.Alloc))
+	fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
+	fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
+	fmt.Printf("\tNumGC = %v", m.NumGC)
+}
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}

weed/storage/needle_map/compact_map_test.go

@@ -0,0 +1,159 @@
+package needle_map
+
+import (
+	"fmt"
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"testing"
+)
+
+func TestOverflow2(t *testing.T) {
+	m := NewCompactMap()
+	m.Set(NeedleId(150088), ToOffset(8), 3000073)
+	m.Set(NeedleId(150073), ToOffset(8), 3000073)
+	m.Set(NeedleId(150089), ToOffset(8), 3000073)
+	m.Set(NeedleId(150076), ToOffset(8), 3000073)
+	m.Set(NeedleId(150124), ToOffset(8), 3000073)
+	m.Set(NeedleId(150137), ToOffset(8), 3000073)
+	m.Set(NeedleId(150147), ToOffset(8), 3000073)
+	m.Set(NeedleId(150145), ToOffset(8), 3000073)
+	m.Set(NeedleId(150158), ToOffset(8), 3000073)
+	m.Set(NeedleId(150162), ToOffset(8), 3000073)
+
+	m.Visit(func(value NeedleValue) error {
+		println("needle key:", value.Key)
+		return nil
+	})
+}
+
+func TestIssue52(t *testing.T) {
+	m := NewCompactMap()
+	m.Set(NeedleId(10002), ToOffset(10002), 10002)
+	if element, ok := m.Get(NeedleId(10002)); ok {
+		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
+	}
+	m.Set(NeedleId(10001), ToOffset(10001), 10001)
+	if element, ok := m.Get(NeedleId(10002)); ok {
+		fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
+	} else {
+		t.Fatal("key 10002 missing after setting 10001")
+	}
+}
+
+func TestCompactMap(t *testing.T) {
+	m := NewCompactMap()
+	for i := uint32(0); i < 100*batch; i += 2 {
+		m.Set(NeedleId(i), ToOffset(int64(i)), i)
+	}
+
+	for i := uint32(0); i < 100*batch; i += 37 {
+		m.Delete(NeedleId(i))
+	}
+
+	for i := uint32(0); i < 10*batch; i += 3 {
+		m.Set(NeedleId(i), ToOffset(int64(i+11)), i+5)
+	}
+
+	//	for i := uint32(0); i < 100; i++ {
+	//		if v := m.Get(Key(i)); v != nil {
+	//			glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
+	//		}
+	//	}
+
+	for i := uint32(0); i < 10*batch; i++ {
+		v, ok := m.Get(NeedleId(i))
+		if i%3 == 0 {
+			if !ok {
+				t.Fatal("key", i, "missing!")
+			}
+			if v.Size != i+5 {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		} else if i%37 == 0 {
+			if ok && v.Size != TombstoneFileSize {
+				t.Fatal("key", i, "should have been deleted needle value", v)
+			}
+		} else if i%2 == 0 {
+			if v.Size != i {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		}
+	}
+
+	for i := uint32(10 * batch); i < 100*batch; i++ {
+		v, ok := m.Get(NeedleId(i))
+		if i%37 == 0 {
+			if ok && v.Size != TombstoneFileSize {
+				t.Fatal("key", i, "should have been deleted needle value", v)
+			}
+		} else if i%2 == 0 {
+			if v == nil {
+				t.Fatal("key", i, "missing")
+			}
+			if v.Size != i {
+				t.Fatal("key", i, "size", v.Size)
+			}
+		}
+	}
+
+}
+
+func TestOverflow(t *testing.T) {
+	cs := NewCompactSection(1)
+
+	cs.setOverflowEntry(1, ToOffset(12), 12)
+	cs.setOverflowEntry(2, ToOffset(12), 12)
+	cs.setOverflowEntry(3, ToOffset(12), 12)
+	cs.setOverflowEntry(4, ToOffset(12), 12)
+	cs.setOverflowEntry(5, ToOffset(12), 12)
+
+	if cs.overflow[2].Key != 3 {
+		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
+	}
+
+	cs.setOverflowEntry(3, ToOffset(24), 24)
+
+	if cs.overflow[2].Key != 3 {
+		t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
+	}
+
+	if cs.overflow[2].Size != 24 {
+		t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
+	}
+
+	cs.deleteOverflowEntry(4)
+
+	if len(cs.overflow) != 4 {
+		t.Fatalf("expecting 4 entries now: %+v", cs.overflow)
+	}
+
+	_, x, _ := cs.findOverflowEntry(5)
+	if x.Key != 5 {
+		t.Fatalf("expecting entry 5 now: %+v", x)
+	}
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+	cs.deleteOverflowEntry(1)
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+	cs.setOverflowEntry(4, ToOffset(44), 44)
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+	cs.setOverflowEntry(1, ToOffset(11), 11)
+
+	for i, x := range cs.overflow {
+		println("overflow[", i, "]:", x.Key)
+	}
+	println()
+
+}

weed/storage/needle_map/needle_value.go

@@ -0,0 +1,17 @@
+package needle_map
+
+import (
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/google/btree"
+)
+
+type NeedleValue struct {
+	Key    NeedleId
+	Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
+	Size   uint32 `comment:"Size of the data portion"`
+}
+
+func (this NeedleValue) Less(than btree.Item) bool {
+	that := than.(NeedleValue)
+	return this.Key < that.Key
+}

weed/storage/needle_map/needle_value_map.go

@@ -0,0 +1,12 @@
+package needle_map
+
+import (
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+type NeedleValueMap interface {
+	Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32)
+	Delete(key NeedleId) uint32
+	Get(key NeedleId) (*NeedleValue, bool)
+	Visit(visit func(NeedleValue) error) error
+}