add btree for volume index

weed/storage/needle_map_memory.go

@@ -5,17 +5,26 @@ import (
 	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 type NeedleMap struct {
-	m CompactMap
+	m needle.NeedleValueMap
 
 	baseNeedleMapper
 }
 
-func NewNeedleMap(file *os.File) *NeedleMap {
+func NewCompactNeedleMap(file *os.File) *NeedleMap {
 	nm := &NeedleMap{
-		m: NewCompactMap(),
+		m: needle.NewCompactMap(),
+	}
+	nm.indexFile = file
+	return nm
+}
+
+func NewBtreeNeedleMap(file *os.File) *NeedleMap {
+	nm := &NeedleMap{
+		m: needle.NewBtreeMap(),
 	}
 	nm.indexFile = file
 	return nm
@@ -25,8 +34,17 @@ const (
 	RowsToRead = 1024
 )
 
-func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
-	nm := NewNeedleMap(file)
+func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
+	nm := NewCompactNeedleMap(file)
+	return doLoading(file, nm)
+}
+
+func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
+	nm := NewBtreeNeedleMap(file)
+	return doLoading(file, nm)
+}
+
+func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
 	e := WalkIndexFile(file, func(key uint64, offset, size uint32) error {
 		if key > nm.MaximumFileKey {
 			nm.MaximumFileKey = key
@@ -34,14 +52,14 @@ func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
 		if offset > 0 && size != TombstoneFileSize {
 			nm.FileCounter++
 			nm.FileByteCounter = nm.FileByteCounter + uint64(size)
-			oldOffset, oldSize := nm.m.Set(Key(key), offset, size)
+			oldOffset, oldSize := nm.m.Set(needle.Key(key), offset, size)
 			glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
 			if oldOffset > 0 && oldSize != TombstoneFileSize {
 				nm.DeletionCounter++
 				nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
 			}
 		} else {
-			oldSize := nm.m.Delete(Key(key))
+			oldSize := nm.m.Delete(needle.Key(key))
 			glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize)
 			nm.DeletionCounter++
 			nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
@@ -84,16 +102,16 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e
 }
 
 func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error {
-	_, oldSize := nm.m.Set(Key(key), offset, size)
+	_, oldSize := nm.m.Set(needle.Key(key), offset, size)
 	nm.logPut(key, oldSize, size)
 	return nm.appendToIndexFile(key, offset, size)
 }
-func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
-	element, ok = nm.m.Get(Key(key))
+func (nm *NeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) {
+	element, ok = nm.m.Get(needle.Key(key))
 	return
 }
 func (nm *NeedleMap) Delete(key uint64, offset uint32) error {
-	deletedBytes := nm.m.Delete(Key(key))
+	deletedBytes := nm.m.Delete(needle.Key(key))
 	nm.logDelete(deletedBytes)
 	return nm.appendToIndexFile(key, offset, TombstoneFileSize)
 }