Add "weed backup" command.

This is a pre-cursor for asynchronous replication.

go/storage/needle_map.go

@@ -2,7 +2,9 @@ package storage
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"sync"
 
 	"github.com/chrislusf/seaweedfs/go/util"
 )
@@ -26,9 +28,60 @@ type NeedleMapper interface {
 	FileCount() int
 	DeletedCount() int
 	MaxFileKey() uint64
+	IndexFileSize() uint64
+	IndexFileContent() ([]byte, error)
+	IndexFileName() string
+}
+
+type baseNeedleMapper struct {
+	indexFile           *os.File
+	indexFileAccessLock sync.Mutex
+
+	mapMetric
+}
+
+func (nm baseNeedleMapper) IndexFileSize() uint64 {
+	stat, err := nm.indexFile.Stat()
+	if err == nil {
+		return uint64(stat.Size())
+	}
+	return 0
+}
+
+func (nm baseNeedleMapper) IndexFileName() string {
+	return nm.indexFile.Name()
+}
+
+func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
+	key = util.BytesToUint64(bytes[:8])
+	offset = util.BytesToUint32(bytes[8:12])
+	size = util.BytesToUint32(bytes[12:16])
+	return
+}
+func (nm baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error {
+	bytes := make([]byte, 16)
+	util.Uint64toBytes(bytes[0:8], key)
+	util.Uint32toBytes(bytes[8:12], offset)
+	util.Uint32toBytes(bytes[12:16], size)
+
+	nm.indexFileAccessLock.Lock()
+	defer nm.indexFileAccessLock.Unlock()
+	if _, err := nm.indexFile.Seek(0, 2); err != nil {
+		return fmt.Errorf("cannot seek end of indexfile %s: %v",
+			nm.indexFile.Name(), err)
+	}
+	_, err := nm.indexFile.Write(bytes)
+	return err
+}
+func (nm baseNeedleMapper) IndexFileContent() ([]byte, error) {
+	nm.indexFileAccessLock.Lock()
+	defer nm.indexFileAccessLock.Unlock()
+	return ioutil.ReadFile(nm.indexFile.Name())
 }
 
 type mapMetric struct {
+	indexFile *os.File
+
 	DeletionCounter     int    `json:"DeletionCounter"`
 	FileCounter         int    `json:"FileCounter"`
 	DeletionByteCounter uint64 `json:"DeletionByteCounter"`
@@ -36,20 +89,6 @@ type mapMetric struct {
 	MaximumFileKey      uint64 `json:"MaxFileKey"`
 }
 
-func appendToIndexFile(indexFile *os.File,
-	key uint64, offset uint32, size uint32) error {
-	bytes := make([]byte, 16)
-	util.Uint64toBytes(bytes[0:8], key)
-	util.Uint32toBytes(bytes[8:12], offset)
-	util.Uint32toBytes(bytes[12:16], size)
-	if _, err := indexFile.Seek(0, 2); err != nil {
-		return fmt.Errorf("cannot seek end of indexfile %s: %v",
-			indexFile.Name(), err)
-	}
-	_, err := indexFile.Write(bytes)
-	return err
-}
-
 func (mm *mapMetric) logDelete(deletedByteCount uint32) {
 	mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
 	mm.DeletionCounter++
@@ -66,3 +105,19 @@ func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
 		mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
 	}
 }
+
+func (mm mapMetric) ContentSize() uint64 {
+	return mm.FileByteCounter
+}
+func (mm mapMetric) DeletedSize() uint64 {
+	return mm.DeletionByteCounter
+}
+func (mm mapMetric) FileCount() int {
+	return mm.FileCounter
+}
+func (mm mapMetric) DeletedCount() int {
+	return mm.DeletionCounter
+}
+func (mm mapMetric) MaxFileKey() uint64 {
+	return mm.MaximumFileKey
+}