Add "weed backup" command.

This is a pre-cursor for asynchronous replication.

go/storage/volume.go

@@ -54,6 +54,9 @@ func (v *Volume) FileName() (fileName string) {
 	}
 	return
 }
+func (v *Volume) DataFile() *os.File {
+	return v.dataFile
+}
 func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind NeedleMapType) error {
 	var e error
 	fileName := v.FileName()
@@ -152,7 +155,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
 	nv, ok := v.nm.Get(n.Id)
 	if ok && nv.Offset > 0 {
 		oldNeedle := new(Needle)
-		_, err := oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
+		err := oldNeedle.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
 		if err != nil {
 			glog.V(0).Infof("Failed to check updated file %v", err)
 			return false
@@ -180,6 +183,30 @@ func (v *Volume) Destroy() (err error) {
 	return
 }
 
+// AppendBlob append a blob to end of the data file, used in replication
+func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
+	if v.readOnly {
+		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
+		return
+	}
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+	if offset, err = v.dataFile.Seek(0, 2); err != nil {
+		glog.V(0).Infof("failed to seek the end of file: %v", err)
+		return
+	}
+	//ensure file writing starting from aligned positions
+	if offset%NeedlePaddingSize != 0 {
+		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
+		if offset, err = v.dataFile.Seek(offset, 0); err != nil {
+			glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err)
+			return
+		}
+	}
+	v.dataFile.Write(b)
+	return
+}
+
 func (v *Volume) write(n *Needle) (size uint32, err error) {
 	glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
 	if v.readOnly {
@@ -250,17 +277,19 @@ func (v *Volume) delete(n *Needle) (uint32, error) {
 	return 0, nil
 }
 
-func (v *Volume) read(n *Needle) (int, error) {
+// read fills in Needle content by looking up n.Id from NeedleMapper
+func (v *Volume) readNeedle(n *Needle) (int, error) {
 	nv, ok := v.nm.Get(n.Id)
 	if !ok || nv.Offset == 0 {
 		return -1, errors.New("Not Found")
 	}
-	bytesRead, err := n.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
+	err := n.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
 	if err != nil {
-		return bytesRead, err
+		return 0, err
 	}
+	bytesRead := len(n.Data)
 	if !n.HasTtl() {
-		return bytesRead, err
+		return bytesRead, nil
 	}
 	ttlMinutes := n.Ttl.Minutes()
 	if ttlMinutes == 0 {