Add "weed backup" command.

This is a pre-cursor for asynchronous replication.

go/weed/weed_server/volume_server.go

@@ -51,6 +51,9 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler))
 	adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler))
 	adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler))
+	adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler))
+	adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler))
+	adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler))
 	adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
 	adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
 	adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))

go/weed/weed_server/volume_server_handlers_read.go

@@ -47,7 +47,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	cookie := n.Cookie
-	count, e := vs.store.Read(volumeId, n)
+	count, e := vs.store.ReadVolumeNeedle(volumeId, n)
 	glog.V(4).Infoln("read bytes", count, "error", e)
 	if e != nil || count <= 0 {
 		glog.V(0).Infoln("read error:", e, r.URL.Path)

go/weed/weed_server/volume_server_handlers_sync.go

@@ -0,0 +1,86 @@
+package weed_server
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/chrislusf/seaweedfs/go/glog"
+	"github.com/chrislusf/seaweedfs/go/storage"
+	"github.com/chrislusf/seaweedfs/go/util"
+)
+
+func (vs *VolumeServer) getVolumeSyncStatusHandler(w http.ResponseWriter, r *http.Request) {
+	v, err := vs.getVolume("volume", r)
+	if v == nil {
+		writeJsonError(w, r, http.StatusBadRequest, err)
+		return
+	}
+	syncStat := v.GetVolumeSyncStatus()
+	if syncStat.Error != "" {
+		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Get Volume %d status error: %s", v.Id, syncStat.Error))
+		glog.V(2).Infoln("getVolumeSyncStatusHandler volume =", r.FormValue("volume"), ", error =", err)
+	} else {
+		writeJsonQuiet(w, r, http.StatusOK, syncStat)
+	}
+}
+
+func (vs *VolumeServer) getVolumeIndexContentHandler(w http.ResponseWriter, r *http.Request) {
+	v, err := vs.getVolume("volume", r)
+	if v == nil {
+		writeJsonError(w, r, http.StatusBadRequest, err)
+		return
+	}
+	content, err := v.IndexFileContent()
+	if err != nil {
+		writeJsonError(w, r, http.StatusInternalServerError, err)
+		return
+	}
+	w.Write(content)
+}
+
+func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *http.Request) {
+	v, err := vs.getVolume("volume", r)
+	if v == nil {
+		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("Not Found volume: %v", err))
+		return
+	}
+	if int(v.SuperBlock.CompactRevision) != util.ParseInt(r.FormValue("revision"), 0) {
+		writeJsonError(w, r, http.StatusExpectationFailed, fmt.Errorf("Requested Volume Revision is %s, but current revision is %d", r.FormValue("revision"), v.SuperBlock.CompactRevision))
+		return
+	}
+	offset := uint32(util.ParseUint64(r.FormValue("offset"), 0))
+	size := uint32(util.ParseUint64(r.FormValue("size"), 0))
+	content, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*storage.NeedlePaddingSize, size)
+	if err != nil {
+		writeJsonError(w, r, http.StatusInternalServerError, err)
+		return
+	}
+
+	id := util.ParseUint64(r.FormValue("id"), 0)
+	n := new(storage.Needle)
+	n.ParseNeedleHeader(content)
+	if id != n.Id {
+		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id))
+		return
+	}
+
+	w.Write(content)
+}
+
+func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) {
+	volumeIdString := r.FormValue(volumeParameterName)
+	if volumeIdString == "" {
+		err := fmt.Errorf("Empty Volume Id: Need to pass in %s=the_volume_id.", volumeParameterName)
+		return nil, err
+	}
+	vid, err := storage.NewVolumeId(volumeIdString)
+	if err != nil {
+		err = fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
+		return nil, err
+	}
+	v := vs.store.GetVolume(vid)
+	if v == nil {
+		return nil, fmt.Errorf("Not Found Volume Id %s: %d", volumeIdString, vid)
+	}
+	return v, nil
+}

go/weed/weed_server/volume_server_handlers_write.go

@@ -53,7 +53,7 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 	glog.V(2).Infoln("deleting", n)
 
 	cookie := n.Cookie
-	count, ok := vs.store.Read(volumeId, n)
+	count, ok := vs.store.ReadVolumeNeedle(volumeId, n)
 
 	if ok != nil {
 		m := make(map[string]uint32)
@@ -94,7 +94,7 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques
 		n.ParsePath(id_cookie)
 		glog.V(4).Infoln("batch deleting", n)
 		cookie := n.Cookie
-		if _, err := vs.store.Read(volumeId, n); err != nil {
+		if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
 			ret = append(ret, operation.DeleteResult{Fid: fid, Error: err.Error()})
 			continue
 		}