add cmd/dump - a dumper

Walk needed to be added to NeedleMap and CompactMap, to be able to add WalkKeys and WalkValues to volume. This is needed for iterating through all the stored needles in a volume - this was dump's purpose.

weed-fs/src/cmd/dump/main.go

@@ -0,0 +1,96 @@
+// Copyright Tamás Gulácsi 2013 All rights reserved
+// Use of this source is governed by the same rules as the weed-fs library.
+// If this would be ambigous, than Apache License 2.0 has to be used.
+//
+// dump dumps the files of a volume to tar or unique files.
+// Each file will have id#mimetype#original_name file format
+
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"flag"
+	"fmt"
+	// "io"
+	"log"
+	"os"
+	"pkg/storage"
+	"strings"
+	"time"
+)
+
+var (
+	volumePath = flag.String("dir", "/tmp", "volume directory")
+	volumeId   = flag.Int("id", 0, "volume Id")
+	dest       = flag.String("out", "-", "output path. Produces tar if path ends with .tar; creates files otherwise.")
+	tarFh      *tar.Writer
+	tarHeader  tar.Header
+	counter    int
+)
+
+func main() {
+	var err error
+
+	flag.Parse()
+
+	if *dest == "-" {
+		*dest = ""
+	}
+	if *dest == "" || strings.HasSuffix(*dest, ".tar") {
+		var fh *os.File
+		if *dest == "" {
+			fh = os.Stdout
+		} else {
+			if fh, err = os.Create(*dest); err != nil {
+				log.Printf("cannot open output tar %s: %s", *dest, err)
+				return
+			}
+		}
+		defer fh.Close()
+		tarFh = tar.NewWriter(fh)
+		defer tarFh.Close()
+		t := time.Now()
+		tarHeader = tar.Header{Mode: 0644,
+			ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
+			Typeflag:   tar.TypeReg,
+			AccessTime: t, ChangeTime: t}
+	}
+
+	v, err := storage.NewVolume(*volumePath, storage.VolumeId(*volumeId), storage.CopyNil)
+	if v == nil || v.Version() == 0 || err != nil {
+		log.Printf("cannot load volume %d from %s (%s): %s", *volumeId, *volumePath, v, err)
+		return
+	}
+	log.Printf("volume: %s (ver. %d)", v, v.Version())
+	if err := v.WalkValues(walker); err != nil {
+		log.Printf("error while walking: %s", err)
+		return
+	}
+
+	log.Printf("%d files written.", counter)
+}
+
+func walker(n *storage.Needle) (err error) {
+	// log.Printf("Id=%d Size=%d Name=%s mime=%s", n.Id, n.Size, n.Name, n.Mime)
+	nm := fmt.Sprintf("%d#%s#%s", n.Id, bytes.Replace(n.Mime, []byte{'/'}, []byte{'_'}, -1), n.Name)
+	// log.Print(nm)
+	if tarFh != nil {
+		tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
+		if err = tarFh.WriteHeader(&tarHeader); err != nil {
+			return err
+		}
+		_, err = tarFh.Write(n.Data)
+	} else {
+		if fh, e := os.Create(*dest + "/" + nm); e != nil {
+			return e
+		} else {
+			defer fh.Close()
+			_, err = fh.Write(n.Data)
+		}
+	}
+	if err == nil {
+		counter++
+	}
+	return
+}

weed-fs/src/cmd/weed/command.go

@@ -24,7 +24,6 @@ type Command struct {
 
 	// Flag is a set of flags specific to this command.
 	Flag flag.FlagSet
-
 }
 
 // Name returns the command's name: the first word in the usage line.

weed-fs/src/cmd/weed/shell.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"bufio"
-  "os"
 	"fmt"
+	"os"
 )
 
 func init() {
@@ -18,8 +18,7 @@ var cmdShell = &Command{
   `,
 }
 
-var (
+var ()
-)
 
 func runShell(command *Command, args []string) bool {
 	r := bufio.NewReader(os.Stdin)
@@ -28,11 +27,11 @@ func runShell(command *Command, args []string) bool {
 	prompt := func() {
 		o.WriteString("> ")
 		o.Flush()
-  };
+	}
 	readLine := func() string {
 		ret, err := r.ReadString('\n')
 		if err != nil {
-       fmt.Fprint(e,err); 
+			fmt.Fprint(e, err)
 			os.Exit(1)
 		}
 		return ret

weed-fs/src/pkg/directory/file_id.go

@@ -3,8 +3,8 @@ package directory
 import (
 	"encoding/hex"
 	"pkg/storage"
-	"strings"
 	"pkg/util"
+	"strings"
 )
 
 type FileId struct {

weed-fs/src/pkg/operation/delete_content.go

@@ -1,8 +1,8 @@
 package operation
 
 import (
-	"net/http"
 	"log"
+	"net/http"
 )
 
 func Delete(url string) error {

weed-fs/src/pkg/operation/lookup_volume_id.go

@@ -2,11 +2,11 @@ package operation
 
 import (
 	"encoding/json"
+	"errors"
+	_ "fmt"
 	"net/url"
 	"pkg/storage"
 	"pkg/util"
-  _ "fmt"
-  "errors"
 )
 
 type Location struct {

weed-fs/src/pkg/operation/upload_content.go

@@ -3,13 +3,13 @@ package operation
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	_ "fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"mime/multipart"
 	"net/http"
-	"errors"
 )
 
 type UploadResult struct {

weed-fs/src/pkg/replication/volume_growth_test.go

@@ -127,4 +127,3 @@ func TestReserveOneVolume(t *testing.T) {
 		t.Log("reserved", c)
 	}
 }
-

weed-fs/src/pkg/sequence/sequence.go

@@ -2,10 +2,10 @@ package sequence
 
 import (
 	"encoding/gob"
+	"log"
 	"os"
 	"path"
 	"sync"
-	"log"
 )
 
 const (

weed-fs/src/pkg/storage/compact_map.go

@@ -175,3 +175,23 @@ func (cm *CompactMap) Peek() {
 		}
 	}
 }
+
+// iterate over the keys by calling iterate on each key till error is returned
+func (cm *CompactMap) Walk(pedestrian func(*NeedleValue) error) (err error) {
+	var i int
+	for _, cs := range cm.list {
+		for key := cs.start; key < cs.end; key++ {
+			if i = cs.binarySearchValues(key); i >= 0 {
+				if err = pedestrian(&cs.values[i]); err != nil {
+					return
+				}
+			}
+		}
+		for _, val := range cs.overflow {
+			if err = pedestrian(val); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

weed-fs/src/pkg/storage/compact_map_perf_test.go

@@ -1,10 +1,10 @@
 package storage
 
 import (
-	"testing"
 	"log"
 	"os"
 	"pkg/util"
+	"testing"
 )
 
 func TestMemoryUsage(t *testing.T) {

weed-fs/src/pkg/storage/needle_map.go

@@ -98,3 +98,8 @@ func (nm *NeedleMap) Close() {
 func (nm *NeedleMap) ContentSize() uint64 {
 	return nm.fileByteCounter
 }
+
+// iterate through all needles using the iterator function
+func (nm *NeedleMap) Walk(pedestrian func(*NeedleValue) error) (err error) {
+	return nm.m.Walk(pedestrian)
+}

weed-fs/src/pkg/storage/needle_read_write.go

@@ -2,10 +2,10 @@ package storage
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"pkg/util"
-	"fmt"
 )
 
 func (n *Needle) Append(w io.Writer, version Version) uint32 {
@@ -62,7 +62,8 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
 	return n.Size
 }
 func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
-	if version == Version1 {
+	switch version {
+	case Version1:
 		bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
 		ret, e := r.Read(bytes)
 		n.readNeedleHeader(bytes)
@@ -72,7 +73,7 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
 			return 0, errors.New("CRC error! Data On Disk Corrupted!")
 		}
 		return ret, e
-	} else if version == Version2 {
+	case Version2:
 		if size == 0 {
 			return 0, nil
 		}
@@ -95,7 +96,7 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
 		}
 		return ret, e
 	}
-	return 0, errors.New("Unsupported Version!")
+	return 0, fmt.Errorf("Unsupported Version! (%d)", version)
 }
 func (n *Needle) readNeedleHeader(bytes []byte) {
 	n.Cookie = util.BytesToUint32(bytes[0:4])

weed-fs/src/pkg/storage/store.go

@@ -65,13 +65,13 @@ func (s *Store) AddVolume(volumeListString string, replicationType string) error
 	}
 	return e
 }
-func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
+func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) (err error) {
 	if s.volumes[vid] != nil {
 		return errors.New("Volume Id " + vid.String() + " already exists!")
 	}
 	log.Println("In dir", s.dir, "adds volume =", vid, ", replicationType =", replicationType)
-	s.volumes[vid] = NewVolume(s.dir, vid, replicationType)
+	s.volumes[vid], err = NewVolume(s.dir, vid, replicationType)
-	return nil
+	return err
 }
 
 func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
@@ -107,7 +107,7 @@ func (s *Store) loadExistingVolumes() {
 				base := name[:len(name)-len(".dat")]
 				if vid, err := NewVolumeId(base); err == nil {
 					if s.volumes[vid] == nil {
-						v := NewVolume(s.dir, vid, CopyNil)
+						if v, e := NewVolume(s.dir, vid, CopyNil); e == nil {
 							s.volumes[vid] = v
 							log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.replicaType, "version =", v.version, "size =", v.Size())
 						}
@@ -116,6 +116,7 @@ func (s *Store) loadExistingVolumes() {
 			}
 		}
 	}
+}
 func (s *Store) Status() []*VolumeInfo {
 	var stats []*VolumeInfo
 	for k, v := range s.volumes {

weed-fs/src/pkg/storage/volume.go

@@ -24,9 +24,9 @@ type Volume struct {
 	accessLock sync.Mutex
 }
 
-func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
+func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
 	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
-	v.load()
+	e = v.load()
 	return
 }
 func (v *Volume) load() error {
@@ -43,6 +43,7 @@ func (v *Volume) load() error {
 	} else {
 		v.maybeWriteSuperBlock()
 	}
+	// TODO: if .idx not exists, but .cdb exists, then use (but don't load!) that
 	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
 	if ie != nil {
 		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
@@ -79,21 +80,23 @@ func (v *Volume) maybeWriteSuperBlock() {
 		v.dataFile.Write(header)
 	}
 }
-func (v *Volume) readSuperBlock() error {
+func (v *Volume) readSuperBlock() (err error) {
 	v.dataFile.Seek(0, 0)
 	header := make([]byte, SuperBlockSize)
 	if _, e := v.dataFile.Read(header); e != nil {
 		return fmt.Errorf("cannot read superblock: %s", e)
 	}
-	var err error
 	v.version, v.replicaType, err = ParseSuperBlock(header)
 	return err
 }
-func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, e error) {
+func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, err error) {
 	version = Version(header[0])
-	var err error
+	if version == 0 {
+		err = errors.New("Zero version impossible - bad superblock!")
+		return
+	}
 	if replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
-		e = fmt.Errorf("cannot read replica type: %s", err)
+		err = fmt.Errorf("cannot read replica type: %s", err)
 	}
 	return
 }
@@ -221,3 +224,39 @@ func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string)
 func (v *Volume) ContentSize() uint64 {
 	return v.nm.fileByteCounter
 }
+
+// Walk over the contained needles (call the function with each NeedleValue till error is returned)
+func (v *Volume) WalkValues(pedestrian func(*Needle) error) error {
+	pedplus := func(nv *NeedleValue) (err error) {
+		n := new(Needle)
+		if nv.Offset > 0 {
+			v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0)
+			if _, err = n.Read(v.dataFile, nv.Size, v.version); err != nil {
+				return
+			}
+			if err = pedestrian(n); err != nil {
+				return
+			}
+		}
+		return nil
+	}
+	return v.nm.Walk(pedplus)
+}
+
+// Walk over the keys
+func (v *Volume) WalkKeys(pedestrian func(Key) error) error {
+	pedplus := func(nv *NeedleValue) (err error) {
+		if nv.Offset > 0 && nv.Key > 0 {
+			if err = pedestrian(nv.Key); err != nil {
+				return
+			}
+		}
+		return nil
+	}
+	return v.nm.Walk(pedplus)
+}
+
+func (v *Volume) String() string {
+	return fmt.Sprintf("%d@%s:v%d:r%s", v.Id, v.dataFile.Name(),
+		v.Version(), v.replicaType)
+}

weed-fs/src/pkg/storage/volume_id.go

@@ -5,6 +5,7 @@ import (
 )
 
 type VolumeId uint32
+
 func NewVolumeId(vid string) (VolumeId, error) {
 	volumeId, err := strconv.ParseUint(vid, 10, 64)
 	return VolumeId(volumeId), err

weed-fs/src/pkg/storage/volume_version.go

@@ -1,7 +1,6 @@
 package storage
 
-import (
+import ()
-)
 
 type Version uint8
 

weed-fs/src/pkg/topology/data_center.go

@@ -1,7 +1,6 @@
 package topology
 
-import (
+import ()
-)
 
 type DataCenter struct {
 	NodeImpl

weed-fs/src/pkg/topology/node_list_test.go

@@ -1,9 +1,9 @@
 package topology
 
 import (
+	_ "fmt"
 	"strconv"
 	"testing"
-  _ "fmt"
 )
 
 func TestXYZ(t *testing.T) {

weed-fs/src/pkg/topology/topology_map.go

@@ -1,7 +1,6 @@
 package topology
 
-import (
+import ()
-)
 
 func (t *Topology) ToMap() interface{} {
 	m := make(map[string]interface{})

weed-fs/src/pkg/util/bytes.go

@@ -31,4 +31,3 @@ func Uint32toBytes(b []byte, v uint32){
 func Uint8toBytes(b []byte, v uint8) {
 	b[0] = byte(v)
 }
-