shell: add ec.decode command

weed/storage/super_block/replica_placement.go

@@ -0,0 +1,53 @@
+package super_block
+
+import (
+	"errors"
+	"fmt"
+)
+
+type ReplicaPlacement struct {
+	SameRackCount       int
+	DiffRackCount       int
+	DiffDataCenterCount int
+}
+
+func NewReplicaPlacementFromString(t string) (*ReplicaPlacement, error) {
+	rp := &ReplicaPlacement{}
+	for i, c := range t {
+		count := int(c - '0')
+		if 0 <= count && count <= 2 {
+			switch i {
+			case 0:
+				rp.DiffDataCenterCount = count
+			case 1:
+				rp.DiffRackCount = count
+			case 2:
+				rp.SameRackCount = count
+			}
+		} else {
+			return rp, errors.New("Unknown Replication Type:" + t)
+		}
+	}
+	return rp, nil
+}
+
+func NewReplicaPlacementFromByte(b byte) (*ReplicaPlacement, error) {
+	return NewReplicaPlacementFromString(fmt.Sprintf("%03d", b))
+}
+
+func (rp *ReplicaPlacement) Byte() byte {
+	ret := rp.DiffDataCenterCount*100 + rp.DiffRackCount*10 + rp.SameRackCount
+	return byte(ret)
+}
+
+func (rp *ReplicaPlacement) String() string {
+	b := make([]byte, 3)
+	b[0] = byte(rp.DiffDataCenterCount + '0')
+	b[1] = byte(rp.DiffRackCount + '0')
+	b[2] = byte(rp.SameRackCount + '0')
+	return string(b)
+}
+
+func (rp *ReplicaPlacement) GetCopyCount() int {
+	return rp.DiffDataCenterCount + rp.DiffRackCount + rp.SameRackCount + 1
+}

weed/storage/super_block/replica_placement_test.go

@@ -0,0 +1,14 @@
+package super_block
+
+import (
+	"testing"
+)
+
+func TestReplicaPlacementSerialDeserial(t *testing.T) {
+	rp, _ := NewReplicaPlacementFromString("001")
+	newRp, _ := NewReplicaPlacementFromByte(rp.Byte())
+	if rp.String() != newRp.String() {
+		println("expected:", rp.String(), "actual:", newRp.String())
+		t.Fail()
+	}
+}

weed/storage/super_block/super_block.go

@@ -0,0 +1,69 @@
+package super_block
+
+import (
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	SuperBlockSize = 8
+)
+
+/*
+* Super block currently has 8 bytes allocated for each volume.
+* Byte 0: version, 1 or 2
+* Byte 1: Replica Placement strategy, 000, 001, 002, 010, etc
+* Byte 2 and byte 3: Time to live. See TTL for definition
+* Byte 4 and byte 5: The number of times the volume has been compacted.
+* Rest bytes: Reserved
+ */
+type SuperBlock struct {
+	Version            needle.Version
+	ReplicaPlacement   *ReplicaPlacement
+	Ttl                *needle.TTL
+	CompactionRevision uint16
+	Extra              *master_pb.SuperBlockExtra
+	ExtraSize          uint16
+}
+
+func (s *SuperBlock) BlockSize() int {
+	switch s.Version {
+	case needle.Version2, needle.Version3:
+		return SuperBlockSize + int(s.ExtraSize)
+	}
+	return SuperBlockSize
+}
+
+func (s *SuperBlock) Bytes() []byte {
+	header := make([]byte, SuperBlockSize)
+	header[0] = byte(s.Version)
+	header[1] = s.ReplicaPlacement.Byte()
+	s.Ttl.ToBytes(header[2:4])
+	util.Uint16toBytes(header[4:6], s.CompactionRevision)
+
+	if s.Extra != nil {
+		extraData, err := proto.Marshal(s.Extra)
+		if err != nil {
+			glog.Fatalf("cannot marshal super block extra %+v: %v", s.Extra, err)
+		}
+		extraSize := len(extraData)
+		if extraSize > 256*256-2 {
+			// reserve a couple of bits for future extension
+			glog.Fatalf("super block extra size is %d bigger than %d", extraSize, 256*256-2)
+		}
+		s.ExtraSize = uint16(extraSize)
+		util.Uint16toBytes(header[6:8], s.ExtraSize)
+
+		header = append(header, extraData...)
+	}
+
+	return header
+}
+
+func (s *SuperBlock) Initialized() bool {
+	return s.ReplicaPlacement != nil && s.Ttl != nil
+}

weed/storage/super_block/super_block_read.go.go

@@ -0,0 +1,44 @@
+package super_block
+
+import (
+	"fmt"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/backend"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+// ReadSuperBlock reads from data file and load it into volume's super block
+func ReadSuperBlock(datBackend backend.BackendStorageFile) (superBlock SuperBlock, err error) {
+
+	header := make([]byte, SuperBlockSize)
+	if _, e := datBackend.ReadAt(header, 0); e != nil {
+		err = fmt.Errorf("cannot read volume %s super block: %v", datBackend.Name(), e)
+		return
+	}
+
+	superBlock.Version = needle.Version(header[0])
+	if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil {
+		err = fmt.Errorf("cannot read replica type: %s", err.Error())
+		return
+	}
+	superBlock.Ttl = needle.LoadTTLFromBytes(header[2:4])
+	superBlock.CompactionRevision = util.BytesToUint16(header[4:6])
+	superBlock.ExtraSize = util.BytesToUint16(header[6:8])
+
+	if superBlock.ExtraSize > 0 {
+		// read more
+		extraData := make([]byte, int(superBlock.ExtraSize))
+		superBlock.Extra = &master_pb.SuperBlockExtra{}
+		err = proto.Unmarshal(extraData, superBlock.Extra)
+		if err != nil {
+			err = fmt.Errorf("cannot read volume %s super block extra: %v", datBackend.Name(), err)
+			return
+		}
+	}
+
+	return
+}

weed/storage/super_block/super_block_test.go

@@ -0,0 +1,25 @@
+package super_block
+
+import (
+	"testing"
+
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+func TestSuperBlockReadWrite(t *testing.T) {
+	rp, _ := NewReplicaPlacementFromByte(byte(001))
+	ttl, _ := needle.ReadTTL("15d")
+	s := &SuperBlock{
+		Version:          needle.CurrentVersion,
+		ReplicaPlacement: rp,
+		Ttl:              ttl,
+	}
+
+	bytes := s.Bytes()
+
+	if !(bytes[2] == 15 && bytes[3] == needle.Day) {
+		println("byte[2]:", bytes[2], "byte[3]:", bytes[3])
+		t.Fail()
+	}
+
+}