prepare for flexible super block

weed/storage/volume_read_write.go

@@ -189,7 +189,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId,
 
 	version := v.Version()
 
-	offset := int64(SuperBlockSize)
+	offset := int64(v.SuperBlock.BlockSize())
 	n, rest, e := ReadNeedleHeader(v.dataFile, version, offset)
 	if e != nil {
 		err = fmt.Errorf("cannot read needle header: %v", e)

weed/storage/volume_super_block.go

@@ -9,7 +9,7 @@ import (
 )
 
 const (
-	SuperBlockSize = 8
+	_SuperBlockSize = 8
 )
 
 /*
@@ -27,11 +27,15 @@ type SuperBlock struct {
 	CompactRevision  uint16
 }
 
+func (s *SuperBlock) BlockSize() int {
+	return _SuperBlockSize
+}
+
 func (s *SuperBlock) Version() Version {
 	return s.version
 }
 func (s *SuperBlock) Bytes() []byte {
-	header := make([]byte, SuperBlockSize)
+	header := make([]byte, _SuperBlockSize)
 	header[0] = byte(s.version)
 	header[1] = s.ReplicaPlacement.Byte()
 	s.Ttl.ToBytes(header[2:4])
@@ -59,18 +63,13 @@ func (v *Volume) maybeWriteSuperBlock() error {
 	}
 	return e
 }
+
 func (v *Volume) readSuperBlock() (err error) {
-	if _, err = v.dataFile.Seek(0, 0); err != nil {
-		return fmt.Errorf("cannot seek to the beginning of %s: %v", v.dataFile.Name(), err)
-	}
-	header := make([]byte, SuperBlockSize)
-	if _, e := v.dataFile.Read(header); e != nil {
-		return fmt.Errorf("cannot read volume %d super block: %v", v.Id, e)
-	}
-	v.SuperBlock, err = ParseSuperBlock(header)
+	v.SuperBlock, err = ReadSuperBlock(v.dataFile)
 	return err
 }
-func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
+
+func parseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
 	superBlock.version = Version(header[0])
 	if superBlock.ReplicaPlacement, err = NewReplicaPlacementFromByte(header[1]); err != nil {
 		err = fmt.Errorf("cannot read replica type: %s", err.Error())
@@ -79,3 +78,17 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
 	superBlock.CompactRevision = util.BytesToUint16(header[4:6])
 	return
 }
+
+// ReadSuperBlock reads from data file and load it into volume's super block
+func ReadSuperBlock(dataFile *os.File) (superBlock SuperBlock, err error) {
+	if _, err = dataFile.Seek(0, 0); err != nil {
+		err = fmt.Errorf("cannot seek to the beginning of %s: %v", dataFile.Name(), err)
+		return
+	}
+	header := make([]byte, _SuperBlockSize)
+	if _, e := dataFile.Read(header); e != nil {
+		err = fmt.Errorf("cannot read volume %s super block: %v", dataFile.Name(), e)
+		return
+	}
+	return parseSuperBlock(header)
+}

weed/storage/volume_vacuum.go

@@ -87,14 +87,7 @@ func (v *Volume) cleanupCompact() error {
 }
 
 func fetchCompactRevisionFromDatFile(file *os.File) (compactRevision uint16, err error) {
-	if _, err = file.Seek(0, 0); err != nil {
-		return 0, fmt.Errorf("cannot seek to the beginning of %s: %v", file.Name(), err)
-	}
-	header := make([]byte, SuperBlockSize)
-	if _, e := file.Read(header); e != nil {
-		return 0, fmt.Errorf("cannot read file %s 's super block: %v", file.Name(), e)
-	}
-	superBlock, err := ParseSuperBlock(header)
+	superBlock, err := ReadSuperBlock(file)
 	if err != nil {
 		return 0, err
 	}
@@ -242,7 +235,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
 	defer idx.Close()
 
 	nm := NewBtreeNeedleMap(idx)
-	new_offset := int64(SuperBlockSize)
+	new_offset := int64(0)
 
 	now := uint64(time.Now().Unix())
 
@@ -250,6 +243,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
 		func(superBlock SuperBlock) error {
 			superBlock.CompactRevision++
 			_, err = dst.Write(superBlock.Bytes())
+			new_offset = int64(superBlock.BlockSize())
 			return err
 		}, true, func(n *Needle, offset int64) error {
 			if n.HasTtl() && now >= n.LastModified+uint64(v.Ttl.Minutes()*60) {
@@ -297,7 +291,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
 
 	v.SuperBlock.CompactRevision++
 	dst.Write(v.SuperBlock.Bytes())
-	new_offset := int64(SuperBlockSize)
+	new_offset := int64(v.SuperBlock.BlockSize())
 
 	WalkIndexFile(oldIndexFile, func(key uint64, offset, size uint32) error {
 		if offset == 0 || size == TombstoneFileSize {