refactor: separating out remote.proto

weed/storage/erasure_coding/ec_volume.go

@@ -3,13 +3,13 @@ package erasure_coding
 import (
 	"errors"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/storage/volume_info"
 	"math"
 	"os"
 	"sort"
 	"sync"
 	"time"
 
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/idx"
@@ -63,10 +63,10 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
 
 	// read volume info
 	ev.Version = needle.Version3
-	if volumeInfo, _, found, _ := pb.MaybeLoadVolumeInfo(dataBaseFileName + ".vif"); found {
+	if volumeInfo, _, found, _ := volume_info.MaybeLoadVolumeInfo(dataBaseFileName + ".vif"); found {
 		ev.Version = needle.Version(volumeInfo.Version)
 	} else {
-		pb.SaveVolumeInfo(dataBaseFileName+".vif", &volume_server_pb.VolumeInfo{Version: uint32(ev.Version)})
+		volume_info.SaveVolumeInfo(dataBaseFileName+".vif", &volume_server_pb.VolumeInfo{Version: uint32(ev.Version)})
 	}
 
 	ev.ShardLocations = make(map[ShardId][]string)

weed/storage/store.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/storage/volume_info"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"path/filepath"
 	"strings"
@@ -10,7 +11,6 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
@@ -474,12 +474,12 @@ func (s *Store) ConfigureVolume(i needle.VolumeId, replication string) error {
 		// load, modify, save
 		baseFileName := strings.TrimSuffix(fileInfo.Name(), filepath.Ext(fileInfo.Name()))
 		vifFile := filepath.Join(location.Directory, baseFileName+".vif")
-		volumeInfo, _, _, err := pb.MaybeLoadVolumeInfo(vifFile)
+		volumeInfo, _, _, err := volume_info.MaybeLoadVolumeInfo(vifFile)
 		if err != nil {
 			return fmt.Errorf("volume %d fail to load vif", i)
 		}
 		volumeInfo.Replication = replication
-		err = pb.SaveVolumeInfo(vifFile, volumeInfo)
+		err = volume_info.SaveVolumeInfo(vifFile, volumeInfo)
 		if err != nil {
 			return fmt.Errorf("volume %d fail to save vif", i)
 		}

weed/storage/volume_info/volume_info.go

@@ -0,0 +1,85 @@
+package volume_info
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+
+	_ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend"
+	"github.com/chrislusf/seaweedfs/weed/util"
+
+	"github.com/golang/protobuf/jsonpb"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// MaybeLoadVolumeInfo load the file data as *volume_server_pb.VolumeInfo, the returned volumeInfo will not be nil
+func MaybeLoadVolumeInfo(fileName string) (volumeInfo *volume_server_pb.VolumeInfo, hasRemoteFile bool, hasVolumeInfoFile bool, err error) {
+
+	volumeInfo = &volume_server_pb.VolumeInfo{}
+
+	glog.V(1).Infof("maybeLoadVolumeInfo checks %s", fileName)
+	if exists, canRead, _, _, _ := util.CheckFile(fileName); !exists || !canRead {
+		if !exists {
+			return
+		}
+		hasVolumeInfoFile = true
+		if !canRead {
+			glog.Warningf("can not read %s", fileName)
+			err = fmt.Errorf("can not read %s", fileName)
+			return
+		}
+		return
+	}
+
+	hasVolumeInfoFile = true
+
+	glog.V(1).Infof("maybeLoadVolumeInfo reads %s", fileName)
+	tierData, readErr := ioutil.ReadFile(fileName)
+	if readErr != nil {
+		glog.Warningf("fail to read %s : %v", fileName, readErr)
+		err = fmt.Errorf("fail to read %s : %v", fileName, readErr)
+		return
+
+	}
+
+	glog.V(1).Infof("maybeLoadVolumeInfo Unmarshal volume info %v", fileName)
+	if err = jsonpb.Unmarshal(bytes.NewReader(tierData), volumeInfo); err != nil {
+		glog.Warningf("unmarshal error: %v", err)
+		err = fmt.Errorf("unmarshal error: %v", err)
+		return
+	}
+
+	if len(volumeInfo.GetFiles()) == 0 {
+		return
+	}
+
+	hasRemoteFile = true
+
+	return
+}
+
+func SaveVolumeInfo(fileName string, volumeInfo *volume_server_pb.VolumeInfo) error {
+
+	if exists, _, canWrite, _, _ := util.CheckFile(fileName); exists && !canWrite {
+		return fmt.Errorf("%s not writable", fileName)
+	}
+
+	m := jsonpb.Marshaler{
+		EmitDefaults: true,
+		Indent:       "  ",
+	}
+
+	text, marshalErr := m.MarshalToString(volumeInfo)
+	if marshalErr != nil {
+		return fmt.Errorf("marshal to %s: %v", fileName, marshalErr)
+	}
+
+	writeErr := ioutil.WriteFile(fileName, []byte(text), 0755)
+	if writeErr != nil {
+		return fmt.Errorf("fail to write %s : %v", fileName, writeErr)
+	}
+
+	return nil
+}

weed/storage/volume_tier.go

@@ -2,11 +2,11 @@ package storage
 
 import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	_ "github.com/chrislusf/seaweedfs/weed/storage/backend/s3_backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	volume_info "github.com/chrislusf/seaweedfs/weed/storage/volume_info"
 )
 
 func (v *Volume) GetVolumeInfo() *volume_server_pb.VolumeInfo {
@@ -16,7 +16,7 @@ func (v *Volume) GetVolumeInfo() *volume_server_pb.VolumeInfo {
 func (v *Volume) maybeLoadVolumeInfo() (found bool) {
 
 	var err error
-	v.volumeInfo, v.hasRemoteFile, found, err = pb.MaybeLoadVolumeInfo(v.FileName(".vif"))
+	v.volumeInfo, v.hasRemoteFile, found, err = volume_info.MaybeLoadVolumeInfo(v.FileName(".vif"))
 
 	if v.volumeInfo.Version == 0 {
 		v.volumeInfo.Version = uint32(needle.CurrentVersion)
@@ -56,6 +56,6 @@ func (v *Volume) SaveVolumeInfo() error {
 
 	tierFileName := v.FileName(".vif")
 
-	return pb.SaveVolumeInfo(tierFileName, v.volumeInfo)
+	return volume_info.SaveVolumeInfo(tierFileName, v.volumeInfo)
 
 }