gRpc for master~volume heartbeat

weed/server/master_grpc_server.go

@@ -0,0 +1,57 @@
+package weed_server
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/topology"
+)
+
+func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error {
+	var dn *topology.DataNode
+	t := ms.Topo
+	for {
+		heartbeat, err := stream.Recv()
+		if err == nil {
+			if dn == nil {
+				t.Sequence.SetMax(heartbeat.MaxFileKey)
+				dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack)
+				dc := t.GetOrCreateDataCenter(dcName)
+				rack := dc.GetOrCreateRack(rackName)
+				dn = rack.GetOrCreateDataNode(heartbeat.Ip,
+					int(heartbeat.Port), heartbeat.PublicUrl,
+					int(heartbeat.MaxVolumeCount))
+				glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
+				if err := stream.Send(&pb.HeartbeatResponse{
+					VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
+					SecretKey:       string(ms.guard.SecretKey),
+				}); err != nil {
+					return err
+				}
+			}
+
+			var volumeInfos []storage.VolumeInfo
+			for _, v := range heartbeat.Volumes {
+				if vi, err := storage.NewVolumeInfo(v); err == nil {
+					volumeInfos = append(volumeInfos, vi)
+				} else {
+					glog.V(0).Infof("Fail to convert joined volume information: %v", err)
+				}
+			}
+			deletedVolumes := dn.UpdateVolumes(volumeInfos)
+			for _, v := range volumeInfos {
+				t.RegisterVolumeLayout(v, dn)
+			}
+			for _, v := range deletedVolumes {
+				t.UnRegisterVolumeLayout(v, dn)
+			}
+
+		} else {
+			glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port)
+			if dn != nil {
+				t.UnRegisterDataNode(dn)
+			}
+			return err
+		}
+	}
+}

weed/server/master_server.go

@@ -72,7 +72,6 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
 	r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
 	r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler)))
-	r.HandleFunc("/dir/join", ms.proxyToLeader(ms.guard.WhiteList(ms.dirJoinHandler)))
 	r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
 	r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
 	r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeLookupHandler)))

weed/server/master_server_handlers_admin.go

@@ -1,21 +1,16 @@
 package weed_server
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/topology"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/golang/protobuf/proto"
 )
 
 func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
@@ -34,37 +29,6 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R
 	ms.Topo.DeleteCollection(r.FormValue("collection"))
 }
 
-func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) {
-	body, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		writeJsonError(w, r, http.StatusBadRequest, err)
-		return
-	}
-	joinMessage := &operation.JoinMessage{}
-	if err = proto.Unmarshal(body, joinMessage); err != nil {
-		writeJsonError(w, r, http.StatusBadRequest, err)
-		return
-	}
-	if *joinMessage.Ip == "" {
-		*joinMessage.Ip = r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]
-	}
-	if glog.V(4) {
-		if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil {
-			glog.V(0).Infoln("json marshaling error: ", jsonError)
-			writeJsonError(w, r, http.StatusBadRequest, jsonError)
-			return
-		} else {
-			glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData))
-		}
-	}
-
-	ms.Topo.ProcessJoinMessage(joinMessage)
-	writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{
-		VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024,
-		SecretKey:       string(ms.guard.SecretKey),
-	})
-}
-
 func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) {
 	m := make(map[string]interface{})
 	m["Version"] = util.VERSION

weed/server/volume_grpc_client.go

@@ -0,0 +1,74 @@
+package weed_server
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+func (vs *VolumeServer) heartbeat() {
+
+	glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode())
+	vs.masterNodes = storage.NewMasterNodes(vs.masterNode)
+	vs.store.SetDataCenter(vs.dataCenter)
+	vs.store.SetRack(vs.rack)
+
+	for {
+		err := vs.doHeartbeat(time.Duration(vs.pulseSeconds) * time.Second)
+		if err != nil {
+			glog.V(0).Infof("heartbeat error: %v", err)
+			time.Sleep(time.Duration(3*vs.pulseSeconds) * time.Second)
+		}
+	}
+}
+
+func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error {
+
+	masterNode, err := vs.masterNodes.FindMaster()
+	if err != nil {
+		return fmt.Errorf("No master found: %v", err)
+	}
+
+	grpcConection, err := grpc.Dial(masterNode, grpc.WithInsecure())
+	if err != nil {
+		return fmt.Errorf("fail to dial: %v", err)
+	}
+	defer grpcConection.Close()
+
+	client := pb.NewSeaweedClient(grpcConection)
+	stream, err := client.SendHeartbeat(context.Background())
+	if err != nil {
+		glog.V(0).Infof("%v.SendHeartbeat(_) = _, %v", client, err)
+		return err
+	}
+	vs.SetMasterNode(masterNode)
+	glog.V(0).Infof("Heartbeat to %s", masterNode)
+
+	vs.store.Client = stream
+	defer func() { vs.store.Client = nil }()
+
+	go func() {
+		for {
+			in, err := stream.Recv()
+			if err != nil {
+				return
+			}
+			vs.store.VolumeSizeLimit = in.GetVolumeSizeLimit()
+			vs.guard.SecretKey = security.Secret(in.GetSecretKey())
+		}
+	}()
+
+	for {
+		if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
+			glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
+			return err
+		}
+		time.Sleep(sleepInterval)
+	}
+}

weed/server/volume_server.go

@@ -1,10 +1,8 @@
 package weed_server
 
 import (
-	"math/rand"
 	"net/http"
 	"sync"
-	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/security"
@@ -19,6 +17,7 @@ type VolumeServer struct {
 	rack         string
 	store        *storage.Store
 	guard        *security.Guard
+	masterNodes  *storage.MasterNodes
 
 	needleMapKind     storage.NeedleMapType
 	FixJpgOrientation bool
@@ -70,36 +69,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 		publicMux.HandleFunc("/", vs.publicReadOnlyHandler)
 	}
 
-	go func() {
-		connected := true
-
-		glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode())
-		vs.store.SetBootstrapMaster(vs.GetMasterNode())
-		vs.store.SetDataCenter(vs.dataCenter)
-		vs.store.SetRack(vs.rack)
-		for {
-			glog.V(4).Infof("Volume server sending to master %s", vs.GetMasterNode())
-			master, secretKey, err := vs.store.SendHeartbeatToMaster()
-			if err == nil {
-				if !connected {
-					connected = true
-					vs.SetMasterNode(master)
-					vs.guard.SecretKey = secretKey
-					glog.V(0).Infoln("Volume Server Connected with master at", master)
-				}
-			} else {
-				glog.V(1).Infof("Volume Server Failed to talk with master %s: %v", vs.masterNode, err)
-				if connected {
-					connected = false
-				}
-			}
-			if connected {
-				time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*(1+rand.Float32())) * time.Millisecond)
-			} else {
-				time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*0.25) * time.Millisecond)
-			}
-		}
-	}()
+	go vs.heartbeat()
 
 	return vs
 }