segment serde

weed/mq/broker/brokder_grpc_admin.go

@@ -53,16 +53,19 @@ func (broker *MessageQueueBroker) AssignSegmentBrokers(c context.Context, reques
 	if err != nil {
 		return ret, err
 	}
-	// good if the segment is still on the brokers
-	isActive, err := broker.checkSegmentsOnBrokers(segment, existingBrokers)
-	if err != nil {
-		return ret, err
-	}
-	if isActive {
-		for _, broker := range existingBrokers {
-			ret.Brokers = append(ret.Brokers, string(broker))
+
+	if len(existingBrokers) > 0 {
+		// good if the segment is still on the brokers
+		isActive, err := broker.checkSegmentsOnBrokers(segment, existingBrokers)
+		if err != nil {
+			return ret, err
+		}
+		if isActive {
+			for _, broker := range existingBrokers {
+				ret.Brokers = append(ret.Brokers, string(broker))
+			}
+			return ret, nil
 		}
-		return ret, nil
 	}
 
 	// randomly pick up to 10 brokers, and find the ones with the lightest load
@@ -72,7 +75,7 @@ func (broker *MessageQueueBroker) AssignSegmentBrokers(c context.Context, reques
 	}
 
 	// save the allocated brokers info for this segment on the filer
-	if err := broker.saveSegmentOnFiler(segment, selectedBrokers); err != nil {
+	if err := broker.saveSegmentBrokersOnFiler(segment, selectedBrokers); err != nil {
 		return ret, err
 	}
 
@@ -82,10 +85,6 @@ func (broker *MessageQueueBroker) AssignSegmentBrokers(c context.Context, reques
 	return ret, nil
 }
 
-func (broker *MessageQueueBroker) checkSegmentOnFiler(segment *mq.Segment) (brokers []pb.ServerAddress, err error) {
-	return
-}
-
 func (broker *MessageQueueBroker) checkSegmentsOnBrokers(segment *mq.Segment, brokers []pb.ServerAddress) (active bool, err error) {
 	var wg sync.WaitGroup
 
@@ -206,7 +205,3 @@ func (broker *MessageQueueBroker) checkBrokerStatus(candidates []pb.ServerAddres
 	wg.Wait()
 	return
 }
-
-func (broker *MessageQueueBroker) saveSegmentOnFiler(segment *mq.Segment, brokers []pb.ServerAddress) (err error) {
-	return
-}

weed/mq/broker/broker_segment_serde.go

@@ -0,0 +1,89 @@
+package broker
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/mq"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/mq_pb"
+	"github.com/golang/protobuf/jsonpb"
+	"time"
+)
+
+func (broker *MessageQueueBroker) checkSegmentOnFiler(segment *mq.Segment) (brokers []pb.ServerAddress, err error) {
+	info, found, err := broker.readSegmentOnFiler(segment)
+	if err != nil {
+		return
+	}
+	if !found {
+		return
+	}
+	for _, b := range info.Brokers {
+		brokers = append(brokers, pb.ServerAddress(b))
+	}
+
+	return
+}
+
+func (broker *MessageQueueBroker) saveSegmentBrokersOnFiler(segment *mq.Segment, brokers []pb.ServerAddress) (err error) {
+	var nodes []string
+	for _, b := range brokers {
+		nodes = append(nodes, string(b))
+	}
+	broker.saveSegmentToFiler(segment, &mq_pb.SegmentInfo{
+		Segment:          segment.ToPbSegment(),
+		StartTsNs:        time.Now().UnixNano(),
+		Brokers:          nodes,
+		StopTsNs:         0,
+		PreviousSegments: nil,
+		NextSegments:     nil,
+	})
+	return
+}
+
+func (broker *MessageQueueBroker) readSegmentOnFiler(segment *mq.Segment) (info *mq_pb.SegmentInfo, found bool, err error) {
+	dir, name := segment.DirAndName()
+
+	found, err = filer_pb.Exists(broker, dir, name, false)
+	if !found || err != nil {
+		return
+	}
+
+	err = pb.WithFilerClient(false, broker.GetFiler(), broker.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		// read filer conf first
+		data, err := filer.ReadInsideFiler(client, dir, name)
+		if err != nil {
+			return fmt.Errorf("ReadEntry: %v", err)
+		}
+
+		// parse into filer conf object
+		info = &mq_pb.SegmentInfo{}
+		if err = jsonpb.Unmarshal(bytes.NewReader(data), info); err != nil {
+			return err
+		}
+		found = true
+		return nil
+	})
+
+	return
+}
+
+func (broker *MessageQueueBroker) saveSegmentToFiler(segment *mq.Segment, info *mq_pb.SegmentInfo) (err error) {
+	dir, name := segment.DirAndName()
+
+	var buf bytes.Buffer
+	filer.ProtoToText(&buf, info)
+
+	err = pb.WithFilerClient(false, broker.GetFiler(), broker.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		// read filer conf first
+		err := filer.SaveInsideFiler(client, dir, name, buf.Bytes())
+		if err != nil {
+			return fmt.Errorf("save segment info: %v", err)
+		}
+		return nil
+	})
+
+	return
+}

weed/mq/broker/broker_server.go

@@ -80,9 +80,15 @@ func (broker *MessageQueueBroker) GetFiler() pb.ServerAddress {
 	return broker.currentFiler
 }
 
-func (broker *MessageQueueBroker) withFilerClient(streamingMode bool, filer pb.ServerAddress, fn func(filer_pb.SeaweedFilerClient) error) error {
+func (broker *MessageQueueBroker) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return pb.WithFilerClient(streamingMode, filer, broker.grpcDialOption, fn)
+	return pb.WithFilerClient(streamingMode, broker.GetFiler(), broker.grpcDialOption, fn)
+
+}
+
+func (broker *MessageQueueBroker) AdjustedUrl(location *filer_pb.Location) string {
+
+	return location.Url
 
 }