gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move initial assignment to rebalance logic

Browse Source
This commit is contained in:
chrislu
2024-05-14 09:09:36 -07:00
parent 6e5075e14e
commit 972e9faaa2
3 changed files with 14 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
weed/mq/broker/broker_grpc_sub_coordinator.go
View File

@@ -4,7 +4,6 @@ import (
"context" "context"
"github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator" "github.com/seaweedfs/seaweedfs/weed/mq/sub_coordinator"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
@@ -37,18 +36,6 @@ func (b *MessageQueueBroker) SubscriberToSubCoordinator(stream mq_pb.SeaweedMess
ctx := stream.Context() ctx := stream.Context()
go func() { go func() {
// try to load the partition assignment from filer
if conf, err := b.readTopicConfFromFiler(topic.FromPbTopic(initMessage.Topic)); err == nil {
// send partition assignment to subscriber
cgi.ResponseChan <- &mq_pb.SubscriberToSubCoordinatorResponse{
Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{
Assignment: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment{
PartitionAssignments: conf.BrokerPartitionAssignments,
},
},
}
}
// process ack messages // process ack messages
for { for {
_, err := stream.Recv() _, err := stream.Recv()

9
weed/mq/broker/broker_server.go
View File

@@ -65,8 +65,8 @@ func NewMessageBroker(option *MessageQueueBrokerOption, grpcDialOption grpc.Dial
Coordinator: coordinator, Coordinator: coordinator,
} }
fca := &sub_coordinator.FilerClientAccessor{ fca := &sub_coordinator.FilerClientAccessor{
GetFilerFn: mqBroker.GetFiler, GetFiler: mqBroker.GetFiler,
GrpcDialOption: grpcDialOption, GetGrpcDialOption: mqBroker.GetGrpcDialOption,
} }
mqBroker.fca = fca mqBroker.fca = fca
coordinator.FilerClientAccessor = fca coordinator.FilerClientAccessor = fca
@@ -126,6 +126,11 @@ func (b *MessageQueueBroker) OnBrokerUpdate(update *master_pb.ClusterNodeUpdate,
} }
func (b *MessageQueueBroker) GetGrpcDialOption() grpc.DialOption {
return b.grpcDialOption
}
func (b *MessageQueueBroker) GetFiler() pb.ServerAddress { func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
return b.currentFiler return b.currentFiler
} }

11
weed/mq/sub_coordinator/consumer_group.go
View File

@@ -77,10 +77,13 @@ func (cg *ConsumerGroup) BalanceConsumerGroupInstances(knownPartitionSlotToBroke
// collect current topic partitions // collect current topic partitions
partitionSlotToBrokerList := knownPartitionSlotToBrokerList partitionSlotToBrokerList := knownPartitionSlotToBrokerList
if partitionSlotToBrokerList == nil { if partitionSlotToBrokerList == nil {
var found bool if conf, err := cg.filerClientAccessor.ReadTopicConfFromFiler(cg.topic); err == nil {
partitionSlotToBrokerList, found = cg.pubBalancer.TopicToBrokers.Get(cg.topic.String()) partitionSlotToBrokerList = pub_balancer.NewPartitionSlotToBrokerList(pub_balancer.MaxPartitionCount)
if !found { for _, assignment := range conf.BrokerPartitionAssignments {
glog.V(0).Infof("topic %s not found in balancer", cg.topic.String()) partitionSlotToBrokerList.AddBroker(assignment.Partition, assignment.LeaderBroker)
}
} else {
glog.V(0).Infof("fail to read topic conf from filer: %v", err)
return return
} }
} }