Merge accumulated changes related to message queue (#5098)

* balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2023-12-11 12:05:54 -08:00
parent 8784553501
commit 580940bf82
58 changed files with 3853 additions and 2752 deletions
--- a/weed/mq/pub_balancer/lookup.go
+++ b/weed/mq/pub_balancer/lookup.go
@@ -0,0 +1,53 @@
+package pub_balancer
+
+import (
+	"errors"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+var (
+	ErrNoBroker = errors.New("no broker")
+)
+
+func (balancer *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish bool, partitionCount int32) (assignments []*mq_pb.BrokerPartitionAssignment, err error) {
+	if partitionCount == 0 {
+		partitionCount = 6
+	}
+	// find existing topic partition assignments
+	for brokerStatsItem := range balancer.Brokers.IterBuffered() {
+		broker, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
+		for topicPartitionStatsItem := range brokerStats.TopicPartitionStats.IterBuffered() {
+			topicPartitionStat := topicPartitionStatsItem.Val
+			if topicPartitionStat.TopicPartition.Namespace == topic.Namespace &&
+				topicPartitionStat.TopicPartition.Name == topic.Name {
+				assignment := &mq_pb.BrokerPartitionAssignment{
+					Partition: &mq_pb.Partition{
+						RingSize:   MaxPartitionCount,
+						RangeStart: topicPartitionStat.RangeStart,
+						RangeStop:  topicPartitionStat.RangeStop,
+					},
+				}
+				// TODO fix follower setting
+				assignment.LeaderBroker = broker
+				assignments = append(assignments, assignment)
+			}
+		}
+	}
+	if len(assignments) > 0 && len(assignments) == int(partitionCount) || !publish {
+		glog.V(0).Infof("existing topic partitions %d: %v", len(assignments), assignments)
+		return assignments, nil
+	}
+
+	// find the topic partitions on the filer
+	// if the topic is not found
+	//   if the request is_for_publish
+	//     create the topic
+	//   if the request is_for_subscribe
+	//     return error not found
+	// t := topic.FromPbTopic(request.Topic)
+	if balancer.Brokers.IsEmpty() {
+		return nil, ErrNoBroker
+	}
+	return allocateTopicPartitions(balancer.Brokers, partitionCount), nil
+}