Merge accumulated changes related to message queue (#5098)

* balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
2023-12-11 12:05:54 -08:00
parent 8784553501
commit 580940bf82
58 changed files with 3853 additions and 2752 deletions
--- a/weed/mq/topic/local_topic.go
+++ b/weed/mq/topic/local_topic.go
@@ -1,10 +1,19 @@
 package topic

+import "sync"
+
 type LocalTopic struct {
 	Topic
 	Partitions []*LocalPartition
 }

+func NewLocalTopic(topic Topic) *LocalTopic {
+	return &LocalTopic{
+		Topic:      topic,
+		Partitions: make([]*LocalPartition, 0),
+	}
+}
+
 func (localTopic *LocalTopic) findPartition(partition Partition) *LocalPartition {
 	for _, localPartition := range localTopic.Partitions {
 		if localPartition.Partition.Equals(partition) {
@@ -27,3 +36,52 @@ func (localTopic *LocalTopic) removePartition(partition Partition) bool {
 	localTopic.Partitions = append(localTopic.Partitions[:foundPartitionIndex], localTopic.Partitions[foundPartitionIndex+1:]...)
 	return true
 }
+
+func (localTopic *LocalTopic) closePartitionPublishers(unixTsNs int64) bool {
+	var wg sync.WaitGroup
+	for _, localPartition := range localTopic.Partitions {
+		if localPartition.UnixTimeNs != unixTsNs {
+			continue
+		}
+		wg.Add(1)
+		go func(localPartition *LocalPartition) {
+			defer wg.Done()
+			localPartition.closePublishers()
+		}(localPartition)
+	}
+	wg.Wait()
+	return true
+}
+
+func (localTopic *LocalTopic) closePartitionSubscribers(unixTsNs int64) bool {
+	var wg sync.WaitGroup
+	for _, localPartition := range localTopic.Partitions {
+		if localPartition.UnixTimeNs != unixTsNs {
+			continue
+		}
+		wg.Add(1)
+		go func(localPartition *LocalPartition) {
+			defer wg.Done()
+			localPartition.closeSubscribers()
+		}(localPartition)
+	}
+	wg.Wait()
+	return true
+}
+
+func (localTopic *LocalTopic) WaitUntilNoPublishers() {
+	for {
+		var wg sync.WaitGroup
+		for _, localPartition := range localTopic.Partitions {
+			wg.Add(1)
+			go func(localPartition *LocalPartition) {
+				defer wg.Done()
+				localPartition.WaitUntilNoPublishers()
+			}(localPartition)
+		}
+		wg.Wait()
+		if len(localTopic.Partitions) == 0 {
+			return
+		}
+	}
+}