subscriber receives partitions and dispatch to processors

weed/mq/client/sub_client/subscriber.go

@@ -4,6 +4,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc"
+	"sync"
 	"time"
 )
 
@@ -30,23 +31,27 @@ type OnCompletionFunc func()
 type TopicSubscriber struct {
 	SubscriberConfig           *SubscriberConfiguration
 	ContentConfig              *ContentConfiguration
-	ProcessorConfig            *ProcessorConfiguration
-	brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment
-	OnEachMessageFunc          OnEachMessageFunc
+	ProcessorConfig               *ProcessorConfiguration
+	brokerPartitionAssignmentChan chan *mq_pb.BrokerPartitionAssignment
+	OnEachMessageFunc             OnEachMessageFunc
 	OnCompletionFunc           OnCompletionFunc
 	bootstrapBrokers           []string
 	waitForMoreMessage         bool
 	alreadyProcessedTsNs       int64
+	activeProcessors           map[topic.Partition]*ProcessorState
+	activeProcessorsLock       sync.Mutex
 }
 
 func NewTopicSubscriber(bootstrapBrokers []string, subscriber *SubscriberConfiguration, content *ContentConfiguration, processor ProcessorConfiguration) *TopicSubscriber {
 	return &TopicSubscriber{
-		SubscriberConfig:     subscriber,
-		ContentConfig:        content,
-		ProcessorConfig:      &processor,
-		bootstrapBrokers:     bootstrapBrokers,
-		waitForMoreMessage:   true,
-		alreadyProcessedTsNs: content.StartTime.UnixNano(),
+		SubscriberConfig:              subscriber,
+		ContentConfig:                 content,
+		ProcessorConfig:               &processor,
+		brokerPartitionAssignmentChan: make(chan *mq_pb.BrokerPartitionAssignment, 1024),
+		bootstrapBrokers:              bootstrapBrokers,
+		waitForMoreMessage:            true,
+		alreadyProcessedTsNs:          content.StartTime.UnixNano(),
+		activeProcessors:              make(map[topic.Partition]*ProcessorState),
 	}
 }