This commit is contained in:
chrislu
2024-05-20 11:05:18 -07:00
parent 5038577f7e
commit 3622196881
15 changed files with 58 additions and 56 deletions

View File

@@ -12,7 +12,6 @@ import (
"time" "time"
) )
func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) { func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) {
var req *mq_pb.SubscribeFollowMeRequest var req *mq_pb.SubscribeFollowMeRequest
req, err = stream.Recv() req, err = stream.Recv()

View File

@@ -130,7 +130,6 @@ func (b *MessageQueueBroker) GetGrpcDialOption() grpc.DialOption {
return b.grpcDialOption return b.grpcDialOption
} }
func (b *MessageQueueBroker) GetFiler() pb.ServerAddress { func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
return b.currentFiler return b.currentFiler
} }

View File

@@ -44,7 +44,7 @@ func main() {
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig) subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
counter := 0 counter := 0
subscriber.SetEachMessageFunc(func(key, value []byte) (error) { subscriber.SetEachMessageFunc(func(key, value []byte) error {
counter++ counter++
println(string(key), "=>", string(value), counter) println(string(key), "=>", string(value), counter)
return nil return nil

View File

@@ -71,7 +71,7 @@ func main() {
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig) subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
counter := 0 counter := 0
subscriber.SetEachMessageFunc(func(key, value []byte) (error) { subscriber.SetEachMessageFunc(func(key, value []byte) error {
counter++ counter++
record := &schema_pb.RecordValue{} record := &schema_pb.RecordValue{}
proto.Unmarshal(value, record) proto.Unmarshal(value, record)

View File

@@ -125,11 +125,11 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
close(partitionOffsetChan) close(partitionOffsetChan)
}() }()
concurrentPartitionLimit := int(sub.ProcessorConfig.MaxPartitionCount) perPartitionConcurrency := int(sub.ProcessorConfig.PerPartitionConcurrency)
if concurrentPartitionLimit <= 0 { if perPartitionConcurrency <= 0 {
concurrentPartitionLimit = 1 perPartitionConcurrency = 1
} }
executors := util.NewLimitedConcurrentExecutor(concurrentPartitionLimit) executors := util.NewLimitedConcurrentExecutor(perPartitionConcurrency)
go func() { go func() {
for ack := range partitionOffsetChan { for ack := range partitionOffsetChan {

View File

@@ -9,7 +9,6 @@ import (
) )
type ProcessorState struct { type ProcessorState struct {
} }
// Subscribe subscribes to a topic's specified partitions. // Subscribe subscribes to a topic's specified partitions.

View File

@@ -26,6 +26,7 @@ func (imt *InflightMessageTracker) InflightMessage(key []byte, tsNs int64) {
imt.messages[string(key)] = tsNs imt.messages[string(key)] = tsNs
imt.timestamps.Add(tsNs) imt.timestamps.Add(tsNs)
} }
// IsMessageAcknowledged returns true if the message has been acknowledged. // IsMessageAcknowledged returns true if the message has been acknowledged.
// If the message is older than the oldest inflight messages, returns false. // If the message is older than the oldest inflight messages, returns false.
// returns false if the message is inflight. // returns false if the message is inflight.
@@ -47,6 +48,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64)
return true return true
} }
// AcknowledgeMessage acknowledges the message with the key and timestamp. // AcknowledgeMessage acknowledges the message with the key and timestamp.
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool { func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool {
imt.mu.Lock() imt.mu.Lock()
@@ -71,12 +73,14 @@ type RingBuffer struct {
head int head int
size int size int
} }
// NewRingBuffer creates a new RingBuffer of the given capacity. // NewRingBuffer creates a new RingBuffer of the given capacity.
func NewRingBuffer(capacity int) *RingBuffer { func NewRingBuffer(capacity int) *RingBuffer {
return &RingBuffer{ return &RingBuffer{
buffer: make([]int64, capacity), buffer: make([]int64, capacity),
} }
} }
// Add adds a new timestamp to the ring buffer. // Add adds a new timestamp to the ring buffer.
func (rb *RingBuffer) Add(timestamp int64) { func (rb *RingBuffer) Add(timestamp int64) {
rb.buffer[rb.head] = timestamp rb.buffer[rb.head] = timestamp
@@ -85,6 +89,7 @@ func (rb *RingBuffer) Add(timestamp int64) {
rb.size++ rb.size++
} }
} }
// Remove removes the specified timestamp from the ring buffer. // Remove removes the specified timestamp from the ring buffer.
func (rb *RingBuffer) Remove(timestamp int64) { func (rb *RingBuffer) Remove(timestamp int64) {
// Perform binary search // Perform binary search