go fmt
This commit is contained in:
@@ -186,7 +186,7 @@ func (b *MessageQueueBroker) getRequestPosition(initMessage *mq_pb.SubscribeMess
|
|||||||
startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2)
|
startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if storedOffset, err := b.readConsumerGroupOffset(initMessage); err == nil{
|
if storedOffset, err := b.readConsumerGroupOffset(initMessage); err == nil {
|
||||||
startPosition = log_buffer.NewMessagePosition(storedOffset, -2)
|
startPosition = log_buffer.NewMessagePosition(storedOffset, -2)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) {
|
func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_SubscribeFollowMeServer) (err error) {
|
||||||
var req *mq_pb.SubscribeFollowMeRequest
|
var req *mq_pb.SubscribeFollowMeRequest
|
||||||
req, err = stream.Recv()
|
req, err = stream.Recv()
|
||||||
|
|||||||
@@ -130,7 +130,6 @@ func (b *MessageQueueBroker) GetGrpcDialOption() grpc.DialOption {
|
|||||||
return b.grpcDialOption
|
return b.grpcDialOption
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
|
func (b *MessageQueueBroker) GetFiler() pb.ServerAddress {
|
||||||
return b.currentFiler
|
return b.currentFiler
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ func main() {
|
|||||||
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
|
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
|
||||||
|
|
||||||
counter := 0
|
counter := 0
|
||||||
subscriber.SetEachMessageFunc(func(key, value []byte) (error) {
|
subscriber.SetEachMessageFunc(func(key, value []byte) error {
|
||||||
counter++
|
counter++
|
||||||
println(string(key), "=>", string(value), counter)
|
println(string(key), "=>", string(value), counter)
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ func main() {
|
|||||||
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
|
subscriber := sub_client.NewTopicSubscriber(brokers, subscriberConfig, contentConfig, processorConfig)
|
||||||
|
|
||||||
counter := 0
|
counter := 0
|
||||||
subscriber.SetEachMessageFunc(func(key, value []byte) (error) {
|
subscriber.SetEachMessageFunc(func(key, value []byte) error {
|
||||||
counter++
|
counter++
|
||||||
record := &schema_pb.RecordValue{}
|
record := &schema_pb.RecordValue{}
|
||||||
proto.Unmarshal(value, record)
|
proto.Unmarshal(value, record)
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
|
|||||||
Concurrency: sub.ProcessorConfig.PerPartitionConcurrency,
|
Concurrency: sub.ProcessorConfig.PerPartitionConcurrency,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});err != nil {
|
}); err != nil {
|
||||||
glog.V(0).Infof("subscriber %s connected to partition %+v at %v: %v", sub.ContentConfig.Topic, assigned.Partition, assigned.LeaderBroker, err)
|
glog.V(0).Infof("subscriber %s connected to partition %+v at %v: %v", sub.ContentConfig.Topic, assigned.Partition, assigned.LeaderBroker, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,16 +120,16 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
|
|||||||
defer sub.OnCompletionFunc()
|
defer sub.OnCompletionFunc()
|
||||||
}
|
}
|
||||||
|
|
||||||
partitionOffsetChan:= make(chan int64, 1024)
|
partitionOffsetChan := make(chan int64, 1024)
|
||||||
defer func() {
|
defer func() {
|
||||||
close(partitionOffsetChan)
|
close(partitionOffsetChan)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
concurrentPartitionLimit := int(sub.ProcessorConfig.MaxPartitionCount)
|
perPartitionConcurrency := int(sub.ProcessorConfig.PerPartitionConcurrency)
|
||||||
if concurrentPartitionLimit <= 0 {
|
if perPartitionConcurrency <= 0 {
|
||||||
concurrentPartitionLimit = 1
|
perPartitionConcurrency = 1
|
||||||
}
|
}
|
||||||
executors := util.NewLimitedConcurrentExecutor(concurrentPartitionLimit)
|
executors := util.NewLimitedConcurrentExecutor(perPartitionConcurrency)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
for ack := range partitionOffsetChan {
|
for ack := range partitionOffsetChan {
|
||||||
@@ -162,7 +162,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig
|
|||||||
processErr := sub.OnEachMessageFunc(m.Data.Key, m.Data.Value)
|
processErr := sub.OnEachMessageFunc(m.Data.Key, m.Data.Value)
|
||||||
if processErr == nil {
|
if processErr == nil {
|
||||||
partitionOffsetChan <- m.Data.TsNs
|
partitionOffsetChan <- m.Data.TsNs
|
||||||
}else{
|
} else {
|
||||||
lastErr = processErr
|
lastErr = processErr
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type ProcessorState struct {
|
type ProcessorState struct {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Subscribe subscribes to a topic's specified partitions.
|
// Subscribe subscribes to a topic's specified partitions.
|
||||||
|
|||||||
@@ -43,10 +43,10 @@ func NewConsumerGroupInstance(instanceId string) *ConsumerGroupInstance {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (cg *ConsumerGroup) OnAddConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
|
func (cg *ConsumerGroup) OnAddConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
|
||||||
cg.onConsumerGroupInstanceChange(true, "add consumer instance " + consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
|
cg.onConsumerGroupInstanceChange(true, "add consumer instance "+consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
|
||||||
}
|
}
|
||||||
func (cg *ConsumerGroup) OnRemoveConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
|
func (cg *ConsumerGroup) OnRemoveConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
|
||||||
cg.onConsumerGroupInstanceChange(false, "remove consumer instance " + consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
|
cg.onConsumerGroupInstanceChange(false, "remove consumer instance "+consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cg *ConsumerGroup) onConsumerGroupInstanceChange(isAdd bool, reason string, maxPartitionCount, rebalanceSeconds int32) {
|
func (cg *ConsumerGroup) onConsumerGroupInstanceChange(isAdd bool, reason string, maxPartitionCount, rebalanceSeconds int32) {
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
type FilerClientAccessor struct {
|
type FilerClientAccessor struct {
|
||||||
GetFiler func() pb.ServerAddress
|
GetFiler func() pb.ServerAddress
|
||||||
GetGrpcDialOption func()grpc.DialOption
|
GetGrpcDialOption func() grpc.DialOption
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fca *FilerClientAccessor) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
func (fca *FilerClientAccessor) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ func (imt *InflightMessageTracker) InflightMessage(key []byte, tsNs int64) {
|
|||||||
imt.messages[string(key)] = tsNs
|
imt.messages[string(key)] = tsNs
|
||||||
imt.timestamps.Add(tsNs)
|
imt.timestamps.Add(tsNs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsMessageAcknowledged returns true if the message has been acknowledged.
|
// IsMessageAcknowledged returns true if the message has been acknowledged.
|
||||||
// If the message is older than the oldest inflight messages, returns false.
|
// If the message is older than the oldest inflight messages, returns false.
|
||||||
// returns false if the message is inflight.
|
// returns false if the message is inflight.
|
||||||
@@ -47,6 +48,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64)
|
|||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// AcknowledgeMessage acknowledges the message with the key and timestamp.
|
// AcknowledgeMessage acknowledges the message with the key and timestamp.
|
||||||
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool {
|
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool {
|
||||||
imt.mu.Lock()
|
imt.mu.Lock()
|
||||||
@@ -71,12 +73,14 @@ type RingBuffer struct {
|
|||||||
head int
|
head int
|
||||||
size int
|
size int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRingBuffer creates a new RingBuffer of the given capacity.
|
// NewRingBuffer creates a new RingBuffer of the given capacity.
|
||||||
func NewRingBuffer(capacity int) *RingBuffer {
|
func NewRingBuffer(capacity int) *RingBuffer {
|
||||||
return &RingBuffer{
|
return &RingBuffer{
|
||||||
buffer: make([]int64, capacity),
|
buffer: make([]int64, capacity),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add adds a new timestamp to the ring buffer.
|
// Add adds a new timestamp to the ring buffer.
|
||||||
func (rb *RingBuffer) Add(timestamp int64) {
|
func (rb *RingBuffer) Add(timestamp int64) {
|
||||||
rb.buffer[rb.head] = timestamp
|
rb.buffer[rb.head] = timestamp
|
||||||
@@ -85,6 +89,7 @@ func (rb *RingBuffer) Add(timestamp int64) {
|
|||||||
rb.size++
|
rb.size++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove removes the specified timestamp from the ring buffer.
|
// Remove removes the specified timestamp from the ring buffer.
|
||||||
func (rb *RingBuffer) Remove(timestamp int64) {
|
func (rb *RingBuffer) Remove(timestamp int64) {
|
||||||
// Perform binary search
|
// Perform binary search
|
||||||
|
|||||||
Reference in New Issue
Block a user