This commit is contained in:
chrislu
2024-05-20 11:05:18 -07:00
parent 5038577f7e
commit 3622196881
15 changed files with 58 additions and 56 deletions

View File

@@ -12,8 +12,8 @@ import (
type ConsumerGroupInstance struct {
InstanceId string
// the consumer group instance may not have an active partition
Partitions []*topic.Partition
ResponseChan chan *mq_pb.SubscriberToSubCoordinatorResponse
Partitions []*topic.Partition
ResponseChan chan *mq_pb.SubscriberToSubCoordinatorResponse
MaxPartitionCount int32
}
type ConsumerGroup struct {
@@ -43,10 +43,10 @@ func NewConsumerGroupInstance(instanceId string) *ConsumerGroupInstance {
}
}
func (cg *ConsumerGroup) OnAddConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
cg.onConsumerGroupInstanceChange(true, "add consumer instance " + consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
cg.onConsumerGroupInstanceChange(true, "add consumer instance "+consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
}
func (cg *ConsumerGroup) OnRemoveConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic, maxPartitionCount, rebalanceSeconds int32) {
cg.onConsumerGroupInstanceChange(false, "remove consumer instance " + consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
cg.onConsumerGroupInstanceChange(false, "remove consumer instance "+consumerGroupInstance, maxPartitionCount, rebalanceSeconds)
}
func (cg *ConsumerGroup) onConsumerGroupInstanceChange(isAdd bool, reason string, maxPartitionCount, rebalanceSeconds int32) {

View File

@@ -17,8 +17,8 @@ type TopicConsumerGroups struct {
type Coordinator struct {
// map topic name to consumer groups
TopicSubscribers cmap.ConcurrentMap[string, *TopicConsumerGroups]
balancer *pub_balancer.Balancer
TopicSubscribers cmap.ConcurrentMap[string, *TopicConsumerGroups]
balancer *pub_balancer.Balancer
FilerClientAccessor *FilerClientAccessor
}

View File

@@ -14,8 +14,8 @@ import (
)
type FilerClientAccessor struct {
GetFiler func() pb.ServerAddress
GetGrpcDialOption func()grpc.DialOption
GetFiler func() pb.ServerAddress
GetGrpcDialOption func() grpc.DialOption
}
func (fca *FilerClientAccessor) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {

View File

@@ -6,15 +6,15 @@ import (
)
type InflightMessageTracker struct {
messages map[string]int64
mu sync.Mutex
timestamps *RingBuffer
messages map[string]int64
mu sync.Mutex
timestamps *RingBuffer
}
func NewInflightMessageTracker(capacity int) *InflightMessageTracker {
return &InflightMessageTracker{
messages: make(map[string]int64),
timestamps: NewRingBuffer(capacity),
messages: make(map[string]int64),
timestamps: NewRingBuffer(capacity),
}
}
@@ -26,6 +26,7 @@ func (imt *InflightMessageTracker) InflightMessage(key []byte, tsNs int64) {
imt.messages[string(key)] = tsNs
imt.timestamps.Add(tsNs)
}
// IsMessageAcknowledged returns true if the message has been acknowledged.
// If the message is older than the oldest inflight messages, returns false.
// returns false if the message is inflight.
@@ -47,6 +48,7 @@ func (imt *InflightMessageTracker) IsMessageAcknowledged(key []byte, tsNs int64)
return true
}
// AcknowledgeMessage acknowledges the message with the key and timestamp.
func (imt *InflightMessageTracker) AcknowledgeMessage(key []byte, tsNs int64) bool {
imt.mu.Lock()
@@ -71,12 +73,14 @@ type RingBuffer struct {
head int
size int
}
// NewRingBuffer creates a new RingBuffer of the given capacity.
func NewRingBuffer(capacity int) *RingBuffer {
return &RingBuffer{
buffer: make([]int64, capacity),
}
}
// Add adds a new timestamp to the ring buffer.
func (rb *RingBuffer) Add(timestamp int64) {
rb.buffer[rb.head] = timestamp
@@ -85,6 +89,7 @@ func (rb *RingBuffer) Add(timestamp int64) {
rb.size++
}
}
// Remove removes the specified timestamp from the ring buffer.
func (rb *RingBuffer) Remove(timestamp int64) {
// Perform binary search

View File

@@ -82,10 +82,10 @@ func doBalanceSticky(partitions []*pub_balancer.PartitionSlotToBroker, consumerI
newPartitionSlots := make([]*PartitionSlotToConsumerInstance, 0, len(partitions))
for _, partition := range partitions {
newPartitionSlots = append(newPartitionSlots, &PartitionSlotToConsumerInstance{
RangeStart: partition.RangeStart,
RangeStop: partition.RangeStop,
UnixTimeNs: partition.UnixTimeNs,
Broker: partition.AssignedBroker,
RangeStart: partition.RangeStart,
RangeStop: partition.RangeStop,
UnixTimeNs: partition.UnixTimeNs,
Broker: partition.AssignedBroker,
FollowerBroker: partition.FollowerBroker,
})
}

View File

@@ -32,7 +32,7 @@ func Test_doBalanceSticky(t *testing.T) {
MaxPartitionCount: 1,
},
},
prevMapping: nil,
prevMapping: nil,
},
wantPartitionSlots: []*PartitionSlotToConsumerInstance{
{
@@ -61,7 +61,7 @@ func Test_doBalanceSticky(t *testing.T) {
MaxPartitionCount: 1,
},
},
prevMapping: nil,
prevMapping: nil,
},
wantPartitionSlots: []*PartitionSlotToConsumerInstance{
{
@@ -90,7 +90,7 @@ func Test_doBalanceSticky(t *testing.T) {
MaxPartitionCount: 1,
},
},
prevMapping: nil,
prevMapping: nil,
},
wantPartitionSlots: []*PartitionSlotToConsumerInstance{
{
@@ -128,7 +128,7 @@ func Test_doBalanceSticky(t *testing.T) {
MaxPartitionCount: 1,
},
},
prevMapping: nil,
prevMapping: nil,
},
wantPartitionSlots: []*PartitionSlotToConsumerInstance{
{