gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

persist consumer group offset

Browse Source 
1. use one follower
2. read write consumer group offset
This commit is contained in:
chrislu
2024-05-19 00:46:12 -07:00
parent 8d5bb7420d
commit b1871427c3
18 changed files with 609 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
weed/mq/broker/broker_grpc_sub_follow.go
View File

@@ -25,11 +25,7 @@ func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_Sub
}
// create an in-memory offset
subscriberProgress := &SubscriberProgress{
Topic: topic.FromPbTopic(initMessage.Topic),
Partition: topic.FromPbPartition(initMessage.Partition),
ConsumerGroup: "consumer_group",
}
var lastOffset int64
// follow each published messages
for {
@@ -46,8 +42,8 @@ func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_Sub
// Process the received message
if ackMessage := req.GetAck(); ackMessage != nil {
subscriberProgress.Offset = ackMessage.TsNs
println("offset", subscriberProgress.Offset)
lastOffset = ackMessage.TsNs
println("offset", lastOffset)
} else if closeMessage := req.GetClose(); closeMessage != nil {
glog.V(0).Infof("topic %v partition %v subscribe stream closed: %v", initMessage.Topic, initMessage.Partition, closeMessage)
return nil
@@ -58,19 +54,47 @@ func (b *MessageQueueBroker) SubscribeFollowMe(stream mq_pb.SeaweedMessaging_Sub
t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition)
topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
offsetFileName := fmt.Sprintf("%s.offset", subscriberProgress.ConsumerGroup)
err = b.saveConsumerGroupOffset(t, p, initMessage.ConsumerGroup, lastOffset)
offsetBytes := make([]byte, 8)
util.Uint64toBytes(offsetBytes, uint64(subscriberProgress.Offset))
err = b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer.SaveInsideFiler(client, partitionDir, offsetFileName, offsetBytes)
})
glog.V(0).Infof("shut down follower for %v %v", t, p)
glog.V(0).Infof("shut down follower for %v", initMessage)
return err
}
func (b *MessageQueueBroker) readConsumerGroupOffset(initMessage *mq_pb.SubscribeMessageRequest_InitMessage) (offset int64, err error) {
t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.PartitionOffset.Partition)
topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
offsetFileName := fmt.Sprintf("%s.offset", initMessage.ConsumerGroup)
err = b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(client, partitionDir, offsetFileName)
if err != nil {
return err
}
if len(data) != 8 {
return fmt.Errorf("no offset found")
}
offset = int64(util.BytesToUint64(data))
return nil
})
return offset, err
}
func (b *MessageQueueBroker) saveConsumerGroupOffset(t topic.Topic, p topic.Partition, consumerGroup string, offset int64) error {
topicDir := fmt.Sprintf("%s/%s/%s", filer.TopicsDir, t.Namespace, t.Name)
partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT)
partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop)
offsetFileName := fmt.Sprintf("%s.offset", consumerGroup)
offsetBytes := make([]byte, 8)
util.Uint64toBytes(offsetBytes, uint64(offset))
return b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
glog.V(0).Infof("saving topic %s partition %v consumer group %s offset %d", t, p, consumerGroup, offset)
return filer.SaveInsideFiler(client, partitionDir, offsetFileName, offsetBytes)
})
}