Accumulated changes for message queue (#6600)

* rename

* set agent address

* refactor

* add agent sub

* pub messages

* grpc new client

* can publish records via agent

* send init message with session id

* fmt

* check cancelled request while waiting

* use sessionId

* handle possible nil stream

* subscriber process messages

* separate debug port

* use atomic int64

* less logs

* minor

* skip io.EOF

* rename

* remove unused

* use saved offsets

* do not reuse session, since always session id is new after restart

remove last active ts from SessionEntry

* simplify printing

* purge unused

* just proxy the subscription, skipping the session step

* adjust offset types

* subscribe offset type and possible value

* start after the known tsns

* avoid wrongly set startPosition

* move

* remove

* refactor

* typo

* fix

* fix changed path
This commit is contained in:
Chris Lu
2025-03-09 23:49:42 -07:00
committed by GitHub
parent 14cb8a24c6
commit 02773a6107
43 changed files with 1086 additions and 1624 deletions

View File

@@ -34,8 +34,8 @@ type TopicPublisher struct {
jobs []*EachPartitionPublishJob
}
func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
tp := &TopicPublisher{
func NewTopicPublisher(config *PublisherConfiguration) (tp *TopicPublisher, err error) {
tp = &TopicPublisher{
partition2Buffer: interval.NewSearchTree[*buffered_queue.BufferedQueue[*mq_pb.DataMessage]](func(a, b int32) int {
return int(a - b)
}),
@@ -46,7 +46,7 @@ func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
if err := tp.startSchedulerThread(&wg); err != nil {
if err = tp.startSchedulerThread(&wg); err != nil {
log.Println(err)
return
}
@@ -54,7 +54,7 @@ func NewTopicPublisher(config *PublisherConfiguration) *TopicPublisher {
wg.Wait()
return tp
return
}
func (p *TopicPublisher) Shutdown() error {