Add message queue agent (#6463)

* scaffold message queue agent

* adjust proto, add mq_agent

* add agent client implementation

* remove unused function

* agent publish server implementation

* adding agent
This commit is contained in:
Chris Lu
2025-01-20 22:19:27 -08:00
committed by GitHub
parent b2f56d9add
commit cc05874d06
57 changed files with 3802 additions and 1562 deletions

View File

@@ -0,0 +1,70 @@
package agent_client
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_agent_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"google.golang.org/grpc"
)
type PublishSession struct {
schema *schema.Schema
partitionCount int
publisherName string
stream grpc.BidiStreamingClient[mq_agent_pb.PublishRecordRequest, mq_agent_pb.PublishRecordResponse]
sessionId int64
}
func NewPublishSession(agentAddress string, topicSchema *schema.Schema, partitionCount int, publisherName string) (*PublishSession, error) {
// call local agent grpc server to create a new session
clientConn, err := pb.GrpcDial(context.Background(), agentAddress, true, grpc.WithInsecure())
if err != nil {
return nil, fmt.Errorf("dial agent server %s: %v", agentAddress, err)
}
agentClient := mq_agent_pb.NewSeaweedMessagingAgentClient(clientConn)
resp, err := agentClient.StartPublishSession(context.Background(), &mq_agent_pb.StartPublishSessionRequest{
Topic: &schema_pb.Topic{
Namespace: topicSchema.Namespace,
Name: topicSchema.Name,
},
PartitionCount: int32(partitionCount),
RecordType: topicSchema.RecordType,
PublisherName: publisherName,
})
if err != nil {
return nil, err
}
if resp.Error != "" {
return nil, fmt.Errorf("start publish session: %v", resp.Error)
}
stream, err := agentClient.PublishRecord(context.Background())
if err != nil {
return nil, fmt.Errorf("publish record: %v", err)
}
return &PublishSession{
schema: topicSchema,
partitionCount: partitionCount,
publisherName: publisherName,
stream: stream,
sessionId: resp.SessionId,
}, nil
}
func (a *PublishSession) CloseSession() error {
if a.schema == nil {
return nil
}
err := a.stream.CloseSend()
if err != nil {
return fmt.Errorf("close send: %v", err)
}
a.schema = nil
return err
}