gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Accumulated changes for message queue (#6600)

Browse Source 
* rename

* set agent address

* refactor

* add agent sub

* pub messages

* grpc new client

* can publish records via agent

* send init message with session id

* fmt

* check cancelled request while waiting

* use sessionId

* handle possible nil stream

* subscriber process messages

* separate debug port

* use atomic int64

* less logs

* minor

* skip io.EOF

* rename

* remove unused

* use saved offsets

* do not reuse session, since always session id is new after restart

remove last active ts from SessionEntry

* simplify printing

* purge unused

* just proxy the subscription, skipping the session step

* adjust offset types

* subscribe offset type and possible value

* start after the known tsns

* avoid wrongly set startPosition

* move

* remove

* refactor

* typo

* fix

* fix changed path
This commit is contained in:
Chris Lu
2025-03-09 23:49:42 -07:00
committed by GitHub
parent 14cb8a24c6
commit 02773a6107
43 changed files with 1086 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
weed/pb/mq_agent.proto
View File

@@ -21,10 +21,6 @@ service SeaweedMessagingAgent {
}
// Subscribing
rpc StartSubscribeSession (StartSubscribeSessionRequest) returns (StartSubscribeSessionResponse) {
}
rpc CloseSubscribeSession (CloseSubscribeSessionRequest) returns (CloseSubscribeSessionResponse) {
}
rpc SubscribeRecord (stream SubscribeRecordRequest) returns (stream SubscribeRecordResponse) {
}
@@ -59,28 +55,19 @@ message PublishRecordResponse {
string error = 2;
}
//////////////////////////////////////////////////
message StartSubscribeSessionRequest {
string consumer_group = 1;
string consumer_group_instance_id = 2;
schema_pb.Topic topic = 4;
repeated schema_pb.PartitionOffset partition_offsets = 5;
string filter = 6;
int32 max_subscribed_partitions = 8;
int32 sliding_window_size = 9;
}
message StartSubscribeSessionResponse {
string error = 1;
int64 session_id = 2;
}
message CloseSubscribeSessionRequest {
int64 session_id = 1;
}
message CloseSubscribeSessionResponse {
string error = 1;
}
//////////////////////////////////////////////////
message SubscribeRecordRequest {
int64 session_id = 1; // session_id is required for the first record
message InitSubscribeRecordRequest {
string consumer_group = 1;
string consumer_group_instance_id = 2;
schema_pb.Topic topic = 4;
repeated schema_pb.PartitionOffset partition_offsets = 5;
schema_pb.OffsetType offset_type = 6;
int64 offset_ts_ns = 7;
string filter = 10;
int32 max_subscribed_partitions = 11;
int32 sliding_window_size = 12;
}
InitSubscribeRecordRequest init = 1;
int64 ack_sequence = 2;
bytes ack_key = 3;
}