gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 4827425146
Author: chrislu <chris.lu@gmail.com>
Date:   Sat Sep 16 15:05:38 2023 -0700

    balancer works

commit 3b50139f68
Author: chrislu <chris.lu@gmail.com>
Date:   Fri Sep 15 22:22:32 2023 -0700

    comments

commit 7f685ce7ba
Author: chrislu <chris.lu@gmail.com>
Date:   Fri Sep 15 22:20:05 2023 -0700

    adjust APIs

commit 436d99443b
Author: chrislu <chris.lu@gmail.com>
Date:   Thu Sep 14 23:49:05 2023 -0700

    receive broker stats

commit b771fefa37
Merge: 0a851ec00 890881037
Author: chrislu <chris.lu@gmail.com>
Date:   Wed Sep 13 00:03:47 2023 -0700

    Merge branch 'master' into sub

commit 0a851ec00b
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Sep 10 22:01:25 2023 -0700

    Create balancer.go

commit 39941edc0b
Author: chrislu <chris.lu@gmail.com>
Date:   Thu Sep 7 23:55:19 2023 -0700

    add publisher shutdown

commit 875f562779
Author: chrislu <chris.lu@gmail.com>
Date:   Wed Sep 6 23:16:41 2023 -0700

    server side send response at least once per second

commit 984b6c54cf
Author: chrislu <chris.lu@gmail.com>
Date:   Wed Sep 6 23:15:29 2023 -0700

    ack interval 128

commit 2492a45499
Author: chrislu <chris.lu@gmail.com>
Date:   Wed Sep 6 22:39:46 2023 -0700

    ack interval

commit ba67e6ca29
Author: chrislu <chris.lu@gmail.com>
Date:   Mon Sep 4 21:43:50 2023 -0700

    api for sub

commit 9e4f985698
Author: chrislu <chris.lu@gmail.com>
Date:   Mon Sep 4 21:43:30 2023 -0700

    publish, benchmark

commit cb470d44df
Author: chrislu <chris.lu@gmail.com>
Date:   Fri Sep 1 00:36:51 2023 -0700

    can pub and sub

commit 1eb2da46d5
Author: chrislu <chris.lu@gmail.com>
Date:   Mon Aug 28 09:02:12 2023 -0700

    connect and publish

commit 504ae8383a
Author: chrislu <chris.lu@gmail.com>
Date:   Mon Aug 28 09:01:25 2023 -0700

    protoc version

commit dbcba75271
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Aug 27 18:59:04 2023 -0700

    rename to lookup

commit c9caf33119
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Aug 27 18:33:46 2023 -0700

    move functions

commit 4d6c18d86f
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Aug 27 17:50:59 2023 -0700

    pub sub initial tests

commit 4eb8e8624d
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Aug 27 13:14:39 2023 -0700

    rename

commit 1990456670
Author: chrislu <chris.lu@gmail.com>
Date:   Sun Aug 27 13:13:14 2023 -0700

    sub

commit 905911853d
Author: chrislu <chris.lu@gmail.com>
Date:   Sat Aug 26 13:39:21 2023 -0700

    adjust proto
This commit is contained in:
chrislu
2023-09-16 15:06:16 -07:00
parent 8908810376
commit 89a1fd1751
36 changed files with 2874 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
weed/pb/mq.proto
View File

@@ -20,8 +20,11 @@ service SeaweedMessaging {
rpc CheckBrokerLoad (CheckBrokerLoadRequest) returns (CheckBrokerLoadResponse) {
}
// control plane for balancer
rpc ConnectToBalancer (stream ConnectToBalancerRequest) returns (stream ConnectToBalancerResponse) {
}
// control plane for topic partitions
rpc FindTopicBrokers (FindTopicBrokersRequest) returns (FindTopicBrokersResponse) {
rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
}
// a pub client will call this to get the topic partitions assignment
rpc RequestTopicPartitions (RequestTopicPartitionsRequest) returns (RequestTopicPartitionsResponse) {
@@ -34,6 +37,8 @@ service SeaweedMessaging {
// data plane
rpc Publish (stream PublishRequest) returns (stream PublishResponse) {
}
rpc Subscribe (SubscribeRequest) returns (stream SubscribeResponse) {
}
}
//////////////////////////////////////////////////
@@ -98,23 +103,36 @@ message CheckBrokerLoadResponse {
}
message FindTopicBrokersRequest {
//////////////////////////////////////////////////
message BrokerStats {
int32 topic_partition_count = 1;
int32 consumer_count = 2;
int32 cpu_usage_percent = 3;
}
message ConnectToBalancerRequest {
message InitMessage {
string broker = 1;
}
oneof message {
InitMessage init = 1;
BrokerStats stats = 2;
}
}
message ConnectToBalancerResponse {
}
//////////////////////////////////////////////////
message LookupTopicBrokersRequest {
Topic topic = 1;
bool is_for_publish = 2;
}
message FindTopicBrokersResponse {
message LookupTopicBrokersResponse {
Topic topic = 1;
TopicPartitionsAssignment topic_partitions_assignment = 2;
repeated BrokerPartitionAssignment broker_partition_assignments = 2;
}
message BrokerPartitionsAssignment {
int32 partition_start = 1;
int32 partition_stop = 2;
string leader_broker = 3;
repeated string follower_brokers = 4;
}
message TopicPartitionsAssignment {
int32 partition_count = 1; // over-sharded partitions, usually 1024
repeated BrokerPartitionsAssignment broker_partitions = 2;
message BrokerPartitionAssignment {
Partition partition = 1;
string leader_broker = 2;
repeated string follower_brokers = 3;
}
message RequestTopicPartitionsRequest {
@@ -122,12 +140,12 @@ message RequestTopicPartitionsRequest {
int32 partition_count = 2;
}
message RequestTopicPartitionsResponse {
TopicPartitionsAssignment topic_partitions_assignment = 1;
repeated BrokerPartitionAssignment broker_partition_assignments = 1;
}
message AssignTopicPartitionsRequest {
Topic topic = 1;
TopicPartitionsAssignment topic_partitions_assignment = 2;
repeated BrokerPartitionAssignment broker_partition_assignments = 2;
bool is_leader = 3;
}
message AssignTopicPartitionsResponse {
@@ -136,21 +154,23 @@ message AssignTopicPartitionsResponse {
message CheckTopicPartitionsStatusRequest {
string namespace = 1;
string topic = 2;
BrokerPartitionsAssignment broker_partitions_assignment = 3;
BrokerPartitionAssignment broker_partition_assignment = 3;
bool should_cancel_if_not_match = 4;
}
message CheckTopicPartitionsStatusResponse {
TopicPartitionsAssignment topic_partitions_assignment = 1;
repeated BrokerPartitionAssignment broker_partition_assignments = 1;
}
//////////////////////////////////////////////////
message DataMessage {
bytes key = 1;
bytes value = 2;
}
message PublishRequest {
message InitMessage {
Segment segment = 1;
}
message DataMessage {
bytes key = 1;
bytes value = 2;
Topic topic = 1;
Partition partition = 2;
int32 ack_interval = 3;
}
oneof message {
InitMessage init = 1;
@@ -161,5 +181,33 @@ message PublishRequest {
message PublishResponse {
int64 ack_sequence = 1;
string error = 2;
bool is_closed = 3;
string redirect_to_broker = 3;
}
message SubscribeRequest {
message Consumer {
string consumer_group = 1;
string consumer_id = 2;
string client_id = 3;
}
message Cursor {
Topic topic = 1;
Partition partition = 2;
oneof offset {
int64 start_offset = 3;
int64 start_timestamp_ns = 4;
}
string filter = 5;
}
Consumer consumer = 1;
Cursor cursor = 2;
}
message SubscribeResponse {
message CtrlMessage {
string error = 1;
string redirect_to_broker = 2;
}
oneof message {
CtrlMessage ctrl = 1;
DataMessage data = 2;
}
}