gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify

Browse Source
This commit is contained in:
chrislu
2024-03-27 23:42:19 -07:00
parent 7463058299
commit ab9fc0ad70
9 changed files with 41 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
weed/mq/broker/broker_grpc_assign.go
View File

@@ -14,7 +14,6 @@ import (
// AssignTopicPartitions Runs on the assigned broker, to execute the topic partition assignment // AssignTopicPartitions Runs on the assigned broker, to execute the topic partition assignment
func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *mq_pb.AssignTopicPartitionsRequest) (*mq_pb.AssignTopicPartitionsResponse, error) { func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *mq_pb.AssignTopicPartitionsRequest) (*mq_pb.AssignTopicPartitionsResponse, error) {
ret := &mq_pb.AssignTopicPartitionsResponse{} ret := &mq_pb.AssignTopicPartitionsResponse{}
self := pb.ServerAddress(fmt.Sprintf("%s:%d", b.option.Ip, b.option.Port))
// drain existing topic partition subscriptions // drain existing topic partition subscriptions
for _, assignment := range request.BrokerPartitionAssignments { for _, assignment := range request.BrokerPartitionAssignments {
@@ -27,7 +26,7 @@ func (b *MessageQueueBroker) AssignTopicPartitions(c context.Context, request *m
} else { } else {
var localPartition *topic.LocalPartition var localPartition *topic.LocalPartition
if localPartition = b.localTopicManager.GetTopicPartition(t, partition); localPartition == nil { if localPartition = b.localTopicManager.GetTopicPartition(t, partition); localPartition == nil {
localPartition = topic.FromPbBrokerPartitionAssignment(self, partition, assignment, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition)) localPartition = topic.NewLocalPartition(partition, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
b.localTopicManager.AddTopicPartition(t, localPartition) b.localTopicManager.AddTopicPartition(t, localPartition)
} }
} }

2
weed/mq/broker/broker_topic_conf_read_write.go
View File

@@ -88,7 +88,7 @@ func (b *MessageQueueBroker) genLocalPartitionFromFiler(t topic.Topic, partition
self := b.option.BrokerAddress() self := b.option.BrokerAddress()
for _, assignment := range conf.BrokerPartitionAssignments { for _, assignment := range conf.BrokerPartitionAssignments {
if assignment.LeaderBroker == string(self) && partition.Equals(topic.FromPbPartition(assignment.Partition)) { if assignment.LeaderBroker == string(self) && partition.Equals(topic.FromPbPartition(assignment.Partition)) {
localPartition = topic.FromPbBrokerPartitionAssignment(b.option.BrokerAddress(), partition, assignment, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition)) localPartition = topic.NewLocalPartition(partition, b.genLogFlushFunc(t, assignment.Partition), b.genLogOnDiskReadFunc(t, assignment.Partition))
b.localTopicManager.AddTopicPartition(t, localPartition) b.localTopicManager.AddTopicPartition(t, localPartition)
isGenerated = true isGenerated = true
break break

6
weed/mq/pub_balancer/balance_brokers_test.go
View File

@@ -21,8 +21,6 @@ func TestBalanceTopicPartitionOnBrokers(t *testing.T) {
Topic: topic.Topic{Namespace: "topic1", Name: "topic1"}, Topic: topic.Topic{Namespace: "topic1", Name: "topic1"},
Partition: topic.Partition{RangeStart: 0, RangeStop: 512, RingSize: 1024}, Partition: topic.Partition{RangeStart: 0, RangeStop: 512, RingSize: 1024},
}, },
ConsumerCount: 1,
IsLeader: true,
}) })
broker2Stats := &BrokerStats{ broker2Stats := &BrokerStats{
TopicPartitionCount: 2, TopicPartitionCount: 2,
@@ -35,16 +33,12 @@ func TestBalanceTopicPartitionOnBrokers(t *testing.T) {
Topic: topic.Topic{Namespace: "topic1", Name: "topic1"}, Topic: topic.Topic{Namespace: "topic1", Name: "topic1"},
Partition: topic.Partition{RangeStart: 512, RangeStop: 1024, RingSize: 1024}, Partition: topic.Partition{RangeStart: 512, RangeStop: 1024, RingSize: 1024},
}, },
ConsumerCount: 1,
IsLeader: true,
}) })
broker2Stats.TopicPartitionStats.Set("topic2:0", &TopicPartitionStats{ broker2Stats.TopicPartitionStats.Set("topic2:0", &TopicPartitionStats{
TopicPartition: topic.TopicPartition{ TopicPartition: topic.TopicPartition{
Topic: topic.Topic{Namespace: "topic2", Name: "topic2"}, Topic: topic.Topic{Namespace: "topic2", Name: "topic2"},
Partition: topic.Partition{RangeStart: 0, RangeStop: 1024, RingSize: 1024}, Partition: topic.Partition{RangeStart: 0, RangeStop: 1024, RingSize: 1024},
}, },
ConsumerCount: 1,
IsLeader: true,
}) })
brokers.Set("broker1", broker1Stats) brokers.Set("broker1", broker1Stats)
brokers.Set("broker2", broker2Stats) brokers.Set("broker2", broker2Stats)

28
weed/mq/pub_balancer/broker_stats.go
View File

@@ -9,15 +9,16 @@ import (
type BrokerStats struct { type BrokerStats struct {
TopicPartitionCount int32 TopicPartitionCount int32
ConsumerCount int32 PublisherCount int32
SubscriberCount int32
CpuUsagePercent int32 CpuUsagePercent int32
TopicPartitionStats cmap.ConcurrentMap[string, *TopicPartitionStats] // key: topic_partition TopicPartitionStats cmap.ConcurrentMap[string, *TopicPartitionStats] // key: topic_partition
Topics []topic.Topic Topics []topic.Topic
} }
type TopicPartitionStats struct { type TopicPartitionStats struct {
topic.TopicPartition topic.TopicPartition
ConsumerCount int32 PublisherCount int32
IsLeader bool SubscriberCount int32
} }
func NewBrokerStats() *BrokerStats { func NewBrokerStats() *BrokerStats {
@@ -26,15 +27,15 @@ func NewBrokerStats() *BrokerStats {
} }
} }
func (bs *BrokerStats) String() string { func (bs *BrokerStats) String() string {
return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}", return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, Publishers:%d, Subscribers:%d CpuUsagePercent:%d, Stats:%+v}",
bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.TopicPartitionStats.Items()) bs.TopicPartitionCount, bs.PublisherCount, bs.SubscriberCount, bs.CpuUsagePercent, bs.TopicPartitionStats.Items())
} }
func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) { func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
bs.TopicPartitionCount = int32(len(stats.Stats)) bs.TopicPartitionCount = int32(len(stats.Stats))
bs.CpuUsagePercent = stats.CpuUsagePercent bs.CpuUsagePercent = stats.CpuUsagePercent
var consumerCount int32 var publisherCount, subscriberCount int32
currentTopicPartitions := bs.TopicPartitionStats.Items() currentTopicPartitions := bs.TopicPartitionStats.Items()
for _, topicPartitionStats := range stats.Stats { for _, topicPartitionStats := range stats.Stats {
tps := &TopicPartitionStats{ tps := &TopicPartitionStats{
@@ -47,10 +48,11 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
UnixTimeNs: topicPartitionStats.Partition.UnixTimeNs, UnixTimeNs: topicPartitionStats.Partition.UnixTimeNs,
}, },
}, },
ConsumerCount: topicPartitionStats.ConsumerCount, PublisherCount: topicPartitionStats.PublisherCount,
IsLeader: topicPartitionStats.IsLeader, SubscriberCount: topicPartitionStats.SubscriberCount,
} }
consumerCount += topicPartitionStats.ConsumerCount publisherCount += topicPartitionStats.PublisherCount
subscriberCount += topicPartitionStats.SubscriberCount
key := tps.TopicPartition.String() key := tps.TopicPartition.String()
bs.TopicPartitionStats.Set(key, tps) bs.TopicPartitionStats.Set(key, tps)
delete(currentTopicPartitions, key) delete(currentTopicPartitions, key)
@@ -59,8 +61,8 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
for key := range currentTopicPartitions { for key := range currentTopicPartitions {
bs.TopicPartitionStats.Remove(key) bs.TopicPartitionStats.Remove(key)
} }
bs.ConsumerCount = consumerCount bs.PublisherCount = publisherCount
bs.SubscriberCount = subscriberCount
} }
func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Partition, isAdd bool) { func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Partition, isAdd bool) {
@@ -74,8 +76,8 @@ func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Parti
UnixTimeNs: partition.UnixTimeNs, UnixTimeNs: partition.UnixTimeNs,
}, },
}, },
ConsumerCount: 0, PublisherCount: 0,
IsLeader: true, SubscriberCount: 0,
} }
key := tps.TopicPartition.String() key := tps.TopicPartition.String()
if isAdd { if isAdd {

9
weed/mq/pub_balancer/repair.go
View File

@@ -14,8 +14,7 @@ func (balancer *Balancer) RepairTopics() []BalanceAction {
} }
type TopicPartitionInfo struct { type TopicPartitionInfo struct {
Leader string Broker string
Followers []string
} }
// RepairMissingTopicPartitions check the stats of all brokers, // RepairMissingTopicPartitions check the stats of all brokers,
@@ -38,11 +37,7 @@ func RepairMissingTopicPartitions(brokers cmap.ConcurrentMap[string, *BrokerStat
tpi = &TopicPartitionInfo{} tpi = &TopicPartitionInfo{}
topicPartitionToInfo[topicPartitionStat.Partition] = tpi topicPartitionToInfo[topicPartitionStat.Partition] = tpi
} }
if topicPartitionStat.IsLeader { tpi.Broker = broker
tpi.Leader = broker
} else {
tpi.Followers = append(tpi.Followers, broker)
}
} }
} }

3
weed/mq/topic/local_manager.go
View File

@@ -97,7 +97,8 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
Name: localTopic.Name, Name: localTopic.Name,
}, },
Partition: localPartition.Partition.ToPbPartition(), Partition: localPartition.Partition.ToPbPartition(),
ConsumerCount: localPartition.ConsumerCount, PublisherCount: int32(localPartition.Publishers.Size()),
SubscriberCount: int32(localPartition.Subscribers.Size()),
} }
// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition) // fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
} }

16
weed/mq/topic/local_partition.go
View File

@@ -22,10 +22,7 @@ type LocalPartition struct {
ListenersCond *sync.Cond ListenersCond *sync.Cond
Partition Partition
isLeader bool
FollowerBrokers []pb.ServerAddress
LogBuffer *log_buffer.LogBuffer LogBuffer *log_buffer.LogBuffer
ConsumerCount int32
Publishers *LocalPartitionPublishers Publishers *LocalPartitionPublishers
Subscribers *LocalPartitionSubscribers Subscribers *LocalPartitionSubscribers
FollowerId int32 FollowerId int32
@@ -37,11 +34,9 @@ type LocalPartition struct {
var TIME_FORMAT = "2006-01-02-15-04-05" var TIME_FORMAT = "2006-01-02-15-04-05"
func NewLocalPartition(partition Partition, isLeader bool, followerBrokers []pb.ServerAddress, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition { func NewLocalPartition(partition Partition, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
lp := &LocalPartition{ lp := &LocalPartition{
Partition: partition, Partition: partition,
isLeader: isLeader,
FollowerBrokers: followerBrokers,
Publishers: NewLocalPartitionPublishers(), Publishers: NewLocalPartitionPublishers(),
Subscribers: NewLocalPartitionSubscribers(), Subscribers: NewLocalPartitionSubscribers(),
} }
@@ -121,15 +116,6 @@ func (p *LocalPartition) GetEarliestInMemoryMessagePosition() log_buffer.Message
return p.LogBuffer.GetEarliestPosition() return p.LogBuffer.GetEarliestPosition()
} }
func FromPbBrokerPartitionAssignment(self pb.ServerAddress, partition Partition, assignment *mq_pb.BrokerPartitionAssignment, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
isLeader := assignment.LeaderBroker == string(self)
followers := make([]pb.ServerAddress, len(assignment.FollowerBrokers))
for i, followerBroker := range assignment.FollowerBrokers {
followers[i] = pb.ServerAddress(followerBroker)
}
return NewLocalPartition(partition, isLeader, followers, logFlushFn, readFromDiskFn)
}
func (p *LocalPartition) closePublishers() { func (p *LocalPartition) closePublishers() {
p.Publishers.SignalShutdown() p.Publishers.SignalShutdown()
} }

4
weed/pb/mq.proto
View File

@@ -97,8 +97,8 @@ message BrokerStats {
message TopicPartitionStats { message TopicPartitionStats {
Topic topic = 1; Topic topic = 1;
Partition partition = 2; Partition partition = 2;
int32 consumer_count = 3; int32 publisher_count = 3;
bool is_leader = 4; int32 subscriber_count = 4;
} }

27
weed/pb/mq_pb/mq.pb.go
View File

@@ -478,8 +478,8 @@ type TopicPartitionStats struct {
Topic *Topic `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` Topic *Topic `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"` Partition *Partition `protobuf:"bytes,2,opt,name=partition,proto3" json:"partition,omitempty"`
ConsumerCount int32 `protobuf:"varint,3,opt,name=consumer_count,json=consumerCount,proto3" json:"consumer_count,omitempty"` PublisherCount int32 `protobuf:"varint,3,opt,name=publisher_count,json=publisherCount,proto3" json:"publisher_count,omitempty"`
IsLeader bool `protobuf:"varint,4,opt,name=is_leader,json=isLeader,proto3" json:"is_leader,omitempty"` SubscriberCount int32 `protobuf:"varint,4,opt,name=subscriber_count,json=subscriberCount,proto3" json:"subscriber_count,omitempty"`
} }
func (x *TopicPartitionStats) Reset() { func (x *TopicPartitionStats) Reset() {
@@ -528,18 +528,18 @@ func (x *TopicPartitionStats) GetPartition() *Partition {
return nil return nil
} }
func (x *TopicPartitionStats) GetConsumerCount() int32 { func (x *TopicPartitionStats) GetPublisherCount() int32 {
if x != nil { if x != nil {
return x.ConsumerCount return x.PublisherCount
} }
return 0 return 0
} }
func (x *TopicPartitionStats) GetIsLeader() bool { func (x *TopicPartitionStats) GetSubscriberCount() int32 {
if x != nil { if x != nil {
return x.IsLeader return x.SubscriberCount
} }
return false return 0
} }
type PublisherToPubBalancerRequest struct { type PublisherToPubBalancerRequest struct {
@@ -2762,7 +2762,7 @@ var file_mq_proto_rawDesc = []byte{
0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x50, 0x61, 0x72, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x50, 0x61, 0x72,
0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xbb, 0x01, 0x0a, 0x13, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xcb, 0x01, 0x0a, 0x13, 0x54, 0x6f, 0x70, 0x69, 0x63,
0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x29, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x29,
0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70,
@@ -2770,11 +2770,12 @@ var file_mq_proto_rawDesc = []byte{
0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x61, 0x72, 0x74,
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e,
0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6f,
0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x70, 0x75, 0x62, 0x6c, 0x69,
0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x6c, 0x65, 0x73, 0x68, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x73, 0x75, 0x62,
0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20,
0x61, 0x64, 0x65, 0x72, 0x22, 0xd3, 0x01, 0x0a, 0x1d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x43,
0x6f, 0x75, 0x6e, 0x74, 0x22, 0xd3, 0x01, 0x0a, 0x1d, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x73, 0x68,
0x65, 0x72, 0x54, 0x6f, 0x50, 0x75, 0x62, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x52, 0x65, 0x72, 0x54, 0x6f, 0x50, 0x75, 0x62, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4d, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4d, 0x0a, 0x04, 0x69, 0x6e, 0x69, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x69, 0x6e, 0x67,