Admin UI: Add message queue to admin UI (#6958)

* add a menu item "Message Queue"

* add a menu item "Message Queue"
  * move the "brokers" link under it.
  * add "topics", "subscribers". Add pages for them.

* refactor

* show topic details

* admin display publisher and subscriber info

* remove publisher and subscribers from the topic row pull down

* collecting more stats from publishers and subscribers

* fix layout

* fix publisher name

* add local listeners for mq broker and agent

* render consumer group offsets

* remove subscribers from left menu

* topic with retention

* support editing topic retention

* show retention when listing topics

* create bucket

* Update s3_buckets_templ.go

* embed the static assets into the binary

fix https://github.com/seaweedfs/seaweedfs/issues/6964

weed/pb/mq_broker.proto

@@ -29,6 +29,12 @@ service SeaweedMessaging {
     }
     rpc LookupTopicBrokers (LookupTopicBrokersRequest) returns (LookupTopicBrokersResponse) {
     }
+    rpc GetTopicConfiguration (GetTopicConfigurationRequest) returns (GetTopicConfigurationResponse) {
+    }
+    rpc GetTopicPublishers (GetTopicPublishersRequest) returns (GetTopicPublishersResponse) {
+    }
+    rpc GetTopicSubscribers (GetTopicSubscribersRequest) returns (GetTopicSubscribersResponse) {
+    }
 
     // invoked by the balancer, running on each broker
     rpc AssignTopicPartitions (AssignTopicPartitionsRequest) returns (AssignTopicPartitionsResponse) {
@@ -96,14 +102,21 @@ message BalanceTopicsResponse {
 }
 
 //////////////////////////////////////////////////
+message TopicRetention {
+    int64 retention_seconds = 1; // retention duration in seconds
+    bool enabled = 2; // whether retention is enabled
+}
+
 message ConfigureTopicRequest {
     schema_pb.Topic topic = 1;
     int32 partition_count = 2;
     schema_pb.RecordType record_type = 3;
+    TopicRetention retention = 4;
 }
 message ConfigureTopicResponse {
     repeated BrokerPartitionAssignment broker_partition_assignments = 2;
     schema_pb.RecordType record_type = 3;
+    TopicRetention retention = 4;
 }
 message ListTopicsRequest {
 }
@@ -122,6 +135,57 @@ message BrokerPartitionAssignment {
     string leader_broker = 2;
     string follower_broker = 3;
 }
+message GetTopicConfigurationRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicConfigurationResponse {
+    schema_pb.Topic topic = 1;
+    int32 partition_count = 2;
+    schema_pb.RecordType record_type = 3;
+    repeated BrokerPartitionAssignment broker_partition_assignments = 4;
+    int64 created_at_ns = 5;
+    int64 last_updated_ns = 6;
+    TopicRetention retention = 7;
+}
+
+message GetTopicPublishersRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicPublishersResponse {
+    repeated TopicPublisher publishers = 1;
+}
+
+message GetTopicSubscribersRequest {
+    schema_pb.Topic topic = 1;
+}
+message GetTopicSubscribersResponse {
+    repeated TopicSubscriber subscribers = 1;
+}
+
+message TopicPublisher {
+    string publisher_name = 1;
+    string client_id = 2;
+    schema_pb.Partition partition = 3;
+    int64 connect_time_ns = 4;
+    int64 last_seen_time_ns = 5;
+    string broker = 6;
+    bool is_active = 7;
+    int64 last_published_offset = 8;
+    int64 last_acked_offset = 9;
+}
+
+message TopicSubscriber {
+    string consumer_group = 1;
+    string consumer_id = 2;
+    string client_id = 3;
+    schema_pb.Partition partition = 4;
+    int64 connect_time_ns = 5;
+    int64 last_seen_time_ns = 6;
+    string broker = 7;
+    bool is_active = 8;
+    int64 current_offset = 9; // last acknowledged offset
+    int64 last_received_offset = 10;
+}
 
 message AssignTopicPartitionsRequest {
     schema_pb.Topic topic = 1;

weed/pb/mq_pb/mq_broker_grpc.pb.go

@@ -25,6 +25,9 @@ const (
 	SeaweedMessaging_ListTopics_FullMethodName                 = "/messaging_pb.SeaweedMessaging/ListTopics"
 	SeaweedMessaging_ConfigureTopic_FullMethodName             = "/messaging_pb.SeaweedMessaging/ConfigureTopic"
 	SeaweedMessaging_LookupTopicBrokers_FullMethodName         = "/messaging_pb.SeaweedMessaging/LookupTopicBrokers"
+	SeaweedMessaging_GetTopicConfiguration_FullMethodName      = "/messaging_pb.SeaweedMessaging/GetTopicConfiguration"
+	SeaweedMessaging_GetTopicPublishers_FullMethodName         = "/messaging_pb.SeaweedMessaging/GetTopicPublishers"
+	SeaweedMessaging_GetTopicSubscribers_FullMethodName        = "/messaging_pb.SeaweedMessaging/GetTopicSubscribers"
 	SeaweedMessaging_AssignTopicPartitions_FullMethodName      = "/messaging_pb.SeaweedMessaging/AssignTopicPartitions"
 	SeaweedMessaging_ClosePublishers_FullMethodName            = "/messaging_pb.SeaweedMessaging/ClosePublishers"
 	SeaweedMessaging_CloseSubscribers_FullMethodName           = "/messaging_pb.SeaweedMessaging/CloseSubscribers"
@@ -48,6 +51,9 @@ type SeaweedMessagingClient interface {
 	ListTopics(ctx context.Context, in *ListTopicsRequest, opts ...grpc.CallOption) (*ListTopicsResponse, error)
 	ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error)
 	LookupTopicBrokers(ctx context.Context, in *LookupTopicBrokersRequest, opts ...grpc.CallOption) (*LookupTopicBrokersResponse, error)
+	GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error)
+	GetTopicPublishers(ctx context.Context, in *GetTopicPublishersRequest, opts ...grpc.CallOption) (*GetTopicPublishersResponse, error)
+	GetTopicSubscribers(ctx context.Context, in *GetTopicSubscribersRequest, opts ...grpc.CallOption) (*GetTopicSubscribersResponse, error)
 	// invoked by the balancer, running on each broker
 	AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error)
 	ClosePublishers(ctx context.Context, in *ClosePublishersRequest, opts ...grpc.CallOption) (*ClosePublishersResponse, error)
@@ -133,6 +139,36 @@ func (c *seaweedMessagingClient) LookupTopicBrokers(ctx context.Context, in *Loo
 	return out, nil
 }
 
+func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicConfigurationResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicConfiguration_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicPublishers(ctx context.Context, in *GetTopicPublishersRequest, opts ...grpc.CallOption) (*GetTopicPublishersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicPublishersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicPublishers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicSubscribers(ctx context.Context, in *GetTopicSubscribersRequest, opts ...grpc.CallOption) (*GetTopicSubscribersResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(GetTopicSubscribersResponse)
+	err := c.cc.Invoke(ctx, SeaweedMessaging_GetTopicSubscribers_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedMessagingClient) AssignTopicPartitions(ctx context.Context, in *AssignTopicPartitionsRequest, opts ...grpc.CallOption) (*AssignTopicPartitionsResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(AssignTopicPartitionsResponse)
@@ -241,6 +277,9 @@ type SeaweedMessagingServer interface {
 	ListTopics(context.Context, *ListTopicsRequest) (*ListTopicsResponse, error)
 	ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error)
 	LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error)
+	GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error)
+	GetTopicPublishers(context.Context, *GetTopicPublishersRequest) (*GetTopicPublishersResponse, error)
+	GetTopicSubscribers(context.Context, *GetTopicSubscribersRequest) (*GetTopicSubscribersResponse, error)
 	// invoked by the balancer, running on each broker
 	AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error)
 	ClosePublishers(context.Context, *ClosePublishersRequest) (*ClosePublishersResponse, error)
@@ -281,6 +320,15 @@ func (UnimplementedSeaweedMessagingServer) ConfigureTopic(context.Context, *Conf
 func (UnimplementedSeaweedMessagingServer) LookupTopicBrokers(context.Context, *LookupTopicBrokersRequest) (*LookupTopicBrokersResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method LookupTopicBrokers not implemented")
 }
+func (UnimplementedSeaweedMessagingServer) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicConfiguration not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetTopicPublishers(context.Context, *GetTopicPublishersRequest) (*GetTopicPublishersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicPublishers not implemented")
+}
+func (UnimplementedSeaweedMessagingServer) GetTopicSubscribers(context.Context, *GetTopicSubscribersRequest) (*GetTopicSubscribersResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetTopicSubscribers not implemented")
+}
 func (UnimplementedSeaweedMessagingServer) AssignTopicPartitions(context.Context, *AssignTopicPartitionsRequest) (*AssignTopicPartitionsResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method AssignTopicPartitions not implemented")
 }
@@ -423,6 +471,60 @@ func _SeaweedMessaging_LookupTopicBrokers_Handler(srv interface{}, ctx context.C
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicConfiguration_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicPublishers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicPublishersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicPublishers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicPublishers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicPublishers(ctx, req.(*GetTopicPublishersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicSubscribers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicSubscribersRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicSubscribers(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: SeaweedMessaging_GetTopicSubscribers_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicSubscribers(ctx, req.(*GetTopicSubscribersRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _SeaweedMessaging_AssignTopicPartitions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(AssignTopicPartitionsRequest)
 	if err := dec(in); err != nil {
@@ -539,6 +641,18 @@ var SeaweedMessaging_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "LookupTopicBrokers",
 			Handler:    _SeaweedMessaging_LookupTopicBrokers_Handler,
 		},
+		{
+			MethodName: "GetTopicConfiguration",
+			Handler:    _SeaweedMessaging_GetTopicConfiguration_Handler,
+		},
+		{
+			MethodName: "GetTopicPublishers",
+			Handler:    _SeaweedMessaging_GetTopicPublishers_Handler,
+		},
+		{
+			MethodName: "GetTopicSubscribers",
+			Handler:    _SeaweedMessaging_GetTopicSubscribers_Handler,
+		},
 		{
 			MethodName: "AssignTopicPartitions",
 			Handler:    _SeaweedMessaging_AssignTopicPartitions_Handler,