fix listing topics

weed/mq/broker/broker_grpc_balancer.go

@@ -24,6 +24,7 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
 	if initMessage != nil {
 		broker.Balancer.Brokers.Set(initMessage.Broker, brokerStats)
 	} else {
+		// TODO fix this
 		return status.Errorf(codes.InvalidArgument, "balancer init message is empty")
 	}
 	defer func() {
@@ -42,8 +43,8 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
 		if receivedStats := req.GetStats(); receivedStats != nil {
 			brokerStats.UpdateStats(receivedStats)
 
-			glog.V(3).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
-			glog.V(3).Infof("received stats: %+v", receivedStats)
+			glog.V(4).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
+			glog.V(4).Infof("received stats: %+v", receivedStats)
 		}
 	}
 

weed/mq/broker/broker_grpc_create.go

@@ -2,6 +2,8 @@ package broker
 
 import (
 	"context"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/mq/balancer"
 	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
@@ -29,12 +31,22 @@ func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_p
 	ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
 
 	for _, bpa := range ret.BrokerPartitionAssignments {
+		// fmt.Printf("create topic %s on %s\n", request.Topic, bpa.LeaderBroker)
 		if doCreateErr := broker.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
 			_, doCreateErr := client.DoCreateTopic(ctx, &mq_pb.DoCreateTopicRequest{
 				Topic:     request.Topic,
 				Partition: bpa.Partition,
 			})
-			return doCreateErr
+			if doCreateErr != nil {
+				return fmt.Errorf("do create topic %s on %s: %v", request.Topic, bpa.LeaderBroker, doCreateErr)
+			}
+			brokerStats, found := broker.Balancer.Brokers.Get(bpa.LeaderBroker)
+			if !found {
+				brokerStats = balancer.NewBrokerStats()
+				broker.Balancer.Brokers.Set(bpa.LeaderBroker, brokerStats)
+			}
+			brokerStats.RegisterAssignment(request.Topic, bpa.Partition)
+			return nil
 		}); doCreateErr != nil {
 			return nil, doCreateErr
 		}

weed/mq/broker/broker_grpc_lookup.go

@@ -2,6 +2,7 @@ package broker
 
 import (
 	"context"
+	"fmt"
 	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -62,7 +63,7 @@ func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb
 	}
 
 	ret := &mq_pb.ListTopicsResponse{}
-	knownTopics := make(map[*mq_pb.Topic]struct{})
+	knownTopics := make(map[string]struct{})
 	for brokerStatsItem := range broker.Balancer.Brokers.IterBuffered() {
 		_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
 		for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
@@ -71,9 +72,11 @@ func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb
 				Namespace: topicPartitionStat.TopicPartition.Namespace,
 				Name:      topicPartitionStat.TopicPartition.Topic,
 			}
-			if _, found := knownTopics[topic]; found {
+			topicKey := fmt.Sprintf("%s/%s", topic.Namespace, topic.Name)
+			if _, found := knownTopics[topicKey]; found {
 				continue
 			}
+			knownTopics[topicKey] = struct{}{}
 			ret.Topics = append(ret.Topics, topic)
 		}
 	}

weed/mq/broker/broker_stats.go

@@ -61,6 +61,7 @@ func (broker *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
 			if err != nil {
 				return fmt.Errorf("send stats message: %v", err)
 			}
+			glog.V(4).Infof("sent stats: %+v", stats)
 
 			time.Sleep(time.Millisecond*5000 + time.Duration(rand.Intn(1000))*time.Millisecond)
 		}