avoid concurrent processing for the same key

weed/mq/broker/broker_grpc_sub.go

@@ -162,6 +162,10 @@ func (b *MessageQueueBroker) SubscribeMessage(stream mq_pb.SeaweedMessaging_Subs
 		// reset the sleep interval count
 		sleepIntervalCount = 0
 
+		for imt.IsInflight(logEntry.Key) {
+			time.Sleep(137 * time.Millisecond)
+		}
+
 		imt.InflightMessage(logEntry.Key, logEntry.TsNs)
 
 		if err := stream.Send(&mq_pb.SubscribeMessageResponse{Message: &mq_pb.SubscribeMessageResponse_Data{

weed/mq/sub_coordinator/inflight_message_tracker.go

@@ -67,6 +67,14 @@ func (imt *InflightMessageTracker) GetOldest() int64 {
 	return imt.timestamps.Oldest()
 }
 
+// IsInflight returns true if the message with the key is inflight.
+func (imt *InflightMessageTracker) IsInflight(key []byte) bool {
+	imt.mu.Lock()
+	defer imt.mu.Unlock()
+	_, found := imt.messages[string(key)]
+	return found
+}
+
 // RingBuffer represents a circular buffer to hold timestamps.
 type RingBuffer struct {
 	buffer []int64