gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

notification.kafka: add SASL authentication and TLS support (#8832)

Browse Source 
* notification.kafka: add SASL authentication and TLS support (#8827)

Wire sarama SASL (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) and TLS
configuration into the Kafka notification producer and consumer,
enabling connections to secured Kafka clusters.

* notification.kafka: validate mTLS config

* kafka notification: validate partial mTLS config, replace panics with errors

- Reject when only one of tls_client_cert/tls_client_key is provided
- Replace three panic() calls in KafkaInput.initialize with returned errors

* kafka notification: enforce minimum TLS 1.2 for Kafka connections
This commit is contained in:
Chris Lu
2026-03-29 13:45:54 -07:00
committed by GitHub
parent 479e72b5ab
commit 937a168d34
7 changed files with 226 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
weed/replication/sub/notification_kafka.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/Shopify/sarama"
"github.com/seaweedfs/seaweedfs/weed/glog"
kafkanotif "github.com/seaweedfs/seaweedfs/weed/notification/kafka"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
@@ -36,25 +37,38 @@ func (k *KafkaInput) Initialize(configuration util.Configuration, prefix string)
configuration.GetString(prefix+"topic"),
configuration.GetString(prefix+"offsetFile"),
configuration.GetInt(prefix+"offsetSaveIntervalSeconds"),
kafkanotif.SASLTLSConfig{
SASLEnabled: configuration.GetBool(prefix + "sasl_enabled"),
SASLMechanism: configuration.GetString(prefix + "sasl_mechanism"),
SASLUsername: configuration.GetString(prefix + "sasl_username"),
SASLPassword: configuration.GetString(prefix + "sasl_password"),
TLSEnabled: configuration.GetBool(prefix + "tls_enabled"),
TLSCACert: configuration.GetString(prefix + "tls_ca_cert"),
TLSClientCert: configuration.GetString(prefix + "tls_client_cert"),
TLSClientKey: configuration.GetString(prefix + "tls_client_key"),
TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
},
)
}
func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int) (err error) {
func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int, saslTLS kafkanotif.SASLTLSConfig) (err error) {
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
if err = kafkanotif.ConfigureSASLTLS(config, saslTLS); err != nil {
return fmt.Errorf("kafka consumer security configuration: %w", err)
}
k.consumer, err = sarama.NewConsumer(hosts, config)
if err != nil {
panic(err)
} else {
glog.V(0).Infof("connected to %v", hosts)
return fmt.Errorf("create kafka consumer: %w", err)
}
glog.V(0).Infof("connected to %v", hosts)
k.topic = topic
k.messageChan = make(chan *sarama.ConsumerMessage, 1)
partitions, err := k.consumer.Partitions(topic)
if err != nil {
panic(err)
return fmt.Errorf("get kafka partitions for topic %q: %w", topic, err)
}
progress := loadProgress(offsetFile)
@@ -77,7 +91,7 @@ func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string,
}
partitionConsumer, err := k.consumer.ConsumePartition(topic, partition, offset)
if err != nil {
panic(err)
return fmt.Errorf("consume kafka topic %q partition %d: %w", topic, partition, err)
}
go func() {
for {