optimization: change the configuration file from being read globally … (#1935)
* optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: change the configuration file from being read globally to being read independently. * optimization: config file changed to dependency injection. * fix: replace global config with dependency injection * fix: replace global config with dependency injection * fix: import the enough param * fix: import the enough param * fix: import the enough param * fix: fix the component check of path * fix: fix the kafka of tls is nil problem * fix: fix the TLS.CACrt is nil error * fix: fix the valiable shadows problem * fix: fix the comflect * optimization: message remove options. * fix: fix the param pass error * fix: find error * fix: find error * fix: find eror * fix: find error * fix: find error * fix: del the undifined func * fix: find error * fix: fix the error * fix: pass config * fix: find error * fix: find error * fix: find error * fix: find error * fix: find error * fix: fix the config * fix: fix the error * fix: fix the config pass error * fix: fix the eror * fix: fix the error * fix: fix the error * fix: fix the error * fix: find error * fix: fix the error * fix: fix the config * fix: add return err * fix: fix the err2 * fix: err * fix: fix the func * fix: del the chinese comment * fix: fix the func * fix: fix the gateway_test logic * fix: s3 * test * test * fix: not found --------- Co-authored-by: luhaoling <2198702716@qq.com> Co-authored-by: withchao <993506633@qq.com>
This commit is contained in:
OpenIM-Gordon
@@ -17,6 +17,8 @@ package kafka
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
)
|
||||
@@ -29,22 +31,31 @@ type Consumer struct {
|
||||
Consumer sarama.Consumer
|
||||
}
|
||||
|
||||
func NewKafkaConsumer(addr []string, topic string, kafkaConfig *sarama.Config) (*Consumer, error) {
|
||||
p := Consumer{
|
||||
Topic: topic,
|
||||
addr: addr,
|
||||
func NewKafkaConsumer(addr []string, topic string, config *config.GlobalConfig) (*Consumer,error) {
|
||||
p := Consumer{}
|
||||
p.Topic = topic
|
||||
p.addr = addr
|
||||
consumerConfig := sarama.NewConfig()
|
||||
if config.Kafka.Username != "" && config.Kafka.Password != "" {
|
||||
consumerConfig.Net.SASL.Enable = true
|
||||
consumerConfig.Net.SASL.User = config.Kafka.Username
|
||||
consumerConfig.Net.SASL.Password = config.Kafka.Password
|
||||
}
|
||||
|
||||
if kafkaConfig.Net.SASL.User != "" && kafkaConfig.Net.SASL.Password != "" {
|
||||
kafkaConfig.Net.SASL.Enable = true
|
||||
var tlsConfig *TLSConfig
|
||||
if config.Kafka.TLS != nil {
|
||||
tlsConfig = &TLSConfig{
|
||||
CACrt: config.Kafka.TLS.CACrt,
|
||||
ClientCrt: config.Kafka.TLS.ClientCrt,
|
||||
ClientKey: config.Kafka.TLS.ClientKey,
|
||||
ClientKeyPwd: config.Kafka.TLS.ClientKeyPwd,
|
||||
InsecureSkipVerify: false,
|
||||
}
|
||||
}
|
||||
|
||||
err := SetupTLSConfig(kafkaConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
err:=SetupTLSConfig(consumerConfig, tlsConfig)
|
||||
if err!=nil{
|
||||
return nil,err
|
||||
}
|
||||
|
||||
consumer, err := sarama.NewConsumer(p.addr, kafkaConfig)
|
||||
consumer, err := sarama.NewConsumer(p.addr, consumerConfig)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err, "NewKafkaConsumer: creating consumer failed")
|
||||
}
|
||||
|
||||
@@ -17,12 +17,12 @@ package kafka
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
|
||||
"strings"
|
||||
)
|
||||
|
||||
type MConsumerGroup struct {
|
||||
@@ -35,22 +35,25 @@ type MConsumerGroupConfig struct {
|
||||
KafkaVersion sarama.KafkaVersion
|
||||
OffsetsInitial int64
|
||||
IsReturnErr bool
|
||||
UserName string
|
||||
Password string
|
||||
}
|
||||
|
||||
func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, topics, addrs []string, groupID string) (*MConsumerGroup, error) {
|
||||
func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, topics, addrs []string, groupID string, tlsConfig *TLSConfig) (*MConsumerGroup, error) {
|
||||
consumerGroupConfig := sarama.NewConfig()
|
||||
consumerGroupConfig.Version = consumerConfig.KafkaVersion
|
||||
consumerGroupConfig.Consumer.Offsets.Initial = consumerConfig.OffsetsInitial
|
||||
consumerGroupConfig.Consumer.Return.Errors = consumerConfig.IsReturnErr
|
||||
if config.Config.Kafka.Username != "" && config.Config.Kafka.Password != "" {
|
||||
if consumerConfig.UserName != "" && consumerConfig.Password != "" {
|
||||
consumerGroupConfig.Net.SASL.Enable = true
|
||||
consumerGroupConfig.Net.SASL.User = config.Config.Kafka.Username
|
||||
consumerGroupConfig.Net.SASL.Password = config.Config.Kafka.Password
|
||||
consumerGroupConfig.Net.SASL.User = consumerConfig.UserName
|
||||
consumerGroupConfig.Net.SASL.Password = consumerConfig.Password
|
||||
}
|
||||
SetupTLSConfig(consumerGroupConfig)
|
||||
|
||||
SetupTLSConfig(consumerGroupConfig, tlsConfig)
|
||||
consumerGroup, err := sarama.NewConsumerGroup(addrs, groupID, consumerGroupConfig)
|
||||
if err != nil {
|
||||
return nil, errs.Wrap(err, strings.Join(topics, ","), strings.Join(addrs, ","), groupID, config.Config.Kafka.Username, config.Config.Kafka.Password)
|
||||
return nil, errs.Wrap(err, strings.Join(topics, ","), strings.Join(addrs, ","), groupID, consumerConfig.UserName, consumerConfig.Password)
|
||||
}
|
||||
|
||||
return &MConsumerGroup{
|
||||
|
||||
@@ -22,12 +22,12 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/OpenIMSDK/protocol/constant"
|
||||
"github.com/OpenIMSDK/tools/errs"
|
||||
"github.com/OpenIMSDK/tools/log"
|
||||
"github.com/OpenIMSDK/tools/mcontext"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
@@ -43,8 +43,15 @@ type Producer struct {
|
||||
producer sarama.SyncProducer
|
||||
}
|
||||
|
||||
type ProducerConfig struct {
|
||||
ProducerAck string
|
||||
CompressType string
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
// NewKafkaProducer initializes a new Kafka producer.
|
||||
func NewKafkaProducer(addr []string, topic string) (*Producer, error) {
|
||||
func NewKafkaProducer(addr []string, topic string, producerConfig *ProducerConfig, tlsConfig *TLSConfig) (*Producer, error) {
|
||||
p := Producer{
|
||||
addr: addr,
|
||||
topic: topic,
|
||||
@@ -59,14 +66,14 @@ func NewKafkaProducer(addr []string, topic string) (*Producer, error) {
|
||||
p.config.Producer.Partitioner = sarama.NewHashPartitioner
|
||||
|
||||
// Configure producer acknowledgement level
|
||||
configureProducerAck(&p, config.Config.Kafka.ProducerAck)
|
||||
configureProducerAck(&p, producerConfig.ProducerAck)
|
||||
|
||||
// Configure message compression
|
||||
configureCompression(&p, config.Config.Kafka.CompressType)
|
||||
configureCompression(&p, producerConfig.CompressType)
|
||||
|
||||
// Get Kafka configuration from environment variables or fallback to config file
|
||||
kafkaUsername := getEnvOrConfig("KAFKA_USERNAME", config.Config.Kafka.Username)
|
||||
kafkaPassword := getEnvOrConfig("KAFKA_PASSWORD", config.Config.Kafka.Password)
|
||||
kafkaUsername := getEnvOrConfig("KAFKA_USERNAME", producerConfig.Username)
|
||||
kafkaPassword := getEnvOrConfig("KAFKA_PASSWORD", producerConfig.Password)
|
||||
kafkaAddr := getKafkaAddrFromEnv(addr) // Updated to use the new function
|
||||
|
||||
// Configure SASL authentication if credentials are provided
|
||||
@@ -80,7 +87,7 @@ func NewKafkaProducer(addr []string, topic string) (*Producer, error) {
|
||||
p.addr = kafkaAddr
|
||||
|
||||
// Set up TLS configuration (if required)
|
||||
SetupTLSConfig(p.config)
|
||||
SetupTLSConfig(p.config, tlsConfig)
|
||||
|
||||
// Create the producer with retries
|
||||
var err error
|
||||
|
||||
@@ -20,19 +20,27 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/IBM/sarama"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/tls"
|
||||
)
|
||||
|
||||
type TLSConfig struct {
|
||||
CACrt string
|
||||
ClientCrt string
|
||||
ClientKey string
|
||||
ClientKeyPwd string
|
||||
InsecureSkipVerify bool
|
||||
}
|
||||
|
||||
// SetupTLSConfig set up the TLS config from config file.
|
||||
func SetupTLSConfig(cfg *sarama.Config) error {
|
||||
if config.Config.Kafka.TLS != nil {
|
||||
func SetupTLSConfig(cfg *sarama.Config, tlsConfig *TLSConfig) error {
|
||||
if tlsConfig != nil {
|
||||
cfg.Net.TLS.Enable = true
|
||||
tlsConfig, err := tls.NewTLSConfig(
|
||||
config.Config.Kafka.TLS.ClientCrt,
|
||||
config.Config.Kafka.TLS.ClientKey,
|
||||
config.Config.Kafka.TLS.CACrt,
|
||||
[]byte(config.Config.Kafka.TLS.ClientKeyPwd),
|
||||
tlsConfig.ClientCrt,
|
||||
tlsConfig.ClientKey,
|
||||
tlsConfig.CACrt,
|
||||
[]byte(tlsConfig.ClientKeyPwd),
|
||||
tlsConfig.InsecureSkipVerify,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user