add MongoDB,Redis,Kafka retry mechanism (#518)

pkg/common/db/cache/init_redis.go

@@ -27,6 +27,11 @@ import (
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/errs"
 )
 
+const (
+	maxRetry = 10 //number of retries
+)
+
+// NewRedis Initialize redis connection
 func NewRedis() (redis.UniversalClient, error) {
 	if len(config.Config.Redis.Address) == 0 {
 		return nil, errors.New("redis address is empty")
@@ -35,25 +40,29 @@ func NewRedis() (redis.UniversalClient, error) {
 	var rdb redis.UniversalClient
 	if len(config.Config.Redis.Address) > 1 {
 		rdb = redis.NewClusterClient(&redis.ClusterOptions{
-			Addrs:    config.Config.Redis.Address,
-			Username: config.Config.Redis.Username,
-			Password: config.Config.Redis.Password, // no password set
-			PoolSize: 50,
+			Addrs:      config.Config.Redis.Address,
+			Username:   config.Config.Redis.Username,
+			Password:   config.Config.Redis.Password, // no password set
+			PoolSize:   50,
+			MaxRetries: maxRetry,
 		})
 	} else {
 		rdb = redis.NewClient(&redis.Options{
-			Addr:     config.Config.Redis.Address[0],
-			Username: config.Config.Redis.Username,
-			Password: config.Config.Redis.Password, // no password set
-			DB:       0,                            // use default DB
-			PoolSize: 100,                          // 连接池大小
+			Addr:       config.Config.Redis.Address[0],
+			Username:   config.Config.Redis.Username,
+			Password:   config.Config.Redis.Password, // no password set
+			DB:         0,                            // use default DB
+			PoolSize:   100,                          // connection pool size
+			MaxRetries: maxRetry,
 		})
 	}
+
+	var err error = nil
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()
-	err := rdb.Ping(ctx).Err()
+	err = rdb.Ping(ctx).Err()
 	if err != nil {
 		return nil, fmt.Errorf("redis ping %w", err)
 	}
-	return rdb, nil
+	return rdb, err
 }

pkg/common/db/cache/init_redis_test.go

@@ -0,0 +1,36 @@
+// Copyright © 2023 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+	"fmt"
+	"github.com/OpenIMSDK/Open-IM-Server/pkg/common/config"
+	"testing"
+)
+
+//TestNewRedis Test redis connection
+func TestNewRedis(t *testing.T) {
+	err := config.InitConfig("config_folder_path")
+	if err != nil {
+		fmt.Println("config load error")
+		return
+	}
+	redis, err := NewRedis()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(redis)
+}

pkg/common/db/unrelation/mongo.go

@@ -31,19 +31,21 @@ import (
 	"github.com/OpenIMSDK/Open-IM-Server/pkg/utils"
 )
 
+const (
+	maxRetry = 10 //number of retries
+)
+
 type Mongo struct {
 	db *mongo.Client
 }
 
+// NewMongo Initialize MongoDB connection
 func NewMongo() (*Mongo, error) {
 	specialerror.AddReplace(mongo.ErrNoDocuments, errs.ErrRecordNotFound)
-	uri := "mongodb://sample.host:27017/?maxPoolSize=20&w=majority"
+	url := "mongodb://sample.host:27017/?maxPoolSize=20&w=majority"
 	if config.Config.Mongo.Uri != "" {
-		// example:
-		// mongodb://$user:$password@mongo1.mongo:27017,mongo2.mongo:27017,mongo3.mongo:27017/$DBDatabase/?replicaSet=rs0&readPreference=secondary&authSource=admin&maxPoolSize=$DBMaxPoolSize
-		uri = config.Config.Mongo.Uri
+		url = config.Config.Mongo.Uri
 	} else {
-		//mongodb://mongodb1.example.com:27317,mongodb2.example.com:27017/?replicaSet=mySet&authSource=authDB
 		mongodbHosts := ""
 		for i, v := range config.Config.Mongo.Address {
 			if i == len(config.Config.Mongo.Address)-1 {
@@ -53,23 +55,34 @@ func NewMongo() (*Mongo, error) {
 			}
 		}
 		if config.Config.Mongo.Password != "" && config.Config.Mongo.Username != "" {
-			uri = fmt.Sprintf("mongodb://%s:%s@%s/%s?maxPoolSize=%d&authSource=admin",
+			url = fmt.Sprintf("mongodb://%s:%s@%s/%s?maxPoolSize=%d&authSource=admin",
 				config.Config.Mongo.Username, config.Config.Mongo.Password, mongodbHosts,
 				config.Config.Mongo.Database, config.Config.Mongo.MaxPoolSize)
 		} else {
-			uri = fmt.Sprintf("mongodb://%s/%s/?maxPoolSize=%d&authSource=admin",
+			url = fmt.Sprintf("mongodb://%s/%s/?maxPoolSize=%d&authSource=admin",
 				mongodbHosts, config.Config.Mongo.Database,
 				config.Config.Mongo.MaxPoolSize)
 		}
 	}
-	fmt.Println("mongo:", uri)
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
-	defer cancel()
-	mongoClient, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
-	if err != nil {
-		return nil, err
+	fmt.Println("mongo:", url)
+	var mongoClient *mongo.Client
+	var err error = nil
+	for i := 0; i <= maxRetry; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+		mongoClient, err = mongo.Connect(ctx, options.Client().ApplyURI(url))
+		if err == nil {
+			return &Mongo{db: mongoClient}, nil
+		}
+		if cmdErr, ok := err.(mongo.CommandError); ok {
+			if cmdErr.Code == 13 || cmdErr.Code == 18 {
+				return nil, err
+			} else {
+				fmt.Printf("Failed to connect to MongoDB: %s\n", err)
+			}
+		}
 	}
-	return &Mongo{db: mongoClient}, nil
+	return nil, err
 }
 
 func (m *Mongo) GetClient() *mongo.Client {