fix: refactoring code of graceful exits (#1885)

* fix: plant a layer

* fix: print chanal

* fix: print sigs

* fix: print the sigs

* fix: reconstruct exit gracefully

* fix: fix the timeout

* fix: fix the netDone

* fix: fix the process exit

* fix: refactor the elegant startup code

* fix: fix the Signal.Notify

* fix: fix the code

* fix: remove not used header import.

* Update init.go

* fix: fix the InitConfig error

* fix: fix branch name

* fix: fix the signal value

* fix: replace the signal with SIGTERM

* fix: fix the script

* fix: fix the unsolve error

* fix: return the SIGTERM received,shutting down

* fix: fix the tranfer exit error

* fix: fix the error

* fix: replace the SIGnal

* fix: del the error return in tranfer

* fix: fix SIGTERM error

* fix: del the unreachalbe code

* fix: fix the make stop print  error

---------

Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com>
Co-authored-by: Xinwei Xiong (cubxxw) <3293172751nss@gmail.com>

pkg/common/ginprometheus/ginprometheus.go

@@ -197,30 +197,32 @@ func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Eng
 }
 
 // SetMetricsPath set metrics paths.
-func (p *Prometheus) SetMetricsPath(e *gin.Engine) {
+func (p *Prometheus) SetMetricsPath(e *gin.Engine) error {
 
 	if p.listenAddress != "" {
 		p.router.GET(p.MetricsPath, prometheusHandler())
-		p.runServer()
+		return p.runServer()
 	} else {
 		e.GET(p.MetricsPath, prometheusHandler())
+		return nil
 	}
 }
 
 // SetMetricsPathWithAuth set metrics paths with authentication.
-func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) {
+func (p *Prometheus) SetMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) error {
 
 	if p.listenAddress != "" {
 		p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler())
-		p.runServer()
+		return p.runServer()
 	} else {
 		e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler())
+		return nil
 	}
 
 }
 
-func (p *Prometheus) runServer() {
-	go p.router.Run(p.listenAddress)
+func (p *Prometheus) runServer() error {
+	return p.router.Run(p.listenAddress)
 }
 
 func (p *Prometheus) getMetrics() []byte {
@@ -366,15 +368,15 @@ func (p *Prometheus) registerMetrics(subsystem string) {
 }
 
 // Use adds the middleware to a gin engine.
-func (p *Prometheus) Use(e *gin.Engine) {
+func (p *Prometheus) Use(e *gin.Engine) error {
 	e.Use(p.HandlerFunc())
-	p.SetMetricsPath(e)
+	return p.SetMetricsPath(e)
 }
 
 // UseWithAuth adds the middleware to a gin engine with BasicAuth.
-func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) {
+func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) error {
 	e.Use(p.HandlerFunc())
-	p.SetMetricsPathWithAuth(e, accounts)
+	return p.SetMetricsPathWithAuth(e, accounts)
 }
 
 // HandlerFunc defines handler function for middleware.

pkg/common/kafka/consumer_group.go

@@ -17,19 +17,16 @@ package kafka
 import (
 	"context"
 	"errors"
-	"strings"
-
+	"fmt"
 	"github.com/IBM/sarama"
 	"github.com/OpenIMSDK/tools/errs"
 	"github.com/OpenIMSDK/tools/log"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"strings"
 )
 
 type MConsumerGroup struct {
-	ctx    context.Context
-	cancel context.CancelFunc
-
 	sarama.ConsumerGroup
 	groupID string
 	topics  []string
@@ -57,9 +54,7 @@ func NewMConsumerGroup(consumerConfig *MConsumerGroupConfig, topics, addrs []str
 		return nil, errs.Wrap(err, strings.Join(topics, ","), strings.Join(addrs, ","), groupID, config.Config.Kafka.Username, config.Config.Kafka.Password)
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
 	return &MConsumerGroup{
-		ctx, cancel,
 		consumerGroup,
 		groupID,
 		topics,
@@ -70,27 +65,21 @@ func (mc *MConsumerGroup) GetContextFromMsg(cMsg *sarama.ConsumerMessage) contex
 	return GetContextWithMQHeader(cMsg.Headers)
 }
 
-func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler) {
-	log.ZDebug(context.Background(), "register consumer group", "groupID", mc.groupID)
+func (mc *MConsumerGroup) RegisterHandleAndConsumer(ctx context.Context, handler sarama.ConsumerGroupHandler, onError func(context.Context, error, string)) {
+	log.ZDebug(ctx, "register consumer group", "groupID", mc.groupID)
 	for {
-		err := mc.ConsumerGroup.Consume(mc.ctx, mc.topics, handler)
-		if errors.Is(err, sarama.ErrClosedConsumerGroup) {
+		err := mc.ConsumerGroup.Consume(ctx, mc.topics, handler)
+		if errors.Is(err, sarama.ErrClosedConsumerGroup) || errors.Is(err, context.Canceled) {
 			return
 		}
-		if mc.ctx.Err() != nil {
-			return
-		}
-
 		if err != nil {
-			log.ZWarn(ctx, "consume err", err, "topic", mc.topics, "groupID", mc.groupID)
-		}
-		if ctx.Err() != nil {
+			errInfo := fmt.Sprintf("consume err: %v, topic: %v, groupID: %s", err, strings.Join(mc.topics, ", "), mc.groupID)
+			onError(ctx, err, errInfo) // 调用回调函数处理错误
 			return
 		}
 	}
 }
 
-func (mc *MConsumerGroup) Close() {
-	mc.cancel()
-	mc.ConsumerGroup.Close()
+func (mc *MConsumerGroup) Close() error {
+	return mc.ConsumerGroup.Close()
 }

pkg/common/startrpc/start.go

@@ -15,8 +15,11 @@
 package startrpc
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"github.com/OpenIMSDK/tools/errs"
+	util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil"
 	"net"
 	"net/http"
 	"os"
@@ -26,14 +29,10 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/OpenIMSDK/tools/errs"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"golang.org/x/sync/errgroup"
-
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"google.golang.org/grpc"
@@ -56,12 +55,13 @@ func Start(
 ) error {
 	fmt.Printf("start %s server, port: %d, prometheusPort: %d, OpenIM version: %s\n",
 		rpcRegisterName, rpcPort, prometheusPort, config.Version)
+	rpcTcpAddr := net.JoinHostPort(network.GetListenIP(config.Config.Rpc.ListenIP), strconv.Itoa(rpcPort))
 	listener, err := net.Listen(
 		"tcp",
-		net.JoinHostPort(network.GetListenIP(config.Config.Rpc.ListenIP), strconv.Itoa(rpcPort)),
+		rpcTcpAddr,
 	)
 	if err != nil {
-		return errs.Wrap(err, network.GetListenIP(config.Config.Rpc.ListenIP), strconv.Itoa(rpcPort))
+		return errs.Wrap(err, "rpc start err", rpcTcpAddr)
 	}
 
 	defer listener.Close()
@@ -108,46 +108,64 @@ func Start(
 		return errs.Wrap(err)
 	}
 
-	var wg errgroup.Group
-
-	wg.Go(func() error {
+	var (
+		netDone    = make(chan struct{}, 2)
+		netErr     error
+		httpServer *http.Server
+	)
+	go func() {
 		if config.Config.Prometheus.Enable && prometheusPort != 0 {
 			metric.InitializeMetrics(srv)
 			// Create a HTTP server for prometheus.
-			httpServer := &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)}
-			if err := httpServer.ListenAndServe(); err != nil {
-				fmt.Fprintf(os.Stderr, "\n\nexit -1: \n%+v PrometheusPort: %d \n\n", err, prometheusPort)
-				os.Exit(-1)
+			httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)}
+			if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+				netErr = errs.Wrap(err, "prometheus start err", httpServer.Addr)
+				netDone <- struct{}{}
 			}
 		}
-		return nil
-	})
-
-	wg.Go(func() error {
-		return errs.Wrap(srv.Serve(listener))
-	})
-
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-	<-sigs
-
-	var (
-		done = make(chan struct{}, 1)
-		gerr error
-	)
-
-	go func() {
-		once.Do(srv.GracefulStop)
-		gerr = wg.Wait()
-		close(done)
 	}()
 
+	go func() {
+		err := srv.Serve(listener)
+		if err != nil {
+			netErr = errs.Wrap(err, "rpc start err: ", rpcTcpAddr)
+			netDone <- struct{}{}
+		}
+	}()
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGTERM)
 	select {
-	case <-done:
-		return gerr
-
-	case <-time.After(15 * time.Second):
-		return errs.Wrap(errors.New("timeout exit"))
+	case <-sigs:
+		util.SIGUSR1Exit()
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		if err := gracefulStopWithCtx(ctx, srv.GracefulStop); err != nil {
+			return err
+		}
+		ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second)
+		defer cancel()
+		err := httpServer.Shutdown(ctx)
+		if err != nil {
+			return errs.Wrap(err, "shutdown err")
+		}
+		return errors.New("SIGTERM EXIT")
+	case <-netDone:
+		close(netDone)
+		return netErr
+	}
+}
+
+func gracefulStopWithCtx(ctx context.Context, f func()) error {
+	done := make(chan struct{}, 1)
+	go func() {
+		f()
+		close(done)
+	}()
+	select {
+	case <-ctx.Done():
+		return errs.Wrap(errors.New("timeout, ctx graceful stop"))
+	case <-done:
+		return nil
 	}
-
 }

pkg/util/genutil/genutil.go

@@ -15,6 +15,7 @@
 package genutil
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -39,3 +40,17 @@ func OutDir(path string) (string, error) {
 	outDir += "/"
 	return outDir, nil
 }
+
+func ExitWithError(err error) {
+	if errors.Is(err, errors.New("SIGTERM EXIT")) {
+		os.Exit(-1)
+	}
+	progName := filepath.Base(os.Args[0])
+	fmt.Fprintf(os.Stderr, "\n\n%s exit -1: \n%+v\n\n", progName, err)
+	os.Exit(-1)
+}
+
+func SIGUSR1Exit() {
+	progName := filepath.Base(os.Args[0])
+	fmt.Printf("\n\n%s receive process terminal SIGTERM exit 0\n\n", progName)
+}