fix: add autoPort && prometheus port discovery

2026-05-11 12:36:00 +08:00 · 2024-12-10 11:43:19 +08:00
parent 7912ac0623
commit 1e68f99d11
38 changed files with 670 additions and 253 deletions
@@ -16,6 +16,7 @@ package startrpc

 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -25,8 +26,10 @@ import (
 	"syscall"
 	"time"

-	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	conf "github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/tools/discovery/etcd"
 	"github.com/openimsdk/tools/utils/datautil"
+	"github.com/openimsdk/tools/utils/jsonutil"
 	"google.golang.org/grpc/status"

 	kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister"
@@ -41,29 +44,47 @@ import (
 )

 // Start rpc server.
-func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP,
-	registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T,
+func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP,
+	registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *conf.Share, config T,
 	watchServiceNames []string,
-	rpcFn func(ctx context.Context,
-		config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error {
+	rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error,
+	options ...grpc.ServerOption) error {

-	rpcPort, err := datautil.GetElemByIndex(rpcPorts, index)
+	var (
+		rpcTcpAddr     string
+		netDone        = make(chan struct{}, 2)
+		netErr         error
+		prometheusPort int
+	)
+
+	registerIP, err := network.GetRpcRegisterIP(registerIP)
 	if err != nil {
 		return err
 	}

-	log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort,
-		"prometheusPorts", prometheusConfig.Ports)
-	rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort))
-
-	listener, err := net.Listen(
-		"tcp",
-		rpcTcpAddr,
-	)
-	if err != nil {
-		return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr)
+	if !autoSetPorts {
+		rpcPort, err := datautil.GetElemByIndex(rpcPorts, index)
+		if err != nil {
+			return err
+		}
+		rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort))
+	} else {
+		rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0")
+	}
+
+	getAutoPort := func() (net.Listener, int, error) {
+		listener, err := net.Listen("tcp", rpcTcpAddr)
+		if err != nil {
+			return nil, 0, errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr)
+		}
+		_, portStr, _ := net.SplitHostPort(listener.Addr().String())
+		port, _ := strconv.Atoi(portStr)
+		return listener, port, nil
+	}
+
+	if autoSetPorts && discovery.Enable != conf.ETCD {
+		return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap()
 	}
-	defer listener.Close()
 	client, err := kdisc.NewDiscoveryRegister(discovery, share, watchServiceNames)
 	if err != nil {
 		return err
@@ -75,19 +96,69 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo
 	// var reg *prometheus.Registry
 	// var metric *grpcprometheus.ServerMetrics
 	if prometheusConfig.Enable {
-		//cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share)
-		//reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics)
-		//options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()),
+		// cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share)
+		// reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics)
+		// options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()),
 		//	grpc.UnaryInterceptor(metric.UnaryServerInterceptor()))
 		options = append(
 			options, mw.GrpcServer(),
 			prommetricsUnaryInterceptor(rpcRegisterName),
 			prommetricsStreamInterceptor(rpcRegisterName),
 		)
+
+		var (
+			listener net.Listener
+		)
+
+		if autoSetPorts {
+			listener, prometheusPort, err = getAutoPort()
+			if err != nil {
+				return err
+			}
+
+			etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient()
+
+			_, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort)))
+			if err != nil {
+				return errs.WrapMsg(err, "etcd put err")
+			}
+		} else {
+			prometheusPort, err = datautil.GetElemByIndex(prometheusConfig.Ports, index)
+			if err != nil {
+				return err
+			}
+			listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort))
+			if err != nil {
+				return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr)
+			}
+		}
+		cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share)
+		go func() {
+			if err := prommetrics.RpcInit(cs, listener); err != nil && !errors.Is(err, http.ErrServerClosed) {
+				netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort))
+				netDone <- struct{}{}
+			}
+			//metric.InitializeMetrics(srv)
+			// Create a HTTP server for prometheus.
+			// httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)}
+			// if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			//	netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr)
+			//	netDone <- struct{}{}
+			// }
+		}()
 	} else {
 		options = append(options, mw.GrpcServer())
 	}

+	listener, port, err := getAutoPort()
+	if err != nil {
+		return err
+	}
+
+	log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", port,
+		"prometheusPort", prometheusPort)
+
+	defer listener.Close()
 	srv := grpc.NewServer(options...)

 	err = rpcFn(ctx, config, client, srv)
@@ -98,43 +169,16 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo
 	err = client.Register(
 		rpcRegisterName,
 		registerIP,
-		rpcPort,
+		port,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
 	)
 	if err != nil {
 		return err
 	}

-	var (
-		netDone = make(chan struct{}, 2)
-		netErr  error
-	)
-	if prometheusConfig.Enable {
-		go func() {
-			prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index)
-			if err != nil {
-				netErr = err
-				netDone <- struct{}{}
-				return
-			}
-			cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share)
-			if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed {
-				netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort))
-				netDone <- struct{}{}
-			}
-			//metric.InitializeMetrics(srv)
-			// Create a HTTP server for prometheus.
-			//httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)}
-			//if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			//	netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr)
-			//	netDone <- struct{}{}
-			//}
-		}()
-	}
-
 	go func() {
 		err := srv.Serve(listener)
-		if err != nil {
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
 			netErr = errs.WrapMsg(err, "rpc start err: ", rpcTcpAddr)
 			netDone <- struct{}{}
 		}