mirror of
https://github.com/openimsdk/open-im-server.git
synced 2026-05-16 06:49:02 +08:00
conflict
This commit is contained in:
withchao
@@ -0,0 +1,96 @@
|
||||
// Copyright © 2024 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package direct
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"strings"
|
||||
|
||||
"github.com/openimsdk/tools/log"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
const (
|
||||
slashSeparator = "/"
|
||||
// EndpointSepChar is the separator char in endpoints.
|
||||
EndpointSepChar = ','
|
||||
|
||||
subsetSize = 32
|
||||
scheme = "direct"
|
||||
)
|
||||
|
||||
type ResolverDirect struct {
|
||||
}
|
||||
|
||||
func NewResolverDirect() *ResolverDirect {
|
||||
return &ResolverDirect{}
|
||||
}
|
||||
|
||||
func (rd *ResolverDirect) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (
|
||||
resolver.Resolver, error) {
|
||||
log.ZDebug(context.Background(), "Build", "target", target)
|
||||
endpoints := strings.FieldsFunc(GetEndpoints(target), func(r rune) bool {
|
||||
return r == EndpointSepChar
|
||||
})
|
||||
endpoints = subset(endpoints, subsetSize)
|
||||
addrs := make([]resolver.Address, 0, len(endpoints))
|
||||
|
||||
for _, val := range endpoints {
|
||||
addrs = append(addrs, resolver.Address{
|
||||
Addr: val,
|
||||
})
|
||||
}
|
||||
if err := cc.UpdateState(resolver.State{
|
||||
Addresses: addrs,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &nopResolver{cc: cc}, nil
|
||||
}
|
||||
func init() {
|
||||
resolver.Register(&ResolverDirect{})
|
||||
}
|
||||
func (rd *ResolverDirect) Scheme() string {
|
||||
return scheme // return your custom scheme name
|
||||
}
|
||||
|
||||
// GetEndpoints returns the endpoints from the given target.
|
||||
func GetEndpoints(target resolver.Target) string {
|
||||
return strings.Trim(target.URL.Path, slashSeparator)
|
||||
}
|
||||
func subset(set []string, sub int) []string {
|
||||
rand.Shuffle(len(set), func(i, j int) {
|
||||
set[i], set[j] = set[j], set[i]
|
||||
})
|
||||
if len(set) <= sub {
|
||||
return set
|
||||
}
|
||||
|
||||
return set[:sub]
|
||||
}
|
||||
|
||||
type nopResolver struct {
|
||||
cc resolver.ClientConn
|
||||
}
|
||||
|
||||
func (n nopResolver) ResolveNow(options resolver.ResolveNowOptions) {
|
||||
|
||||
}
|
||||
|
||||
func (n nopResolver) Close() {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright © 2024 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package direct
|
||||
|
||||
//import (
|
||||
// "context"
|
||||
// "fmt"
|
||||
//
|
||||
// config2 "github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
// "github.com/openimsdk/tools/errs"
|
||||
// "google.golang.org/grpc"
|
||||
// "google.golang.org/grpc/credentials/insecure"
|
||||
//)
|
||||
//
|
||||
//type ServiceAddresses map[string][]int
|
||||
//
|
||||
//func getServiceAddresses(rpcRegisterName *config2.RpcRegisterName,
|
||||
// rpcPort *config2.RpcPort, longConnSvrPort []int) ServiceAddresses {
|
||||
// return ServiceAddresses{
|
||||
// rpcRegisterName.OpenImUserName: rpcPort.OpenImUserPort,
|
||||
// rpcRegisterName.OpenImFriendName: rpcPort.OpenImFriendPort,
|
||||
// rpcRegisterName.OpenImMsgName: rpcPort.OpenImMessagePort,
|
||||
// rpcRegisterName.OpenImMessageGatewayName: longConnSvrPort,
|
||||
// rpcRegisterName.OpenImGroupName: rpcPort.OpenImGroupPort,
|
||||
// rpcRegisterName.OpenImAuthName: rpcPort.OpenImAuthPort,
|
||||
// rpcRegisterName.OpenImPushName: rpcPort.OpenImPushPort,
|
||||
// rpcRegisterName.OpenImConversationName: rpcPort.OpenImConversationPort,
|
||||
// rpcRegisterName.OpenImThirdName: rpcPort.OpenImThirdPort,
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//type ConnDirect struct {
|
||||
// additionalOpts []grpc.DialOption
|
||||
// currentServiceAddress string
|
||||
// conns map[string][]*grpc.ClientConn
|
||||
// resolverDirect *ResolverDirect
|
||||
// config *config2.GlobalConfig
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetClientLocalConns() map[string][]*grpc.ClientConn {
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) {
|
||||
// return "", nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) Register(serviceName, host string, port int, opts ...grpc.DialOption) error {
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) UnRegister() error {
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) CreateRpcRootNodes(serviceNames []string) error {
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) RegisterConf2Registry(key string, conf []byte) error {
|
||||
// return nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetConfFromRegistry(key string) ([]byte, error) {
|
||||
// return nil, nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) Close() {
|
||||
//
|
||||
//}
|
||||
//
|
||||
//func NewConnDirect(config *config2.GlobalConfig) (*ConnDirect, error) {
|
||||
// return &ConnDirect{
|
||||
// conns: make(map[string][]*grpc.ClientConn),
|
||||
// resolverDirect: NewResolverDirect(),
|
||||
// config: config,
|
||||
// }, nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetConns(ctx context.Context,
|
||||
// serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) {
|
||||
//
|
||||
// if conns, exists := cd.conns[serviceName]; exists {
|
||||
// return conns, nil
|
||||
// }
|
||||
// ports := getServiceAddresses(&cd.config.RpcRegisterName,
|
||||
// &cd.config.RpcPort, cd.config.LongConnSvr.OpenImMessageGatewayPort)[serviceName]
|
||||
// var connections []*grpc.ClientConn
|
||||
// for _, port := range ports {
|
||||
// conn, err := cd.dialServiceWithoutResolver(ctx, fmt.Sprintf(cd.config.Rpc.ListenIP+":%d", port), append(cd.additionalOpts, opts...)...)
|
||||
// if err != nil {
|
||||
// return nil, errs.Wrap(fmt.Errorf("connect to port %d failed,serviceName %s, IP %s", port, serviceName, cd.config.Rpc.ListenIP))
|
||||
// }
|
||||
// connections = append(connections, conn)
|
||||
// }
|
||||
//
|
||||
// if len(connections) == 0 {
|
||||
// return nil, errs.New("no connections found for service", "serviceName", serviceName).Wrap()
|
||||
// }
|
||||
// return connections, nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
// // Get service addresses
|
||||
// addresses := getServiceAddresses(&cd.config.RpcRegisterName,
|
||||
// &cd.config.RpcPort, cd.config.LongConnSvr.OpenImMessageGatewayPort)
|
||||
// address, ok := addresses[serviceName]
|
||||
// if !ok {
|
||||
// return nil, errs.New("unknown service name", "serviceName", serviceName).Wrap()
|
||||
// }
|
||||
// var result string
|
||||
// for _, addr := range address {
|
||||
// if result != "" {
|
||||
// result = result + "," + fmt.Sprintf(cd.config.Rpc.ListenIP+":%d", addr)
|
||||
// } else {
|
||||
// result = fmt.Sprintf(cd.config.Rpc.ListenIP+":%d", addr)
|
||||
// }
|
||||
// }
|
||||
// // Try to dial a new connection
|
||||
// conn, err := cd.dialService(ctx, result, append(cd.additionalOpts, opts...)...)
|
||||
// if err != nil {
|
||||
// return nil, errs.WrapMsg(err, "address", result)
|
||||
// }
|
||||
//
|
||||
// // Store the new connection
|
||||
// cd.conns[serviceName] = append(cd.conns[serviceName], conn)
|
||||
// return conn, nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) GetSelfConnTarget() string {
|
||||
// return cd.currentServiceAddress
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) AddOption(opts ...grpc.DialOption) {
|
||||
// cd.additionalOpts = append(cd.additionalOpts, opts...)
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) CloseConn(conn *grpc.ClientConn) {
|
||||
// if conn != nil {
|
||||
// conn.Close()
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) dialService(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
// options := append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
// conn, err := grpc.DialContext(ctx, cd.resolverDirect.Scheme()+":///"+address, options...)
|
||||
//
|
||||
// if err != nil {
|
||||
// return nil, errs.WrapMsg(err, "address", address)
|
||||
// }
|
||||
// return conn, nil
|
||||
//}
|
||||
//
|
||||
//func (cd *ConnDirect) dialServiceWithoutResolver(ctx context.Context, address string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
// options := append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
// conn, err := grpc.DialContext(ctx, address, options...)
|
||||
//
|
||||
// if err != nil {
|
||||
// return nil, errs.Wrap(err)
|
||||
// }
|
||||
// return conn, nil
|
||||
//}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright © 2024 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package direct // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/direct"
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
|
||||
"github.com/openimsdk/tools/discovery"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/openimsdk/tools/discovery/kubernetes"
|
||||
|
||||
"github.com/openimsdk/tools/discovery/etcd"
|
||||
"github.com/openimsdk/tools/errs"
|
||||
)
|
||||
|
||||
// NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type.
|
||||
func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string, watchNames []string) (discovery.SvcDiscoveryRegistry, error) {
|
||||
if runtimeEnv == config.KUBERNETES {
|
||||
return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace,
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallSendMsgSize(1024*1024*20),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
switch discovery.Enable {
|
||||
case config.ETCD:
|
||||
return etcd.NewSvcDiscoveryRegistry(
|
||||
discovery.Etcd.RootDirectory,
|
||||
discovery.Etcd.Address,
|
||||
watchNames,
|
||||
etcd.WithDialTimeout(10*time.Second),
|
||||
etcd.WithMaxCallSendMsgSize(20*1024*1024),
|
||||
etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password))
|
||||
default:
|
||||
return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright © 2023 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
func setupTestEnvironment() {
|
||||
os.Setenv("ZOOKEEPER_SCHEMA", "openim")
|
||||
os.Setenv("ZOOKEEPER_ADDRESS", "172.28.0.1")
|
||||
os.Setenv("ZOOKEEPER_PORT", "12181")
|
||||
os.Setenv("ZOOKEEPER_USERNAME", "")
|
||||
os.Setenv("ZOOKEEPER_PASSWORD", "")
|
||||
}
|
||||
|
||||
//func TestNewDiscoveryRegister(t *testing.T) {
|
||||
// setupTestEnvironment()
|
||||
// conf := config.NewGlobalConfig()
|
||||
// tests := []struct {
|
||||
// envType string
|
||||
// gatewayName string
|
||||
// expectedError bool
|
||||
// expectedResult bool
|
||||
// }{
|
||||
// {"zookeeper", "MessageGateway", false, true},
|
||||
// {"k8s", "MessageGateway", false, true},
|
||||
// {"direct", "MessageGateway", false, true},
|
||||
// {"invalid", "MessageGateway", true, false},
|
||||
// }
|
||||
//
|
||||
// for _, test := range tests {
|
||||
// conf.Envs.Discovery = test.envType
|
||||
// conf.RpcRegisterName.OpenImMessageGatewayName = test.gatewayName
|
||||
// client, err := NewDiscoveryRegister(conf)
|
||||
//
|
||||
// if test.expectedError {
|
||||
// assert.Error(t, err)
|
||||
// } else {
|
||||
// assert.NoError(t, err)
|
||||
// if test.expectedResult {
|
||||
// assert.Implements(t, (*discovery.SvcDiscoveryRegistry)(nil), client)
|
||||
// } else {
|
||||
// assert.Nil(t, client)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright © 2024 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package discovery // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery"
|
||||
@@ -0,0 +1,9 @@
|
||||
package etcd
|
||||
|
||||
const (
|
||||
ConfigKeyPrefix = "/open-im/config/"
|
||||
RestartKey = "restart"
|
||||
EnableConfigCenterKey = "enable-config-center"
|
||||
Enable = "enable"
|
||||
Disable = "disable"
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright © 2024 OpenIM. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/kubernetes"
|
||||
@@ -0,0 +1,270 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
type KubernetesConnManager struct {
|
||||
clientset *kubernetes.Clientset
|
||||
namespace string
|
||||
dialOptions []grpc.DialOption
|
||||
|
||||
rpcTargets map[string]string
|
||||
selfTarget string
|
||||
|
||||
mu sync.RWMutex
|
||||
connMap map[string][]*grpc.ClientConn
|
||||
}
|
||||
|
||||
// NewKubernetesConnManager creates a new connection manager that uses Kubernetes services for service discovery.
|
||||
func NewKubernetesConnManager(namespace string, options ...grpc.DialOption) (*KubernetesConnManager, error) {
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create in-cluster config: %v", err)
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create clientset: %v", err)
|
||||
}
|
||||
|
||||
k := &KubernetesConnManager{
|
||||
clientset: clientset,
|
||||
namespace: namespace,
|
||||
dialOptions: options,
|
||||
connMap: make(map[string][]*grpc.ClientConn),
|
||||
}
|
||||
|
||||
go k.watchEndpoints()
|
||||
|
||||
return k, nil
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) initializeConns(serviceName string) error {
|
||||
port, err := k.getServicePort(serviceName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
endpoints, err := k.clientset.CoreV1().Endpoints(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get endpoints for service %s: %v", serviceName, err)
|
||||
}
|
||||
|
||||
// fmt.Println("Endpoints:", endpoints, "endpoints.Subsets:", endpoints.Subsets)
|
||||
|
||||
var conns []*grpc.ClientConn
|
||||
for _, subset := range endpoints.Subsets {
|
||||
for _, address := range subset.Addresses {
|
||||
target := fmt.Sprintf("%s:%d", address.IP, port)
|
||||
// fmt.Println("IP target:", target)
|
||||
conn, err := grpc.Dial(target, append(k.dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to dial endpoint %s: %v", target, err)
|
||||
}
|
||||
conns = append(conns, conn)
|
||||
}
|
||||
}
|
||||
|
||||
k.mu.Lock()
|
||||
k.connMap[serviceName] = conns
|
||||
k.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetConns returns gRPC client connections for a given Kubernetes service name.
|
||||
func (k *KubernetesConnManager) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) {
|
||||
k.mu.RLock()
|
||||
|
||||
conns, exists := k.connMap[serviceName]
|
||||
k.mu.RUnlock()
|
||||
if exists {
|
||||
return conns, nil
|
||||
}
|
||||
|
||||
k.mu.Lock()
|
||||
// Check if another goroutine has already initialized the connections when we released the read lock
|
||||
conns, exists = k.connMap[serviceName]
|
||||
if exists {
|
||||
return conns, nil
|
||||
}
|
||||
k.mu.Unlock()
|
||||
|
||||
if err := k.initializeConns(serviceName); err != nil {
|
||||
fmt.Println("Failed to initialize connections:", err)
|
||||
return nil, fmt.Errorf("failed to initialize connections for service %s: %v", serviceName, err)
|
||||
}
|
||||
|
||||
return k.connMap[serviceName], nil
|
||||
}
|
||||
|
||||
// GetConn returns a single gRPC client connection for a given Kubernetes service name.
|
||||
func (k *KubernetesConnManager) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
||||
var target string
|
||||
|
||||
if k.rpcTargets[serviceName] == "" {
|
||||
var err error
|
||||
|
||||
svcPort, err := k.getServicePort(serviceName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
target = fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceName, k.namespace, svcPort)
|
||||
|
||||
// fmt.Println("SVC target:", target)
|
||||
} else {
|
||||
target = k.rpcTargets[serviceName]
|
||||
}
|
||||
|
||||
return grpc.DialContext(
|
||||
ctx,
|
||||
target,
|
||||
append([]grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024*1024*10), grpc.MaxCallSendMsgSize(1024*1024*20)),
|
||||
}, k.dialOptions...)...,
|
||||
)
|
||||
}
|
||||
|
||||
// GetSelfConnTarget returns the connection target for the current service.
|
||||
func (k *KubernetesConnManager) GetSelfConnTarget() string {
|
||||
if k.selfTarget == "" {
|
||||
hostName := os.Getenv("HOSTNAME")
|
||||
|
||||
pod, err := k.clientset.CoreV1().Pods(k.namespace).Get(context.Background(), hostName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Printf("failed to get pod %s: %v \n", hostName, err)
|
||||
}
|
||||
|
||||
for pod.Status.PodIP == "" {
|
||||
pod, err = k.clientset.CoreV1().Pods(k.namespace).Get(context.TODO(), hostName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
log.Printf("Error getting pod: %v \n", err)
|
||||
}
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
var selfPort int32
|
||||
|
||||
for _, port := range pod.Spec.Containers[0].Ports {
|
||||
if port.ContainerPort != 10001 {
|
||||
selfPort = port.ContainerPort
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
k.selfTarget = fmt.Sprintf("%s:%d", pod.Status.PodIP, selfPort)
|
||||
}
|
||||
|
||||
return k.selfTarget
|
||||
}
|
||||
|
||||
// AddOption appends gRPC dial options to the existing options.
|
||||
func (k *KubernetesConnManager) AddOption(opts ...grpc.DialOption) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
k.dialOptions = append(k.dialOptions, opts...)
|
||||
}
|
||||
|
||||
// CloseConn closes a given gRPC client connection.
|
||||
func (k *KubernetesConnManager) CloseConn(conn *grpc.ClientConn) {
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// Close closes all gRPC connections managed by KubernetesConnManager.
|
||||
func (k *KubernetesConnManager) Close() {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
for _, conns := range k.connMap {
|
||||
for _, conn := range conns {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
k.connMap = make(map[string][]*grpc.ClientConn)
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) Register(serviceName, host string, port int, opts ...grpc.DialOption) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) UnRegister() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) getServicePort(serviceName string) (int32, error) {
|
||||
var svcPort int32
|
||||
|
||||
svc, err := k.clientset.CoreV1().Services(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
fmt.Print("namespace:", k.namespace)
|
||||
return 0, fmt.Errorf("failed to get service %s: %v", serviceName, err)
|
||||
}
|
||||
|
||||
if len(svc.Spec.Ports) == 0 {
|
||||
return 0, fmt.Errorf("service %s has no ports defined", serviceName)
|
||||
}
|
||||
|
||||
for _, port := range svc.Spec.Ports {
|
||||
// fmt.Println(serviceName, " Now Get Port:", port.Port)
|
||||
if port.Port != 10001 {
|
||||
svcPort = port.Port
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return svcPort, nil
|
||||
}
|
||||
|
||||
// watchEndpoints listens for changes in Pod resources.
|
||||
func (k *KubernetesConnManager) watchEndpoints() {
|
||||
informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute*10)
|
||||
informer := informerFactory.Core().V1().Pods().Informer()
|
||||
|
||||
// Watch for Pod changes (add, update, delete)
|
||||
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
k.handleEndpointChange(obj)
|
||||
},
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
k.handleEndpointChange(newObj)
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
k.handleEndpointChange(obj)
|
||||
},
|
||||
})
|
||||
|
||||
informerFactory.Start(context.Background().Done())
|
||||
<-context.Background().Done() // Block forever
|
||||
}
|
||||
|
||||
func (k *KubernetesConnManager) handleEndpointChange(obj interface{}) {
|
||||
endpoint, ok := obj.(*v1.Endpoints)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
serviceName := endpoint.Name
|
||||
if err := k.initializeConns(serviceName); err != nil {
|
||||
fmt.Printf("Error initializing connections for %s: %v\n", serviceName, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user