From e6f1e3b4c68d4e09fa08a6e5c4deea0dd704f6ad Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Thu, 11 Jan 2024 15:05:47 +0800 Subject: [PATCH] Allow using a dedicated port for ALS receiver (#11749) --- docs/en/changes/changes.md | 1 + .../en/setup/backend/backend-load-balancer.md | 48 +++++++++++++- .../setup/backend/configuration-vocabulary.md | 8 +++ .../envoy/EnvoyMetricReceiverConfig.java | 19 ++++++ .../envoy/EnvoyMetricReceiverProvider.java | 64 +++++++++++++++++-- .../src/main/resources/application.yml | 9 +++ 6 files changed, 143 insertions(+), 6 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 16d6d3cc6b..477673fe03 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -17,6 +17,7 @@ * [Break Change] Change the configuration field of `ui_template` and `ui_menu` in Elasticsearch storage from **keyword** type to **text**. * Support Service Hierarchy auto matching. * Add `namespace` suffix for `K8S_SERVICE_NAME_RULE/ISTIO_SERVICE_NAME_RULE` and `metadata-service-mapping.yaml` as default. +* Allow using a dedicated port for ALS receiver. #### UI diff --git a/docs/en/setup/backend/backend-load-balancer.md b/docs/en/setup/backend/backend-load-balancer.md index 4773bb47dc..f496a7d92b 100644 --- a/docs/en/setup/backend/backend-load-balancer.md +++ b/docs/en/setup/backend/backend-load-balancer.md @@ -33,7 +33,7 @@ each of the OAP instance can have similar amount of gRPC connections. Before that, you need to calculate the number of connections for each OAP instance as follows: -``` +```shell NUMBER_OF_SERVICE_PODS= # Each service Pod has 2 connections to OAP @@ -75,3 +75,49 @@ spec: app: oap EOF ``` + +By this approach, we can limit the connections to port 11800 per OAP instance, but there +is another corner case when the amount of service Pods are huge. Because the limiting is +on connection level, and each service Pod has 2 connections to OAP port 11800, one for +Envoy ALS to send access log, the other one for Envoy metrics, and because the traffic +of the 2 connections can vary very much, if the number of service Pods is large enough, +an extreme case might happen that one OAP instance is serving all Envoy metrics connections +and the other OAP instance is serving all Envoy ALS connections, which in turn might be +unbalanced again, to solve this, we can split the ALS connections to a dedicated port, +and limit the connections to that port only. + +You can set the environment variable `SW_ALS_GRPC_PORT` to a port number other than `0` +when deploying skywalking, and limit connections to that port only in the `EnvoyFilter`: + +```shell +export SW_ALS_GRPC_PORT=11802 + +kubectl -n $SKYWALKING_NAMESPACE apply -f - < 0) { + grpcServer.setMaxMessageSize(config.getMaxMessageSize()); + } + if (config.getMaxConcurrentCallsPerConnection() > 0) { + grpcServer.setMaxConcurrentCallsPerConnection(config.getMaxConcurrentCallsPerConnection()); + } + if (config.getGRPCThreadPoolSize() > 0) { + grpcServer.setThreadPoolSize(config.getGRPCThreadPoolSize()); + } + grpcServer.initialize(); + + this.receiverGRPCHandlerRegister = new GRPCHandlerRegisterImpl(grpcServer); + } } @Override @@ -78,15 +116,25 @@ public class EnvoyMetricReceiverProvider extends ModuleProvider { .load(TCPOALDefine.INSTANCE); } - GRPCHandlerRegister service = getManager().find(SharingServerModule.NAME) - .provider() - .getService(GRPCHandlerRegister.class); if (config.isAcceptMetricsService()) { final MetricServiceGRPCHandler handler = new MetricServiceGRPCHandler(getManager(), config); + // Always use the sharing gRPC server to accept metrics connections. + final var service = getManager() + .find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); service.addHandler(handler); service.addHandler(new MetricServiceGRPCHandlerV3(handler)); } - final AccessLogServiceGRPCHandler handler = new AccessLogServiceGRPCHandler(getManager(), config); + + final var service = + Objects.nonNull(receiverGRPCHandlerRegister) ? + receiverGRPCHandlerRegister : + getManager() + .find(SharingServerModule.NAME) + .provider() + .getService(GRPCHandlerRegister.class); + final var handler = new AccessLogServiceGRPCHandler(getManager(), config); service.addHandler(handler); service.addHandler(new AccessLogServiceGRPCHandlerV3(handler)); service.addHandler(new SatelliteAccessLogServiceGRPCHandlerV3(handler)); @@ -94,7 +142,13 @@ public class EnvoyMetricReceiverProvider extends ModuleProvider { @Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { - + try { + if (Objects.nonNull(grpcServer) && !RunningMode.isInitMode()) { + grpcServer.start(); + } + } catch (ServerException e) { + throw new ModuleStartException(e.getMessage(), e); + } } @Override diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index f47a325e62..a0f105b37d 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -322,6 +322,15 @@ envoy-metric: # service name, users can use this config to exclude ServiceEntries that # they don't want to be used. Comma separated. istioServiceEntryIgnoredNamespaces: ${SW_ISTIO_SERVICE_ENTRY_IGNORED_NAMESPACES:""} + gRPCHost: ${SW_ALS_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_ALS_GRPC_PORT:0} + maxConcurrentCallsPerConnection: ${SW_ALS_GRPC_MAX_CONCURRENT_CALL:0} + maxMessageSize: ${SW_ALS_GRPC_MAX_MESSAGE_SIZE:0} + gRPCThreadPoolSize: ${SW_ALS_GRPC_THREAD_POOL_SIZE:0} + gRPCSslEnabled: ${SW_ALS_GRPC_SSL_ENABLED:false} + gRPCSslKeyPath: ${SW_ALS_GRPC_SSL_KEY_PATH:""} + gRPCSslCertChainPath: ${SW_ALS_GRPC_SSL_CERT_CHAIN_PATH:""} + gRPCSslTrustedCAsPath: ${SW_ALS_GRPC_SSL_TRUSTED_CAS_PATH:""} kafka-fetcher: selector: ${SW_KAFKA_FETCHER:-}