From 4b508b930dcce5345d2634c669fc4f318e1c0834 Mon Sep 17 00:00:00 2001 From: Wan Kai Date: Thu, 3 Jul 2025 15:28:26 +0800 Subject: [PATCH] Support custom GRPCClient health checker logic. (#13353) [NOTICE] Roll back score meaning in GraphQL health check API. --- dist-material/release-docs/LICENSE | 20 ++-- docs/en/changes/changes.md | 3 +- docs/en/setup/backend/backend-health-check.md | 4 +- oap-server-bom/pom.xml | 5 + .../server/core/query/type/HealthStatus.java | 2 +- .../health/HealthCheckServiceHandler.java | 24 ++--- .../src/main/proto/HealthCheckService.proto | 40 -------- .../provider/HealthCheckerHttpService.java | 2 +- .../provider/HealthCheckerProvider.java | 11 ++- .../server-library/library-client/pom.xml | 4 + .../library/client/grpc/GRPCClient.java | 93 ++++++++++++++----- .../src/main/resources/query-protocol | 2 +- oap-server/server-starter/pom.xml | 1 + pom.xml | 2 +- 14 files changed, 116 insertions(+), 97 deletions(-) delete mode 100644 oap-server/server-core/src/main/proto/HealthCheckService.proto diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 60d6a5924f..43d5b2ed7b 100644 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -226,7 +226,7 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.18.2 Apache-2.0 https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-module-kotlin/2.13.4 Apache-2.0 https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1 Apache-2.0 - https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-common-protos/2.41.0 Apache-2.0 + https://mvnrepository.com/artifact/com.google.api.grpc/proto-google-common-protos/2.48.0 Apache-2.0 https://mvnrepository.com/artifact/com.google.auto.service/auto-service-annotations/1.0.1 Apache-2.0 https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305/3.0.2 Apache-2.0 https://mvnrepository.com/artifact/com.google.code.gson/gson/2.9.0 Apache-2.0 @@ -290,16 +290,16 @@ The text of each license is the standard Apache 2.0 license. https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-scheduling/6.7.1 Apache-2.0 https://mvnrepository.com/artifact/io.fabric8/kubernetes-model-storageclass/6.7.1 Apache-2.0 https://mvnrepository.com/artifact/io.fabric8/zjsonpatch/0.3.0 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-api/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-context/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-core/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-grpclb/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-netty/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-protobuf/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-protobuf-lite/1.68.1 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-api/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-context/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-core/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-grpclb/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-netty/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-protobuf/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-protobuf-lite/1.70.0 Apache-2.0 https://mvnrepository.com/artifact/io.grpc/grpc-services/1.70.0 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.68.1 Apache-2.0 - https://mvnrepository.com/artifact/io.grpc/grpc-util/1.68.1 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-stub/1.70.0 Apache-2.0 + https://mvnrepository.com/artifact/io.grpc/grpc-util/1.70.0 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-commons/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-core/1.14.4 Apache-2.0 https://mvnrepository.com/artifact/io.micrometer/micrometer-observation/1.14.4 Apache-2.0 diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 03d0b2080f..2f8b4a3c46 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -37,7 +37,8 @@ * chore: add a warning log when connecting to ES takes too long. * Fix the query time range in the metadata API. * OAP gRPC-Client support `Health Check`. -* [Break Change] `Health Check` make response 1 represents healthy, 0 represents unhealthy. +* [Break Change] `health_check_xx` metrics make response 1 represents healthy, 0 represents unhealthy. +* Bump up grpc to 1.70.0. #### UI diff --git a/docs/en/setup/backend/backend-health-check.md b/docs/en/setup/backend/backend-health-check.md index a8201ab5a0..c717851ca1 100644 --- a/docs/en/setup/backend/backend-health-check.md +++ b/docs/en/setup/backend/backend-health-check.md @@ -36,7 +36,7 @@ If the OAP server is healthy, the response should be { "data": { "checkHealth": { - "score": 1, + "score": 0, "details": "" } } @@ -49,7 +49,7 @@ If some modules are unhealthy (e.g. storage H2 is down), then the result may loo { "data": { "checkHealth": { - "score": 0, + "score": 1, "details": "storage_h2," } } diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 5ab55640bd..050d3ad115 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -253,6 +253,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-services + ${grpc.version} + io.netty netty-tcnative-boringssl-static diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HealthStatus.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HealthStatus.java index ab073d4d58..6a65bc7402 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HealthStatus.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HealthStatus.java @@ -26,7 +26,7 @@ import lombok.ToString; @Setter @ToString public class HealthStatus { - // score == 1 means healthy, otherwise it's unhealthy. + // score == 0 means healthy and no unhealthy component or connection, otherwise it's unhealthy. private int score; private String details; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/health/HealthCheckServiceHandler.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/health/HealthCheckServiceHandler.java index 53a877692c..611da0a9d0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/health/HealthCheckServiceHandler.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/health/HealthCheckServiceHandler.java @@ -18,17 +18,15 @@ package org.apache.skywalking.oap.server.core.remote.health; -import grpc.health.v1.HealthCheckService; -import grpc.health.v1.HealthGrpc; +import io.grpc.health.v1.HealthCheckRequest; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.health.v1.HealthGrpc; import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class HealthCheckServiceHandler extends HealthGrpc.HealthImplBase implements GRPCHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(HealthCheckServiceHandler.class); - /** * By my test, consul didn't send the service. * @@ -36,15 +34,13 @@ public class HealthCheckServiceHandler extends HealthGrpc.HealthImplBase impleme * @param responseObserver status */ @Override - public void check(HealthCheckService.HealthCheckRequest request, - StreamObserver responseObserver) { - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Received the gRPC server health check with the service name of {}", request.getService()); + public void check(HealthCheckRequest request, StreamObserver responseObserver) { + if (log.isDebugEnabled()) { + log.debug("Received the gRPC server health check with the service name of {}", request.getService()); } - HealthCheckService.HealthCheckResponse.Builder response = HealthCheckService.HealthCheckResponse.newBuilder(); - response.setStatus(HealthCheckService.HealthCheckResponse.ServingStatus.SERVING); + HealthCheckResponse.Builder response = HealthCheckResponse.newBuilder(); + response.setStatus(HealthCheckResponse.ServingStatus.SERVING); responseObserver.onNext(response.build()); responseObserver.onCompleted(); diff --git a/oap-server/server-core/src/main/proto/HealthCheckService.proto b/oap-server/server-core/src/main/proto/HealthCheckService.proto deleted file mode 100644 index ecae37ca89..0000000000 --- a/oap-server/server-core/src/main/proto/HealthCheckService.proto +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -//This is a health check proto, provided by gRPC team. Please don't change it. -//https://github.com/grpc/grpc/blob/master/doc/health-checking.md -syntax = "proto3"; - -package grpc.health.v1; - -message HealthCheckRequest { - string service = 1; -} - -message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - } - ServingStatus status = 1; -} - -service Health { - rpc Check (HealthCheckRequest) returns (HealthCheckResponse); -} \ No newline at end of file diff --git a/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerHttpService.java b/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerHttpService.java index 7b520fcaa5..356d76a79e 100644 --- a/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerHttpService.java +++ b/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerHttpService.java @@ -37,7 +37,7 @@ public class HealthCheckerHttpService { final var status = healthQueryService.checkHealth(); log.info("Health status: {}", status); - if (status.getScore() == 1) { + if (status.getScore() == 0) { return HttpResponse.of(HttpStatus.OK); } return HttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE); diff --git a/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerProvider.java b/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerProvider.java index 2051b5b0df..6bec3853ad 100644 --- a/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerProvider.java +++ b/oap-server/server-health-checker/src/main/java/org/apache/skywalking/oap/server/health/checker/provider/HealthCheckerProvider.java @@ -25,7 +25,6 @@ import java.util.Arrays; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.CoreModule; @@ -100,18 +99,22 @@ public class HealthCheckerProvider extends ModuleProvider { @Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { ses.scheduleAtFixedRate(() -> { StringBuilder unhealthyModules = new StringBuilder(); - AtomicBoolean hasUnhealthyModule = new AtomicBoolean(false); + AtomicDouble unhealthyModule = new AtomicDouble(0); Stream.ofAll(collector.collect()) .flatMap(metricFamily -> metricFamily.samples) .filter(sample -> metricsCreator.isHealthCheckerMetrics(sample.name)) .forEach(sample -> { if (sample.value < 1) { unhealthyModules.append(metricsCreator.extractModuleName(sample.name)).append(","); - hasUnhealthyModule.set(true); + unhealthyModule.updateAndGet(v -> v + 1); } }); - score.set(hasUnhealthyModule.get() ? 0 : 1); + if (unhealthyModule.get() > 0) { + score.set(unhealthyModule.get()); + } else { + score.set(0); + } details.set(unhealthyModules.toString()); }, 2, config.getCheckIntervalSeconds(), TimeUnit.SECONDS); diff --git a/oap-server/server-library/library-client/pom.xml b/oap-server/server-library/library-client/pom.xml index 94e9dd0742..12952e30c9 100755 --- a/oap-server/server-library/library-client/pom.xml +++ b/oap-server/server-library/library-client/pom.xml @@ -53,6 +53,10 @@ io.grpc grpc-netty + + io.grpc + grpc-services + io.netty netty-codec-http2 diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/grpc/GRPCClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/grpc/GRPCClient.java index ce54de19a5..1ab0ee3e33 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/grpc/GRPCClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/grpc/GRPCClient.java @@ -18,26 +18,27 @@ package org.apache.skywalking.oap.server.library.client.grpc; -import io.grpc.ConnectivityState; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.health.v1.HealthCheckRequest; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.health.v1.HealthGrpc; import io.grpc.netty.NettyChannelBuilder; import io.netty.handler.ssl.SslContext; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.library.client.Client; import org.apache.skywalking.oap.server.library.client.healthcheck.DelegatedHealthChecker; import org.apache.skywalking.oap.server.library.client.healthcheck.HealthCheckable; import org.apache.skywalking.oap.server.library.util.HealthChecker; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class GRPCClient implements Client, HealthCheckable { - - private static final Logger LOGGER = LoggerFactory.getLogger(GRPCClient.class); - @Getter private final String host; @@ -54,6 +55,35 @@ public class GRPCClient implements Client, HealthCheckable { private boolean enableHealthCheck = false; + private long initialDelay = 5; // Initial delay for health check in seconds + + private long period = 20; // Period for health check in seconds + + // The default health check runnable that checks the health of the gRPC channel. + private Runnable healthCheckRunnable = () -> { + if (getChannel() != null && !getChannel().isShutdown()) { + HealthGrpc.HealthBlockingStub healthStub = HealthGrpc.newBlockingStub(getChannel()); + HealthCheckRequest request = HealthCheckRequest.newBuilder().setService("").build(); + try { + HealthCheckResponse response = healthStub.check(request); + handleStateChange(response); + } catch (StatusRuntimeException s) { + if (s.getStatus().getCode() == Status.Code.UNIMPLEMENTED) { + log.warn("Health check is not implemented on the remote gRPC server, regard as healthy. Host: {}, Port: {}", getHost(), getPort()); + healthChecker.health(); + } else { + log.warn("Health check failed for gRPC channel. Host: {}, Port: {}", getHost(), getPort(), s); + healthChecker.unHealth(s); + } + } catch (Throwable t) { + log.warn("Health check failed for gRPC channel. Host: {}, Port: {}", getHost(), getPort(), t); + healthChecker.unHealth(t); + } + } else { + healthChecker.unHealth("gRPC channel is not available or shutting down. Host: " + getHost() + ", Port: " + getPort()); + } + }; + public GRPCClient(String host, int port) { this.host = host; this.port = port; @@ -81,7 +111,7 @@ public class GRPCClient implements Client, HealthCheckable { try { channel.shutdownNow(); } catch (Throwable t) { - LOGGER.error(t.getMessage(), t); + log.error(t.getMessage(), t); } finally { if (healthCheckExecutor != null) { healthCheckExecutor.shutdownNow(); @@ -114,32 +144,51 @@ public class GRPCClient implements Client, HealthCheckable { this.enableHealthCheck = true; } + /** + * Override the default health check runnable with a custom one. + * Must override before calling connect() + * This can be used to provide a different health check logic. + * + * @param healthCheckRunnable The custom health check runnable. + * @param initialDelay Initial delay before the first health check. + * @param period Period between subsequent health checks. + */ + public void overrideCheckerRunnable(final Runnable healthCheckRunnable, final long initialDelay, final long period) { + this.healthCheckRunnable = healthCheckRunnable; + if (initialDelay < 0) { + throw new IllegalArgumentException("initialDelay must be non-negative. Provided value: " + initialDelay); + } + if (period < 0) { + throw new IllegalArgumentException("period must be non-negative. Provided value: " + period); + } + this.initialDelay = initialDelay; + this.period = period; + } + private void checkHealth() { if (healthCheckExecutor == null) { healthCheckExecutor = Executors.newSingleThreadScheduledExecutor(); - healthCheckExecutor.scheduleAtFixedRate( - () -> { - ConnectivityState currentState = channel.getState(true); // true means try to connect - handleStateChange(currentState); - }, 5, 10, TimeUnit.SECONDS + healthCheckExecutor.scheduleAtFixedRate(healthCheckRunnable, initialDelay, period, TimeUnit.SECONDS ); } } - private void handleStateChange(ConnectivityState newState) { - switch (newState) { - case READY: - case IDLE: + private void handleStateChange(HealthCheckResponse response) { + switch (response.getStatus()) { + case SERVING: this.healthChecker.health(); break; - case CONNECTING: - this.healthChecker.unHealth("gRPC connecting, waiting for ready. Host: " + host + ", Port: " + port); + case NOT_SERVING: + this.healthChecker.unHealth("Remote gRPC Server NOT_SERVING. Host: " + host + ", Port: " + port); break; - case TRANSIENT_FAILURE: - this.healthChecker.unHealth("gRPC connection failed, will retry. Host: " + host + ", Port: " + port); + case SERVICE_UNKNOWN: + this.healthChecker.unHealth("Remote gRPC Server SERVICE_UNKNOWN. Host: " + host + ", Port: " + port); break; - case SHUTDOWN: - this.healthChecker.unHealth("gRPC channel is shutting down. Host: " + host + ", Port: " + port); + case UNKNOWN: + this.healthChecker.unHealth("Remote gRPC Server UNKNOWN. Host: " + host + ", Port: " + port); + break; + case UNRECOGNIZED: + this.healthChecker.unHealth("Remote gRPC Server UNRECOGNIZED. Host: " + host + ", Port: " + port); break; } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 021c0ad768..e6d2b99597 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 021c0ad768f8f6f64dceead9d79a3dd7e9ad8dd9 +Subproject commit e6d2b99597a6489ab58f8a2ff32f212c8ef9b388 diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index 2b06383038..d1971572aa 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -317,6 +317,7 @@ hierarchy-definition.yml bydb.dependencies.properties bydb.yml + bydb-topn.yml oal/ fetcher-prom-rules/ envoy-metrics-rules/ diff --git a/pom.xml b/pom.xml index 07648f1858..5cffda4179 100755 --- a/pom.xml +++ b/pom.xml @@ -165,7 +165,7 @@ 1.14.9 - 1.68.1 + 1.70.0 4.1.118.Final 2.0.69.Final 2.9.0