From 90d525b164d471f92355557273f4d266ccfbdc1d Mon Sep 17 00:00:00 2001 From: Alvin <329772643@qq.com> Date: Sun, 11 Aug 2019 12:30:17 +0800 Subject: [PATCH] Improve agent grpc auto reconnect (#3240) * improve auto connect * add config * use int instread of atomic * update format * update doc * update var name and doc * fix --- .../org/apache/skywalking/apm/agent/core/conf/Config.java | 7 +++++++ .../skywalking/apm/agent/core/remote/GRPCChannel.java | 6 +++++- .../apm/agent/core/remote/GRPCChannelManager.java | 8 +++++--- docs/en/setup/service-agent/java-agent/README.md | 1 + 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java index 572839110..145f1d26f 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java @@ -100,6 +100,13 @@ public class Config { * after receiving reset command */ public static int COOL_DOWN_THRESHOLD = 10; + + /** + * Force reconnection period of grpc, based on grpc_channel_check_interval. + * If count of check grpc channel status more than this number. + * The channel check will call channel.getState(true) to requestConnection. + */ + public static long FORCE_RECONNECTION_PERIOD = 1; } public static class Collector { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannel.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannel.java index 267d3ccd3..8e3afecff 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannel.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannel.java @@ -75,7 +75,11 @@ public class GRPCChannel { } public boolean isConnected() { - return originChannel.getState(false) == ConnectivityState.READY; + return isConnected(false); + } + + public boolean isConnected(boolean requestConnection) { + return originChannel.getState(requestConnection) == ConnectivityState.READY; } public static class Builder { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannelManager.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannelManager.java index 952a11515..6aed148fa 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannelManager.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/GRPCChannelManager.java @@ -51,6 +51,7 @@ public class GRPCChannelManager implements BootService, Runnable { private List listeners = Collections.synchronizedList(new LinkedList()); private volatile List grpcServers; private volatile int selectedIdx = -1; + private volatile int reconnectCount = 0; @Override public void prepare() throws Throwable { @@ -115,13 +116,14 @@ public class GRPCChannelManager implements BootService, Runnable { .addChannelDecorator(new AgentIDDecorator()) .addChannelDecorator(new AuthenticationDecorator()) .build(); - notify(GRPCChannelStatus.CONNECTED); + reconnectCount = 0; reconnect = false; - } else if (managedChannel.isConnected()) { + } else if (managedChannel.isConnected(++reconnectCount > Config.Agent.FORCE_RECONNECTION_PERIOD)) { // Reconnect to the same server is automatically done by GRPC, // therefore we are responsible to check the connectivity and // set the state and notify listeners + reconnectCount = 0; notify(GRPCChannelStatus.CONNECTED); reconnect = false; } @@ -168,7 +170,7 @@ public class GRPCChannelManager implements BootService, Runnable { private boolean isNetworkError(Throwable throwable) { if (throwable instanceof StatusRuntimeException) { - StatusRuntimeException statusRuntimeException = (StatusRuntimeException)throwable; + StatusRuntimeException statusRuntimeException = (StatusRuntimeException) throwable; return statusEquals(statusRuntimeException.getStatus(), Status.UNAVAILABLE, Status.PERMISSION_DENIED, diff --git a/docs/en/setup/service-agent/java-agent/README.md b/docs/en/setup/service-agent/java-agent/README.md index caaf775df..d390e87b0 100755 --- a/docs/en/setup/service-agent/java-agent/README.md +++ b/docs/en/setup/service-agent/java-agent/README.md @@ -76,6 +76,7 @@ property key | Description | Default | `agent.cause_exception_depth`|How depth the agent goes, when log all cause exceptions.|`5`| `agent.active_v1_header `|Deactivate V1 header in default.|`false`| `agent.cool_down_threshold `|How long should the agent wait (in minute) before re-registering to the OAP server after receiving reset command.|`10`| +`agent.force_reconnection_period `|Force reconnection period of grpc, based on grpc_channel_check_interval.|`1`| `collector.grpc_channel_check_interval`|grpc channel status check interval.|`30`| `collector.app_and_service_register_check_interval`|application and service registry check interval.|`3`| `collector.backend_service`|Collector SkyWalking trace receiver service addresses.|`127.0.0.1:11800`|