From 91ffae2526920e9f7c62a14d210f250608bf030d Mon Sep 17 00:00:00 2001 From: caoyixiong <32847163+IanCao@users.noreply.github.com> Date: Wed, 15 May 2019 00:29:23 +0800 Subject: [PATCH] support Nacos cluster (#2640) * Fix #2546 * Change the stackdepth type * Fix bug * Fix CI failed * fix #2599 * fix ci * add license * add ActiveTag error methods * fix * fix * nacos * nacos * nacos * nacos * fix ci * support nacos cluster * fix yml * fix license * fix ci * remove optional * update license * update license and doc * fix --- apm-dist/release-docs/LICENSE | 2 + apm-dist/release-docs/NOTICE | 38 +++++ .../licenses/LICENSE-fastjson.txt | 13 ++ docker/config/application.yml | 3 + docs/en/setup/backend/backend-cluster.md | 17 ++- .../cluster-nacos-plugin/pom.xml | 71 ++++++++++ .../nacos/ClusterModuleNacosConfig.java | 31 ++++ .../nacos/ClusterModuleNacosProvider.java | 84 +++++++++++ .../plugin/nacos/NacosCoordinator.java | 79 +++++++++++ ...g.oap.server.library.module.ModuleProvider | 19 +++ .../nacos/ClusterModuleNacosProviderTest.java | 106 ++++++++++++++ .../plugin/nacos/NacosCoordinatorTest.java | 134 ++++++++++++++++++ oap-server/server-cluster-plugin/pom.xml | 1 + .../core/cluster/ServiceQueryException.java | 29 ++++ oap-server/server-starter/pom.xml | 5 + .../src/main/assembly/application.yml | 3 + .../src/main/resources/application.yml | 3 + 17 files changed, 636 insertions(+), 2 deletions(-) create mode 100644 apm-dist/release-docs/licenses/LICENSE-fastjson.txt create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/pom.xml create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosConfig.java create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProvider.java create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinator.java create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProviderTest.java create mode 100644 oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinatorTest.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cluster/ServiceQueryException.java diff --git a/apm-dist/release-docs/LICENSE b/apm-dist/release-docs/LICENSE index 8b134b293..f8c1911e6 100644 --- a/apm-dist/release-docs/LICENSE +++ b/apm-dist/release-docs/LICENSE @@ -217,6 +217,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. Apache: groovy 2.4.5-indy: https://github.com/apache/groovy, Apache 2.0 Apache: httpcomponents 4.x.x: http://hc.apache.org/index.html, Apache 2.0 + Apache: fastjson 1.2.47: https://github.com/alibaba/fastjson, Apache 2.0 ======================================================================== Apache 2.0 licenses @@ -307,6 +308,7 @@ The text of each license is the standard Apache 2.0 license. consul-client 1.2.6: https://github.com/rickfast/consul-client, Apache 2.0 okhttp 3.9.0: https://github.com/square/okhttp, Apache 2.0 prometheus client_java 0.6.0: https://github.com/prometheus/client_java, Apache 2.0 + nacos 1.0.0: https://github.com/alibaba/nacos, Apache 2.0 proto files from istio/istio: https://github.com/istio/istio Apache 2.0 proto files from istio/api: https://github.com/istio/api Apache 2.0 proto files from envoyproxy/data-plane-api: https://github.com/envoyproxy/data-plane-api Apache 2.0 diff --git a/apm-dist/release-docs/NOTICE b/apm-dist/release-docs/NOTICE index a904f3729..60a4f4552 100644 --- a/apm-dist/release-docs/NOTICE +++ b/apm-dist/release-docs/NOTICE @@ -831,3 +831,41 @@ from and not be held liable to the user for any such damages as noted above as far as the program is concerned. ------ + +=========================================================================== +nacos-1.0.0 Notice +=========================================================================== +Nacos +Copyright 2018-2019 The Apache Software Foundation + +This product includes software developed at +The Alibaba MiddleWare Group. + +------ +This product has a bundle Spring Boot: + The Spring Boot Project + ================= + +Please visit the Spring Boot web site for more information: + + * https://spring.io/projects/spring-boot + +Copyright 2014 The Spring Boot Project + +The Spring Boot Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------ \ No newline at end of file diff --git a/apm-dist/release-docs/licenses/LICENSE-fastjson.txt b/apm-dist/release-docs/licenses/LICENSE-fastjson.txt new file mode 100644 index 000000000..b8500e573 --- /dev/null +++ b/apm-dist/release-docs/licenses/LICENSE-fastjson.txt @@ -0,0 +1,13 @@ +Copyright 1999-2019 Alibaba Group Holding Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/docker/config/application.yml b/docker/config/application.yml index 7865829d1..afbd3e7ad 100644 --- a/docker/config/application.yml +++ b/docker/config/application.yml @@ -33,6 +33,9 @@ cluster: # serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} # Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 # hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +# nacos: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} core: default: # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate diff --git a/docs/en/setup/backend/backend-cluster.md b/docs/en/setup/backend/backend-cluster.md index a6206e550..0ec9116ec 100644 --- a/docs/en/setup/backend/backend-cluster.md +++ b/docs/en/setup/backend/backend-cluster.md @@ -9,7 +9,7 @@ with each other. - [Kubernetes](#kubernetes). When backend cluster are deployed inside kubernetes, you could choose this by using k8s native APIs to manage cluster. - [Consul](#consul). Use Consul as backend cluster management implementor, to coordinate backend instances. - +- [Nacos](#nacos). Use Nacos to coordinate backend instances. ## Zookeeper coordinator Zookeeper is a very common and wide used cluster coordinator. Set the **cluster** module's implementor @@ -77,4 +77,17 @@ Same as Zookeeper coordinator, in some cases, oap default gRPC host and port in core are not suitable for internal communication among the oap nodes. The following setting are provided to set the hot and port manually, based on your own LAN env. - internalComHost, the host registered and other oap node use this to communicate with current node. -- internalComPort, the port registered and other oap node use this to communicate with current node. \ No newline at end of file +- internalComPort, the port registered and other oap node use this to communicate with current node. + + +## Nacos +Set the **cluster** module's implementor to **nacos** in +the yml to active. + +```yaml +cluster: + nacos: + serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} + # Nacos cluster nodes, example: 10.0.0.1:8848,10.0.0.2:8848,10.0.0.3:8848 + hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} +``` \ No newline at end of file diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/pom.xml b/oap-server/server-cluster-plugin/cluster-nacos-plugin/pom.xml new file mode 100644 index 000000000..15889ad14 --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/pom.xml @@ -0,0 +1,71 @@ + + + + + server-cluster-plugin + org.apache.skywalking + 6.2.0-SNAPSHOT + + 4.0.0 + + cluster-nacos-plugin + + 1.0.0 + + + + org.apache.skywalking + server-core + ${project.version} + + + com.alibaba.nacos + nacos-client + ${nacos.version} + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + net.jcip + jcip-annotations + + + + + \ No newline at end of file diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosConfig.java b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosConfig.java new file mode 100644 index 000000000..2b5590d85 --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosConfig.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.cluster.plugin.nacos; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +/** + * @author caoyixiong + */ +public class ClusterModuleNacosConfig extends ModuleConfig { + @Setter @Getter private String serviceName; + @Setter @Getter private String hostPort; +} diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProvider.java b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProvider.java new file mode 100644 index 000000000..34b3bc37f --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProvider.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.cluster.plugin.nacos; + +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingFactory; +import com.alibaba.nacos.api.naming.NamingService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.cluster.ClusterModule; +import org.apache.skywalking.oap.server.core.cluster.ClusterNodesQuery; +import org.apache.skywalking.oap.server.core.cluster.ClusterRegister; +import org.apache.skywalking.oap.server.library.module.*; + +/** + * @author caoyixiong + */ +public class ClusterModuleNacosProvider extends ModuleProvider { + + private final ClusterModuleNacosConfig config; + private NamingService namingService; + + public ClusterModuleNacosProvider() { + super(); + this.config = new ClusterModuleNacosConfig(); + } + + @Override + public String name() { + return "nacos"; + } + + @Override + public Class module() { + return ClusterModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return config; + } + + @Override + public void prepare() throws ServiceNotProvidedException, ModuleStartException { + try { + namingService = NamingFactory.createNamingService(config.getHostPort()); + } catch (NacosException e) { + throw new ModuleStartException(e.getMessage(), e); + } + NacosCoordinator coordinator = new NacosCoordinator(namingService, config); + this.registerServiceImplementation(ClusterRegister.class, coordinator); + this.registerServiceImplementation(ClusterNodesQuery.class, coordinator); + } + + @Override + public void start() throws ServiceNotProvidedException { + + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException { + + } + + @Override + public String[] requiredModules() { + return new String[]{CoreModule.NAME}; + } +} diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinator.java b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinator.java new file mode 100644 index 000000000..10d56f4ab --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinator.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.cluster.plugin.nacos; + +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.pojo.Instance; +import org.apache.skywalking.oap.server.core.cluster.*; +import org.apache.skywalking.oap.server.core.remote.client.Address; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * @author caoyixiong + */ +public class NacosCoordinator implements ClusterRegister, ClusterNodesQuery { + + private final NamingService namingService; + private final ClusterModuleNacosConfig config; + private volatile Address selfAddress; + + public NacosCoordinator(NamingService namingService, ClusterModuleNacosConfig config) { + this.namingService = namingService; + this.config = config; + } + + @Override + public List queryRemoteNodes() { + List result = new ArrayList<>(); + try { + List instances = namingService.selectInstances(config.getServiceName(), true); + if (CollectionUtils.isNotEmpty(instances)) { + instances.forEach(instance -> { + if (Objects.nonNull(selfAddress)) { + if (selfAddress.getHost().equals(instance.getIp()) && selfAddress.getPort() == instance.getPort()) { + result.add(new RemoteInstance(new Address(instance.getIp(), instance.getPort(), true))); + } else { + result.add(new RemoteInstance(new Address(instance.getIp(), instance.getPort(), false))); + } + } + }); + } + } catch (NacosException e) { + throw new ServiceQueryException(e.getErrMsg()); + } + return result; + } + + @Override + public void registerRemote(RemoteInstance remoteInstance) throws ServiceRegisterException { + String host = remoteInstance.getAddress().getHost(); + int port = remoteInstance.getAddress().getPort(); + try { + namingService.registerInstance(config.getServiceName(), host, port); + } catch (Exception e) { + throw new ServiceRegisterException(e.getMessage()); + } + this.selfAddress = remoteInstance.getAddress(); + } +} diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 000000000..1a1c0e89f --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.cluster.plugin.nacos.ClusterModuleNacosProvider \ No newline at end of file diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProviderTest.java b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProviderTest.java new file mode 100644 index 000000000..b60777b18 --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/ClusterModuleNacosProviderTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.cluster.plugin.nacos; + +import com.alibaba.nacos.api.naming.NamingFactory; +import com.alibaba.nacos.api.naming.NamingService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.cluster.ClusterModule; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.powermock.reflect.Whitebox; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * @author caoyixiong + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(NamingFactory.class) +@PowerMockIgnore("javax.management.*") +public class ClusterModuleNacosProviderTest { + + private static final String SERVICE_NAME = "test-service_name"; + + private ClusterModuleNacosProvider provider = new ClusterModuleNacosProvider(); + + @Test + public void name() { + assertEquals("nacos", provider.name()); + } + + @Test + public void module() { + assertEquals(ClusterModule.class, provider.module()); + } + + + @Test + public void createConfigBeanIfAbsent() { + ModuleConfig moduleConfig = provider.createConfigBeanIfAbsent(); + assertTrue(moduleConfig instanceof ClusterModuleNacosConfig); + } + + @Test(expected = ModuleStartException.class) + public void prepareWithNonHost() throws Exception { + provider.prepare(); + } + + @Test + public void prepare() throws Exception { + PowerMockito.mockStatic(NamingFactory.class); + ClusterModuleNacosConfig nacosConfig = new ClusterModuleNacosConfig(); + nacosConfig.setHostPort("10.0.0.1:1000,10.0.0.2:1001"); + nacosConfig.setServiceName(SERVICE_NAME); + Whitebox.setInternalState(provider, "config", nacosConfig); + NamingService namingService = mock(NamingService.class); + PowerMockito.when(NamingFactory.createNamingService("10.0.0.1:1000,10.0.0.2:1001")).thenReturn(namingService); + provider.prepare(); + ArgumentCaptor addressCaptor = ArgumentCaptor.forClass(String.class); + PowerMockito.verifyStatic(); + NamingFactory.createNamingService(addressCaptor.capture()); + String data = addressCaptor.getValue(); + assertEquals("10.0.0.1:1000,10.0.0.2:1001", data); + } + + @Test + public void start() { + provider.start(); + } + + @Test + public void notifyAfterCompleted() { + provider.notifyAfterCompleted(); + } + + @Test + public void requiredModules() { + String[] modules = provider.requiredModules(); + assertArrayEquals(new String[]{CoreModule.NAME}, modules); + } +} diff --git a/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinatorTest.java b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinatorTest.java new file mode 100644 index 000000000..b362d5a54 --- /dev/null +++ b/oap-server/server-cluster-plugin/cluster-nacos-plugin/src/test/java/org/apache/skywalking/oap/server/cluster/plugin/nacos/NacosCoordinatorTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.cluster.plugin.nacos; + +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.pojo.Instance; +import org.apache.skywalking.oap.server.core.cluster.RemoteInstance; +import org.apache.skywalking.oap.server.core.remote.client.Address; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author caoyixiong + */ +public class NacosCoordinatorTest { + private NamingService namingService = mock(NamingService.class); + private ClusterModuleNacosConfig nacosConfig = new ClusterModuleNacosConfig(); + private NacosCoordinator coordinator; + + private Address remoteAddress = new Address("10.0.0.1", 1000, false); + private Address selfRemoteAddress = new Address("10.0.0.2", 1001, true); + + private static final String SERVICE_NAME = "test-service"; + + @Before + public void setUp() throws NacosException { + nacosConfig.setServiceName(SERVICE_NAME); + coordinator = new NacosCoordinator(namingService, nacosConfig); + } + + @Test + @SuppressWarnings("unchecked") + public void queryRemoteNodesWithNonOrEmpty() throws NacosException { + when(namingService.selectInstances(anyString(), anyBoolean())).thenReturn(null, Collections.emptyList()); + assertEquals(0, coordinator.queryRemoteNodes().size()); + } + + @Test + public void queryRemoteNodes() throws NacosException { + registerSelfRemote(); + List instances = mockInstance(); + when(namingService.selectInstances(anyString(), anyBoolean())).thenReturn(instances); + List remoteInstances = coordinator.queryRemoteNodes(); + assertEquals(2, remoteInstances.size()); + + RemoteInstance selfInstance = remoteInstances.get(0); + validate(selfRemoteAddress, selfInstance); + + RemoteInstance notSelfInstance = remoteInstances.get(1); + validate(remoteAddress, notSelfInstance); + } + + @Test + public void queryRemoteNodesWithNullSelf() throws NacosException { + List instances = mockInstance(); + when(namingService.selectInstances(anyString(), anyBoolean())).thenReturn(instances); + List remoteInstances = coordinator.queryRemoteNodes(); + assertTrue(remoteInstances.isEmpty()); + } + + @Test + public void registerRemote() throws NacosException { + registerRemote(remoteAddress); + } + + @Test + public void registerSelfRemote() throws NacosException { + registerRemote(selfRemoteAddress); + } + + private void validate(Address originArress, RemoteInstance instance) { + Address instanceAddress = instance.getAddress(); + assertEquals(originArress.getHost(), instanceAddress.getHost()); + assertEquals(originArress.getPort(), instanceAddress.getPort()); + } + + private void registerRemote(Address address) throws NacosException { + coordinator.registerRemote(new RemoteInstance(address)); + + ArgumentCaptor serviceNameArgumentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor hostArgumentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor portArgumentCaptor = ArgumentCaptor.forClass(Integer.class); + verify(namingService).registerInstance(serviceNameArgumentCaptor.capture(), hostArgumentCaptor.capture(), portArgumentCaptor.capture()); + + assertEquals(SERVICE_NAME, serviceNameArgumentCaptor.getValue()); + assertEquals(address.getHost(), hostArgumentCaptor.getValue()); + assertEquals(Long.valueOf(address.getPort()), Long.valueOf(portArgumentCaptor.getValue())); + } + + private List mockInstance() { + Instance remoteInstance = new Instance(); + Instance selfInstance = new Instance(); + selfInstance.setIp(selfRemoteAddress.getHost()); + selfInstance.setPort(selfRemoteAddress.getPort()); + + remoteInstance.setIp(remoteAddress.getHost()); + remoteInstance.setPort(remoteAddress.getPort()); + + List instances = new ArrayList<>(); + instances.add(selfInstance); + instances.add(remoteInstance); + return instances; + } +} diff --git a/oap-server/server-cluster-plugin/pom.xml b/oap-server/server-cluster-plugin/pom.xml index 153015a17..2cd0a94b9 100644 --- a/oap-server/server-cluster-plugin/pom.xml +++ b/oap-server/server-cluster-plugin/pom.xml @@ -32,6 +32,7 @@ cluster-standalone-plugin cluster-kubernetes-plugin cluster-consul-plugin + cluster-nacos-plugin diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cluster/ServiceQueryException.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cluster/ServiceQueryException.java new file mode 100644 index 000000000..495b8c3b9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cluster/ServiceQueryException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.cluster; + +/** + * @author caoyixiong + */ +public class ServiceQueryException extends RuntimeException { + + public ServiceQueryException(String message) { + super(message); + } +} \ No newline at end of file diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index e7d91a53c..44653f9ef 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -70,6 +70,11 @@ cluster-consul-plugin ${project.version} + + org.apache.skywalking + cluster-nacos-plugin + ${project.version} + diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml index cd71767a5..05edfc04f 100644 --- a/oap-server/server-starter/src/main/assembly/application.yml +++ b/oap-server/server-starter/src/main/assembly/application.yml @@ -33,6 +33,9 @@ cluster: # serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} # Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 # hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +# nacos: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} core: default: # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 08ccd0635..ced2d4373 100644 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -33,6 +33,9 @@ cluster: # serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} # Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 # hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +# nacos: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} core: default: # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate