[Feature] Service instance dependency (#3978)

* dispatcher

* getServiceInstanceTopology implementation

* CacheUpdateTimer

* fix compilation failure

* fix compilation failure

* fix esDao

* Update dao list

* dispatcher

* getServiceInstanceTopology implementation

* CacheUpdateTimer

* fix compilation failure

* fix compilation failure

* fix esDao

* Update dao list

* test e2e

* add official_analysis.oal & update getServiceInstanceTopology

* test instance topo for single service

* fix gql

* fix verifyServiceInstances

* verifyServiceInstanceTopo

* fix verifyServiceInstanceTopo

* fix ServiceInstanceInventory#name

* fix ServiceInstanceInventory#name

* e2e test

* fix action

* fix provider port

* fix Instance Node Type

* re run

* fix expected-data

* add e2e mysql

* set @Test timeout

* fix ci

* increase timeout

* re check

* test ttl es7

* test ttl

* fix action

* merge group

* test ttl es7

* replace official_analysis.oal

* replace official_analysis.oal

* comments

* add TODO

* add TODO

* fix filed name

.github/workflows/e2e.yaml

@@ -71,10 +71,12 @@ jobs:
           ./mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install
           ./mvnw -f test/e2e/pom.xml -pl e2e-base clean install
       - name: Cluster Tests (ES6/ZK/JDK8)
-        run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-cluster/test-runner
+        run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner
+      - name: Cluster With Gateway Tests (ES6/ZK/JDK8)
+        run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner
+      - name: Cluster Tests (ES7/ZK/JDK8)
+        run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner
       - name: TTL ES Tests(JDK8)
         run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es
       - name: TTL ES7 Tests(JDK8)
-        run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.0.0 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es
-      - name: Cluster Tests (ES7/ZK/JDK8)
-        run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.0.0 && bash -x test/e2e/run.sh e2e-cluster/test-runner
+        run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es

docs/en/guides/storage-extention.md

@@ -28,6 +28,11 @@ Here is the list of all DAO interfaces in storage
 1. IAggregationQueryDAO
 1. IAlarmQueryDAO
 1. IHistoryDeleteDAO
+1. IMetricsDAO
+1. IRecordDAO
+1. IRegisterDAO
+1. ILogQueryDAO
+1. ITopNRecordsQueryDAO
 
 ## Register all service implementations
 In `public void prepare()`, use `this#registerServiceImplementation` method to do register binding your implementation with the above interfaces.

oap-server/server-bootstrap/src/main/resources/official_analysis.oal

@@ -53,6 +53,24 @@ service_relation_server_p75 = from(ServiceRelation.latency).filter(detectPoint =
 service_relation_client_p50 = from(ServiceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p50(10);
 service_relation_server_p50 = from(ServiceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p50(10);
 
+// Service Instance relation scope metrics for topology
+service_instance_relation_client_cpm = from(ServiceInstanceRelation.*).filter(detectPoint == DetectPoint.CLIENT).cpm();
+service_instance_relation_server_cpm = from(ServiceInstanceRelation.*).filter(detectPoint == DetectPoint.SERVER).cpm();
+service_instance_relation_client_call_sla = from(ServiceInstanceRelation.*).filter(detectPoint == DetectPoint.CLIENT).percent(status == true);
+service_instance_relation_server_call_sla = from(ServiceInstanceRelation.*).filter(detectPoint == DetectPoint.SERVER).percent(status == true);
+service_instance_relation_client_resp_time = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).longAvg();
+service_instance_relation_server_resp_time = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).longAvg();
+service_instance_relation_client_p99 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p99(10);
+service_instance_relation_server_p99 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p99(10);
+service_instance_relation_client_p95 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p95(10);
+service_instance_relation_server_p95 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p95(10);
+service_instance_relation_client_p90 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p90(10);
+service_instance_relation_server_p90 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p90(10);
+service_instance_relation_client_p75 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p75(10);
+service_instance_relation_server_p75 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p75(10);
+service_instance_relation_client_p50 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.CLIENT).p50(10);
+service_instance_relation_server_p50 = from(ServiceInstanceRelation.latency).filter(detectPoint == DetectPoint.SERVER).p50(10);
+
 // Service Instance Scope metrics
 service_instance_sla = from(ServiceInstance.*).percent(status == true);
 service_instance_resp_time= from(ServiceInstance.latency).longAvg();

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceCallRelationDispatcher.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.instance;
+
+import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
+import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
+import org.apache.skywalking.oap.server.core.source.ServiceInstanceRelation;
+
+/**
+ * @author zhangwei
+ */
+public class ServiceInstanceCallRelationDispatcher implements SourceDispatcher<ServiceInstanceRelation> {
+
+    @Override
+    public void dispatch(ServiceInstanceRelation source) {
+        switch (source.getDetectPoint()) {
+            case SERVER:
+                serverSide(source);
+                break;
+            case CLIENT:
+                clientSide(source);
+                break;
+        }
+    }
+
+    private void serverSide(ServiceInstanceRelation source) {
+        ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics();
+        metrics.setTimeBucket(source.getTimeBucket());
+        metrics.setSourceServiceId(source.getSourceServiceId());
+        metrics.setSourceServiceInstanceId(source.getSourceServiceInstanceId());
+        metrics.setDestServiceId(source.getDestServiceId());
+        metrics.setDestServiceInstanceId(source.getDestServiceInstanceId());
+        metrics.setComponentId(source.getComponentId());
+        metrics.buildEntityId();
+        MetricsStreamProcessor.getInstance().in(metrics);
+    }
+
+    private void clientSide(ServiceInstanceRelation source) {
+        ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics();
+        metrics.setTimeBucket(source.getTimeBucket());
+        metrics.setSourceServiceId(source.getSourceServiceId());
+        metrics.setSourceServiceInstanceId(source.getSourceServiceInstanceId());
+        metrics.setDestServiceId(source.getDestServiceId());
+        metrics.setDestServiceInstanceId(source.getDestServiceInstanceId());
+        metrics.setComponentId(source.getComponentId());
+        metrics.buildEntityId();
+        MetricsStreamProcessor.getInstance().in(metrics);
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationClientSideMetrics.java

@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.instance;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.analysis.Stream;
+import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
+import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.StorageBuilder;
+import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.IDColumn;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author zhangwei
+ */
+@Stream(name = ServiceInstanceRelationClientSideMetrics.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE_INSTANCE_RELATION, builder = ServiceInstanceRelationClientSideMetrics.Builder.class, processor = MetricsStreamProcessor.class)
+public class ServiceInstanceRelationClientSideMetrics extends Metrics {
+
+    public static final String INDEX_NAME = "service_instance_relation_client_side";
+    public static final String SOURCE_SERVICE_ID = "source_service_id";
+    public static final String SOURCE_SERVICE_INSTANCE_ID = "source_service_instance_id";
+    public static final String DEST_SERVICE_ID = "dest_service_id";
+    public static final String DEST_SERVICE_INSTANCE_ID = "dest_service_instance_id";
+    public static final String COMPONENT_ID = "component_id";
+
+    @Setter @Getter @Column(columnName = SOURCE_SERVICE_ID) private int sourceServiceId;
+    @Setter @Getter @Column(columnName = SOURCE_SERVICE_INSTANCE_ID) @IDColumn private int sourceServiceInstanceId;
+    @Setter @Getter @Column(columnName = DEST_SERVICE_ID) private int destServiceId;
+    @Setter @Getter @Column(columnName = DEST_SERVICE_INSTANCE_ID) @IDColumn private int destServiceInstanceId;
+    @Setter @Getter @Column(columnName = COMPONENT_ID) @IDColumn private int componentId;
+    @Setter(AccessLevel.PRIVATE) @Getter @Column(columnName = ENTITY_ID) @IDColumn private String entityId;
+
+    @Override
+    public String id() {
+        String splitJointId = String.valueOf(getTimeBucket());
+        splitJointId += Const.ID_SPLIT + RelationDefineUtil.buildEntityId(
+                new RelationDefineUtil.RelationDefine(sourceServiceInstanceId, destServiceInstanceId, componentId)
+        );
+        return splitJointId;
+    }
+
+    public void buildEntityId() {
+        String splitJointId = String.valueOf(sourceServiceInstanceId);
+        splitJointId += Const.ID_SPLIT + destServiceInstanceId;
+        splitJointId += Const.ID_SPLIT + componentId;
+        entityId = splitJointId;
+    }
+
+    @Override
+    public void combine(Metrics metrics) {
+
+    }
+
+    @Override
+    public void calculate() {
+
+    }
+
+    @Override
+    public Metrics toHour() {
+        ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInHour());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public Metrics toDay() {
+        ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInHour());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public Metrics toMonth() {
+        ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInHour());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public int remoteHashCode() {
+        int result = sourceServiceInstanceId;
+        result = 31 * result + destServiceInstanceId;
+        result = 31 * result + componentId;
+        return result;
+    }
+
+    @Override
+    public void deserialize(RemoteData remoteData) {
+        setEntityId(remoteData.getDataStrings(0));
+
+        setSourceServiceId(remoteData.getDataIntegers(0));
+        setSourceServiceInstanceId(remoteData.getDataIntegers(1));
+        setDestServiceId(remoteData.getDataIntegers(2));
+        setDestServiceInstanceId(remoteData.getDataIntegers(3));
+        setComponentId(remoteData.getDataIntegers(4));
+
+        setTimeBucket(remoteData.getDataLongs(0));
+    }
+
+    @Override
+    public RemoteData.Builder serialize() {
+        RemoteData.Builder remoteBuilder = RemoteData.newBuilder();
+
+        remoteBuilder.addDataIntegers(getSourceServiceId());
+        remoteBuilder.addDataIntegers(getSourceServiceInstanceId());
+        remoteBuilder.addDataIntegers(getDestServiceId());
+        remoteBuilder.addDataIntegers(getDestServiceInstanceId());
+        remoteBuilder.addDataIntegers(getComponentId());
+
+        remoteBuilder.addDataStrings(getEntityId());
+
+        remoteBuilder.addDataLongs(getTimeBucket());
+        return remoteBuilder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ServiceInstanceRelationClientSideMetrics that = (ServiceInstanceRelationClientSideMetrics) o;
+
+        if (sourceServiceInstanceId != that.sourceServiceInstanceId) return false;
+        if (destServiceInstanceId != that.destServiceInstanceId) return false;
+        return componentId == that.componentId;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = sourceServiceInstanceId;
+        result = 31 * result + destServiceInstanceId;
+        result = 31 * result + componentId;
+        return result;
+    }
+
+    public static class Builder implements StorageBuilder<ServiceInstanceRelationClientSideMetrics> {
+
+
+        @Override
+        public ServiceInstanceRelationClientSideMetrics map2Data(Map<String, Object> dbMap) {
+            ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics();
+            metrics.setEntityId((String) dbMap.get(ENTITY_ID));
+            metrics.setSourceServiceId(((Number)dbMap.get(SOURCE_SERVICE_ID)).intValue());
+            metrics.setSourceServiceInstanceId(((Number) dbMap.get(SOURCE_SERVICE_INSTANCE_ID)).intValue());
+            metrics.setDestServiceId(((Number)dbMap.get(DEST_SERVICE_ID)).intValue());
+            metrics.setDestServiceInstanceId(((Number) dbMap.get(DEST_SERVICE_INSTANCE_ID)).intValue());
+            metrics.setComponentId(((Number) dbMap.get(COMPONENT_ID)).intValue());
+            metrics.setTimeBucket(((Number) dbMap.get(TIME_BUCKET)).longValue());
+            return metrics;
+        }
+
+        @Override
+        public Map<String, Object> data2Map(ServiceInstanceRelationClientSideMetrics storageData) {
+            Map<String, Object> map = new HashMap<>();
+            map.put(ENTITY_ID, storageData.getEntityId());
+            map.put(SOURCE_SERVICE_ID, storageData.getSourceServiceId());
+            map.put(SOURCE_SERVICE_INSTANCE_ID, storageData.getSourceServiceInstanceId());
+            map.put(DEST_SERVICE_ID, storageData.getDestServiceId());
+            map.put(DEST_SERVICE_INSTANCE_ID, storageData.getDestServiceInstanceId());
+            map.put(COMPONENT_ID, storageData.getComponentId());
+            map.put(TIME_BUCKET, storageData.getTimeBucket());
+            return map;
+        }
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/instance/ServiceInstanceRelationServerSideMetrics.java

@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.instance;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.analysis.Stream;
+import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
+import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
+import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
+import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData;
+import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
+import org.apache.skywalking.oap.server.core.storage.StorageBuilder;
+import org.apache.skywalking.oap.server.core.storage.annotation.Column;
+import org.apache.skywalking.oap.server.core.storage.annotation.IDColumn;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author zhangwei
+ */
+@Stream(name = ServiceInstanceRelationServerSideMetrics.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE_INSTANCE_RELATION, builder = ServiceInstanceRelationServerSideMetrics.Builder.class, processor = MetricsStreamProcessor.class)
+public class ServiceInstanceRelationServerSideMetrics extends Metrics {
+
+    public static final String INDEX_NAME = "service_instance_relation_server_side";
+    public static final String SOURCE_SERVICE_ID = "source_service_id";
+    public static final String SOURCE_SERVICE_INSTANCE_ID = "source_service_instance_id";
+    public static final String DEST_SERVICE_ID = "dest_service_id";
+    public static final String DEST_SERVICE_INSTANCE_ID = "dest_service_instance_id";
+    public static final String COMPONENT_ID = "component_id";
+
+    @Setter @Getter @Column(columnName = SOURCE_SERVICE_ID) private int sourceServiceId;
+    @Setter @Getter @Column(columnName = SOURCE_SERVICE_INSTANCE_ID) @IDColumn private int sourceServiceInstanceId;
+    @Setter @Getter @Column(columnName = DEST_SERVICE_ID) private int destServiceId;
+    @Setter @Getter @Column(columnName = DEST_SERVICE_INSTANCE_ID) @IDColumn private int destServiceInstanceId;
+    @Setter @Getter @Column(columnName = COMPONENT_ID) @IDColumn private int componentId;
+    @Setter(AccessLevel.PRIVATE) @Getter @Column(columnName = ENTITY_ID) @IDColumn private String entityId;
+
+    @Override
+    public String id() {
+        String splitJointId = String.valueOf(getTimeBucket());
+        splitJointId += Const.ID_SPLIT + RelationDefineUtil.buildEntityId(
+                new RelationDefineUtil.RelationDefine(sourceServiceInstanceId, destServiceInstanceId, componentId)
+        );
+        return splitJointId;
+    }
+
+    public void buildEntityId() {
+        String splitJointId = String.valueOf(sourceServiceInstanceId);
+        splitJointId += Const.ID_SPLIT + destServiceInstanceId;
+        splitJointId += Const.ID_SPLIT + componentId;
+        entityId = splitJointId;
+    }
+
+    @Override
+    public void combine(Metrics metrics) {
+
+    }
+
+    @Override
+    public void calculate() {
+
+    }
+
+    @Override
+    public Metrics toHour() {
+        ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInHour());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public Metrics toDay() {
+        ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInDay());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public Metrics toMonth() {
+        ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics();
+        metrics.setTimeBucket(toTimeBucketInMonth());
+        metrics.setSourceServiceId(getSourceServiceId());
+        metrics.setSourceServiceInstanceId(getSourceServiceInstanceId());
+        metrics.setDestServiceId(getDestServiceId());
+        metrics.setDestServiceInstanceId(getDestServiceInstanceId());
+        metrics.setComponentId(getComponentId());
+        metrics.setEntityId(getEntityId());
+        return metrics;
+    }
+
+    @Override
+    public int remoteHashCode() {
+        int result = sourceServiceInstanceId;
+        result = 31 * result + destServiceInstanceId;
+        result = 31 * result + componentId;
+        return result;
+    }
+
+    @Override
+    public void deserialize(RemoteData remoteData) {
+        setEntityId(remoteData.getDataStrings(0));
+
+        setSourceServiceId(remoteData.getDataIntegers(0));
+        setSourceServiceInstanceId(remoteData.getDataIntegers(1));
+        setDestServiceId(remoteData.getDataIntegers(2));
+        setDestServiceInstanceId(remoteData.getDataIntegers(3));
+        setComponentId(remoteData.getDataIntegers(4));
+
+        setTimeBucket(remoteData.getDataLongs(0));
+    }
+
+    @Override
+    public RemoteData.Builder serialize() {
+        RemoteData.Builder remoteBuilder = RemoteData.newBuilder();
+
+        remoteBuilder.addDataIntegers(getSourceServiceId());
+        remoteBuilder.addDataIntegers(getSourceServiceInstanceId());
+        remoteBuilder.addDataIntegers(getDestServiceId());
+        remoteBuilder.addDataIntegers(getDestServiceInstanceId());
+        remoteBuilder.addDataIntegers(getComponentId());
+
+        remoteBuilder.addDataStrings(getEntityId());
+
+        remoteBuilder.addDataLongs(getTimeBucket());
+        return remoteBuilder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ServiceInstanceRelationServerSideMetrics that = (ServiceInstanceRelationServerSideMetrics) o;
+
+        if (sourceServiceInstanceId != that.sourceServiceInstanceId) return false;
+        if (destServiceInstanceId != that.destServiceInstanceId) return false;
+        return componentId == that.componentId;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = sourceServiceInstanceId;
+        result = 31 * result + destServiceInstanceId;
+        result = 31 * result + componentId;
+        return result;
+    }
+
+    public static class Builder implements StorageBuilder<ServiceInstanceRelationServerSideMetrics> {
+
+        @Override
+        public ServiceInstanceRelationServerSideMetrics map2Data(Map<String, Object> dbMap) {
+            ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics();
+            metrics.setEntityId((String) dbMap.get(ENTITY_ID));
+            metrics.setSourceServiceId(((Number)dbMap.get(SOURCE_SERVICE_ID)).intValue());
+            metrics.setSourceServiceInstanceId(((Number) dbMap.get(SOURCE_SERVICE_INSTANCE_ID)).intValue());
+            metrics.setDestServiceId(((Number)dbMap.get(DEST_SERVICE_ID)).intValue());
+            metrics.setDestServiceInstanceId(((Number) dbMap.get(DEST_SERVICE_INSTANCE_ID)).intValue());
+            metrics.setComponentId(((Number) dbMap.get(COMPONENT_ID)).intValue());
+            metrics.setTimeBucket(((Number) dbMap.get(TIME_BUCKET)).longValue());
+            return metrics;
+        }
+
+        @Override
+        public Map<String, Object> data2Map(ServiceInstanceRelationServerSideMetrics storageData) {
+            Map<String, Object> map = new HashMap<>();
+            map.put(ENTITY_ID, storageData.getEntityId());
+            map.put(SOURCE_SERVICE_ID, storageData.getSourceServiceId());
+            map.put(SOURCE_SERVICE_INSTANCE_ID, storageData.getSourceServiceInstanceId());
+            map.put(DEST_SERVICE_ID, storageData.getDestServiceId());
+            map.put(DEST_SERVICE_INSTANCE_ID, storageData.getDestServiceInstanceId());
+            map.put(COMPONENT_ID, storageData.getComponentId());
+            map.put(TIME_BUCKET, storageData.getTimeBucket());
+            return map;
+        }
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceCallRelationDispatcher.java

@@ -16,7 +16,7 @@
  *
  */
 
-package org.apache.skywalking.oap.server.core.analysis.manual.servicerelation;
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.service;
 
 import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
 import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java

@@ -16,7 +16,7 @@
  *
  */
 
-package org.apache.skywalking.oap.server.core.analysis.manual.servicerelation;
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.service;
 
 import java.util.*;
 import lombok.*;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java

@@ -16,7 +16,7 @@
  *
  */
 
-package org.apache.skywalking.oap.server.core.analysis.manual.servicerelation;
+package org.apache.skywalking.oap.server.core.analysis.manual.relation.service;
 
 import java.util.*;
 import lombok.*;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/cache/CacheUpdateTimer.java

@@ -54,6 +54,7 @@ public enum CacheUpdateTimer {
 
     private void update(ModuleDefineHolder moduleDefineHolder) {
         updateServiceInventory(moduleDefineHolder);
+        updateServiceInstanceInventory(moduleDefineHolder);
         updateNetAddressInventory(moduleDefineHolder);
     }
 
@@ -77,6 +78,26 @@ public enum CacheUpdateTimer {
         });
     }
 
+    private void updateServiceInstanceInventory(ModuleDefineHolder moduleDefineHolder) {
+        IServiceInstanceInventoryCacheDAO instanceInventoryCacheDAO = moduleDefineHolder.find(StorageModule.NAME).provider().getService(IServiceInstanceInventoryCacheDAO.class);
+        ServiceInstanceInventoryCache instanceInventoryCache = moduleDefineHolder.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
+        List<ServiceInstanceInventory> instanceInventories = instanceInventoryCacheDAO.loadLastUpdate(System.currentTimeMillis() - 60000);
+
+        instanceInventories.forEach(instanceInventory -> {
+            ServiceInstanceInventory cache = instanceInventoryCache.get(instanceInventory.getSequence());
+            if (Objects.nonNull(cache)) {
+                if (cache.getMappingServiceInstanceId() != instanceInventory.getMappingServiceInstanceId()) {
+                    cache.setMappingServiceInstanceId(instanceInventory.getMappingServiceInstanceId());
+                    cache.setServiceInstanceNodeType(instanceInventory.getServiceInstanceNodeType());
+                    cache.setProperties(instanceInventory.getProperties());
+                    logger.info("Update the cache of service instance inventory, instance id: {}", instanceInventory.getSequence());
+                }
+            } else {
+                logger.warn("Unable to found the id of {} in service instance inventory cache.", instanceInventory.getSequence());
+            }
+        });
+    }
+
     private void updateNetAddressInventory(ModuleDefineHolder moduleDefineHolder) {
         INetworkAddressInventoryCacheDAO addressInventoryCacheDAO = moduleDefineHolder.find(StorageModule.NAME).provider().getService(INetworkAddressInventoryCacheDAO.class);
         NetworkAddressInventoryCache addressInventoryCache = moduleDefineHolder.find(CoreModule.NAME).provider().getService(NetworkAddressInventoryCache.class);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ServiceInstanceTopologyBuilder.java

@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.query;
+
+import groovy.util.logging.Slf4j;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
+import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
+import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
+import org.apache.skywalking.oap.server.core.query.entity.*;
+import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
+import org.apache.skywalking.oap.server.core.register.ServiceInventory;
+import org.apache.skywalking.oap.server.core.source.DetectPoint;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.util.BooleanUtils;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Objects.isNull;
+
+/**
+ * @author zhangwei
+ */
+@Slf4j
+public class ServiceInstanceTopologyBuilder {
+
+    private final ServiceInventoryCache serviceInventoryCache;
+    private final ServiceInstanceInventoryCache serviceInstanceInventoryCache;
+    private final IComponentLibraryCatalogService componentLibraryCatalogService;
+
+    public ServiceInstanceTopologyBuilder(ModuleManager moduleManager) {
+        this.serviceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
+        this.serviceInstanceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
+        this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class);
+    }
+
+    ServiceInstanceTopology build(List<Call.CallDetail> serviceInstanceRelationClientCalls, List<Call.CallDetail> serviceInstanceRelationServerCalls) {
+        filterZeroSourceOrTargetReference(serviceInstanceRelationClientCalls);
+        filterZeroSourceOrTargetReference(serviceInstanceRelationServerCalls);
+
+        Map<Integer, ServiceInstanceNode> nodes = new HashMap<>();
+        List<Call> calls = new LinkedList<>();
+        HashMap<String, Call> callMap = new HashMap<>();
+
+        for (Call.CallDetail clientCall : serviceInstanceRelationClientCalls) {
+            ServiceInstanceInventory sourceInstance = serviceInstanceInventoryCache.get(clientCall.getSource());
+            ServiceInstanceInventory targetInstance = serviceInstanceInventoryCache.get(clientCall.getTarget());
+
+            if (isNull(sourceInstance) || isNull(targetInstance)) {
+                continue;
+            }
+
+            if (targetInstance.getMappingServiceInstanceId() != Const.NONE) {
+                continue;
+            }
+
+            if (!nodes.containsKey(sourceInstance.getSequence())) {
+                ServiceInventory sourceService = serviceInventoryCache.get(sourceInstance.getServiceId());
+                nodes.put(sourceInstance.getSequence(), buildNode(sourceService, sourceInstance));
+            }
+
+            if (!nodes.containsKey(targetInstance.getSequence())) {
+                ServiceInventory targetService = serviceInventoryCache.get(targetInstance.getServiceId());
+                nodes.put(targetInstance.getSequence(), buildNode(targetService, targetInstance));
+                if (BooleanUtils.valueToBoolean(targetInstance.getIsAddress())) {
+                    nodes.get(targetInstance.getSequence()).setType(componentLibraryCatalogService.getServerNameBasedOnComponent(clientCall.getComponentId()));
+                }
+            }
+
+            String callId = sourceInstance.getSequence() + Const.ID_SPLIT + targetInstance.getSequence();
+            if (!callMap.containsKey(callId)) {
+                Call call = new Call();
+
+                callMap.put(callId, call);
+
+                call.setSource(clientCall.getSource());
+                call.setTarget(clientCall.getTarget());
+                call.setId(clientCall.getId());
+                call.addDetectPoint(DetectPoint.CLIENT);
+                call.addSourceComponent(componentLibraryCatalogService.getComponentName(clientCall.getComponentId()));
+                calls.add(call);
+            } else {
+                Call call = callMap.get(callId);
+                call.addDetectPoint(DetectPoint.CLIENT);
+                call.addSourceComponent(componentLibraryCatalogService.getComponentName(clientCall.getComponentId()));
+            }
+        }
+
+        for (Call.CallDetail serverCall : serviceInstanceRelationServerCalls) {
+            ServiceInstanceInventory sourceInstance = serviceInstanceInventoryCache.get(serverCall.getSource());
+            ServiceInstanceInventory targetInstance = serviceInstanceInventoryCache.get(serverCall.getTarget());
+
+            if (isNull(sourceInstance) || isNull(targetInstance)) {
+                continue;
+            }
+
+            if (sourceInstance.getSequence() == Const.USER_INSTANCE_ID) {
+                if (!nodes.containsKey(sourceInstance.getSequence())) {
+                    ServiceInstanceNode visualUserNode = new ServiceInstanceNode();
+                    visualUserNode.setId(sourceInstance.getSequence());
+                    visualUserNode.setName(Const.USER_CODE);
+                    visualUserNode.setServiceId(Const.USER_SERVICE_ID);
+                    visualUserNode.setServiceName(Const.USER_CODE);
+                    visualUserNode.setType(Const.USER_CODE.toUpperCase());
+                    visualUserNode.setReal(false);
+                    nodes.put(sourceInstance.getSequence(), visualUserNode);
+                }
+            }
+
+            if (BooleanUtils.valueToBoolean(sourceInstance.getIsAddress())) {
+                if (!nodes.containsKey(sourceInstance.getSequence())) {
+                    ServiceInventory sourceService = serviceInventoryCache.get(sourceInstance.getServiceId());
+                    ServiceInstanceNode conjecturalNode = new ServiceInstanceNode();
+                    conjecturalNode.setId(sourceInstance.getSequence());
+                    conjecturalNode.setName(sourceInstance.getName());
+                    conjecturalNode.setServiceId(sourceService.getSequence());
+                    conjecturalNode.setServiceName(sourceService.getName());
+                    conjecturalNode.setType(componentLibraryCatalogService.getServerNameBasedOnComponent(serverCall.getComponentId()));
+                    conjecturalNode.setReal(true);
+                    nodes.put(sourceInstance.getSequence(), conjecturalNode);
+                }
+            }
+
+            String callId = sourceInstance.getSequence() + Const.ID_SPLIT + targetInstance.getSequence();
+            if (!callMap.containsKey(callId)) {
+                Call call = new Call();
+                callMap.put(callId, call);
+
+                call.setSource(serverCall.getSource());
+                call.setTarget(serverCall.getTarget());
+                call.setId(callId);
+                call.addDetectPoint(DetectPoint.SERVER);
+                call.addTargetComponent(componentLibraryCatalogService.getComponentName(serverCall.getComponentId()));
+
+                calls.add(call);
+            } else {
+                Call call = callMap.get(callId);
+
+                call.addDetectPoint(DetectPoint.SERVER);
+                call.addTargetComponent(componentLibraryCatalogService.getComponentName(serverCall.getComponentId()));
+            }
+
+            if (!nodes.containsKey(sourceInstance.getSequence())) {
+                ServiceInventory sourceService = serviceInventoryCache.get(sourceInstance.getServiceId());
+                nodes.put(sourceInstance.getSequence(), buildNode(sourceService, sourceInstance));
+            }
+
+            if (!nodes.containsKey(targetInstance.getSequence())) {
+                ServiceInventory targetService = serviceInventoryCache.get(targetInstance.getServiceId());
+                nodes.put(targetInstance.getSequence(), buildNode(targetService, targetInstance));
+            }
+
+            if (nodes.containsKey(targetInstance.getSequence())) {
+                nodes.get(targetInstance.getSequence()).setType(componentLibraryCatalogService.getComponentName(serverCall.getComponentId()));
+            }
+        }
+
+        ServiceInstanceTopology topology = new ServiceInstanceTopology();
+        topology.getCalls().addAll(calls);
+        topology.getNodes().addAll(nodes.values());
+        return topology;
+    }
+
+    private ServiceInstanceNode buildNode(ServiceInventory serviceInventory, ServiceInstanceInventory instanceInventory) {
+        ServiceInstanceNode instanceNode = new ServiceInstanceNode();
+        instanceNode.setId(instanceInventory.getSequence());
+        instanceNode.setName(instanceInventory.getName());
+        instanceNode.setServiceId(serviceInventory.getSequence());
+        instanceNode.setServiceName(serviceInventory.getName());
+        if (BooleanUtils.valueToBoolean(instanceInventory.getIsAddress())) {
+            instanceNode.setReal(false);
+        } else {
+            instanceNode.setReal(true);
+        }
+        return instanceNode;
+    }
+
+    private void filterZeroSourceOrTargetReference(List<Call.CallDetail> serviceRelationClientCalls) {
+        for (int i = serviceRelationClientCalls.size() - 1; i >= 0; i--) {
+            Call.CallDetail call = serviceRelationClientCalls.get(i);
+            if (call.getSource() == 0 || call.getTarget() == 0) {
+                serviceRelationClientCalls.remove(i);
+            }
+        }
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java

@@ -19,11 +19,6 @@
 package org.apache.skywalking.oap.server.core.query;
 
 import com.google.common.base.Strings;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
 import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.analysis.Downsampling;
@@ -31,6 +26,7 @@ import org.apache.skywalking.oap.server.core.cache.EndpointInventoryCache;
 import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
 import org.apache.skywalking.oap.server.core.query.entity.Call;
 import org.apache.skywalking.oap.server.core.query.entity.Node;
+import org.apache.skywalking.oap.server.core.query.entity.ServiceInstanceTopology;
 import org.apache.skywalking.oap.server.core.query.entity.Topology;
 import org.apache.skywalking.oap.server.core.source.DetectPoint;
 import org.apache.skywalking.oap.server.core.storage.StorageModule;
@@ -42,6 +38,12 @@ import org.apache.skywalking.oap.server.library.util.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
 /**
  * @author peng-yongsheng
  */
@@ -93,9 +95,7 @@ public class TopologyQueryService implements Service {
         List<Call.CallDetail> serviceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceRelations(downsampling, startTB, endTB);
 
         TopologyBuilder builder = new TopologyBuilder(moduleManager);
-        Topology topology = builder.build(serviceRelationClientCalls, serviceRelationServerCalls);
-
-        return topology;
+        return builder.build(serviceRelationClientCalls, serviceRelationServerCalls);
     }
 
     public Topology getServiceTopology(final Downsampling downsampling, final long startTB, final long endTB, final int serviceId) throws IOException {
@@ -127,6 +127,16 @@ public class TopologyQueryService implements Service {
         return topology;
     }
 
+    public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId, final Downsampling downsampling, final long startTB, final long endTB) throws IOException {
+        logger.debug("ClientServiceId: {}, ServerServiceId: {}, Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", clientServiceId, serverServiceId, downsampling, startTB, endTB);
+
+        List<Call.CallDetail> serviceInstanceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB);
+        List<Call.CallDetail> serviceInstanceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB);
+
+        ServiceInstanceTopologyBuilder builder = new ServiceInstanceTopologyBuilder(moduleManager);
+        return builder.build(serviceInstanceRelationClientCalls, serviceInstanceRelationServerCalls);
+    }
+
     public Topology getEndpointTopology(final Downsampling downsampling, final long startTB, final long endTB, final int endpointId) throws IOException {
         List<Call.CallDetail> serverSideCalls = getTopologyQueryDAO().loadSpecifiedDestOfServerSideEndpointRelations(downsampling, startTB, endTB, endpointId);
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ServiceInstanceNode.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.query.entity;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+public class ServiceInstanceNode {
+
+    private int id;
+    private String name;
+    private int serviceId;
+    private String serviceName;
+    private String type;
+    private boolean isReal;
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ServiceInstanceTopology.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.query.entity;
+
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author zhangwei
+ */
+@Getter
+public class ServiceInstanceTopology {
+
+    private final List<ServiceInstanceNode> nodes;
+    private final List<Call> calls;
+
+    public ServiceInstanceTopology() {
+        this.nodes = new ArrayList<>();
+        this.calls = new ArrayList<>();
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInstanceInventory.java

@@ -52,8 +52,10 @@ public class ServiceInstanceInventory extends RegisterSource {
     public static final String NAME = "name";
     public static final String INSTANCE_UUID = "instance_uuid";
     public static final String SERVICE_ID = "service_id";
-    private static final String IS_ADDRESS = "is_address";
+    public static final String IS_ADDRESS = "is_address";
     private static final String ADDRESS_ID = "address_id";
+    public static final String NODE_TYPE = "node_type";
+    public static final String MAPPING_SERVICE_ID = "mapping_service_instance_id";
     public static final String PROPERTIES = "properties";
     private static final Gson GSON = new Gson();
 
@@ -62,9 +64,13 @@ public class ServiceInstanceInventory extends RegisterSource {
     @Setter @Getter @Column(columnName = SERVICE_ID) private int serviceId;
     @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress;
     @Setter @Getter @Column(columnName = ADDRESS_ID) private int addressId;
+    @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PACKAGE) @Column(columnName = NODE_TYPE) private int nodeType;
+    @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceInstanceId;
     @Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop;
     @Getter private JsonObject properties;
 
+    @Setter @Getter private boolean resetServiceInstanceMapping = false;
+
     public static String buildId(int serviceId, String uuid) {
         return serviceId + Const.ID_SPLIT + uuid + Const.ID_SPLIT + BooleanUtils.FALSE + Const.ID_SPLIT + Const.NONE;
     }
@@ -73,6 +79,14 @@ public class ServiceInstanceInventory extends RegisterSource {
         return serviceId + Const.ID_SPLIT + BooleanUtils.TRUE + Const.ID_SPLIT + addressId;
     }
 
+    public NodeType getServiceInstanceNodeType() {
+        return NodeType.get(nodeType);
+    }
+
+    public void setServiceInstanceNodeType(NodeType nodeType) {
+        this.nodeType = nodeType.value();
+    }
+
     @Override public String id() {
         if (BooleanUtils.TRUE == isAddress) {
             return buildId(serviceId, addressId);
@@ -129,12 +143,35 @@ public class ServiceInstanceInventory extends RegisterSource {
         return true;
     }
 
+    public ServiceInstanceInventory getClone() {
+        ServiceInstanceInventory inventory = new ServiceInstanceInventory();
+        inventory.setInstanceUUID(instanceUUID);
+        inventory.setName(name);
+        inventory.setServiceId(serviceId);
+        inventory.setIsAddress(isAddress);
+        inventory.setAddressId(addressId);
+        inventory.setNodeType(nodeType);
+        inventory.setMappingServiceInstanceId(mappingServiceInstanceId);
+        inventory.setProp(prop);
+        inventory.setResetServiceInstanceMapping(resetServiceInstanceMapping);
+
+        inventory.setSequence(getSequence());
+        inventory.setRegisterTime(getRegisterTime());
+        inventory.setHeartbeatTime(getHeartbeatTime());
+        inventory.setLastUpdateTime(getLastUpdateTime());
+
+        return inventory;
+    }
+
     @Override public RemoteData.Builder serialize() {
         RemoteData.Builder remoteBuilder = RemoteData.newBuilder();
         remoteBuilder.addDataIntegers(getSequence());
         remoteBuilder.addDataIntegers(serviceId);
         remoteBuilder.addDataIntegers(isAddress);
         remoteBuilder.addDataIntegers(addressId);
+        remoteBuilder.addDataIntegers(nodeType);
+        remoteBuilder.addDataIntegers(mappingServiceInstanceId);
+        remoteBuilder.addDataIntegers(resetServiceInstanceMapping ? 1 : 0);
 
         remoteBuilder.addDataLongs(getRegisterTime());
         remoteBuilder.addDataLongs(getHeartbeatTime());
@@ -143,6 +180,7 @@ public class ServiceInstanceInventory extends RegisterSource {
         remoteBuilder.addDataStrings(Strings.isNullOrEmpty(name) ? Const.EMPTY_STRING : name);
         remoteBuilder.addDataStrings(Strings.isNullOrEmpty(instanceUUID) ? Const.EMPTY_STRING : instanceUUID);
         remoteBuilder.addDataStrings(Strings.isNullOrEmpty(prop) ? Const.EMPTY_STRING : prop);
+
         return remoteBuilder;
     }
 
@@ -151,6 +189,9 @@ public class ServiceInstanceInventory extends RegisterSource {
         setServiceId(remoteData.getDataIntegers(1));
         setIsAddress(remoteData.getDataIntegers(2));
         setAddressId(remoteData.getDataIntegers(3));
+        setNodeType(remoteData.getDataIntegers(4));
+        setMappingServiceInstanceId(remoteData.getDataIntegers(5));
+        setResetServiceInstanceMapping(remoteData.getDataIntegers(6) == 1);
 
         setRegisterTime(remoteData.getDataLongs(0));
         setHeartbeatTime(remoteData.getDataLongs(1));
@@ -165,6 +206,25 @@ public class ServiceInstanceInventory extends RegisterSource {
         return 0;
     }
 
+    @Override public boolean combine(RegisterSource registerSource) {
+        boolean isChanged = super.combine(registerSource);
+        ServiceInstanceInventory instanceInventory = (ServiceInstanceInventory) registerSource;
+
+        if (instanceInventory.getLastUpdateTime() >= this.getLastUpdateTime()) {
+            this.nodeType = instanceInventory.getNodeType();
+            this.resetServiceInstanceMapping = instanceInventory.isResetServiceInstanceMapping();
+            setProp(instanceInventory.getProp());
+            if (instanceInventory.isResetServiceInstanceMapping()) {
+                this.mappingServiceInstanceId = Const.NONE;
+            } else if (Const.NONE != instanceInventory.getMappingServiceInstanceId()) {
+                this.mappingServiceInstanceId = instanceInventory.mappingServiceInstanceId;
+            }
+            isChanged = true;
+        }
+
+        return isChanged;
+    }
+
     public static class Builder implements StorageBuilder<ServiceInstanceInventory> {
 
         @Override public ServiceInstanceInventory map2Data(Map<String, Object> dbMap) {
@@ -178,6 +238,9 @@ public class ServiceInstanceInventory extends RegisterSource {
             inventory.setHeartbeatTime(((Number)dbMap.get(HEARTBEAT_TIME)).longValue());
             inventory.setLastUpdateTime(((Number)dbMap.get(LAST_UPDATE_TIME)).longValue());
 
+            inventory.setNodeType(((Number)dbMap.get(NODE_TYPE)).intValue());
+            inventory.setMappingServiceInstanceId(((Number)dbMap.get(MAPPING_SERVICE_ID)).intValue());
+
             inventory.setName((String)dbMap.get(NAME));
             inventory.setInstanceUUID((String)dbMap.get(INSTANCE_UUID));
             inventory.setProp((String)dbMap.get(PROPERTIES));
@@ -195,6 +258,9 @@ public class ServiceInstanceInventory extends RegisterSource {
             map.put(HEARTBEAT_TIME, storageData.getHeartbeatTime());
             map.put(LAST_UPDATE_TIME, storageData.getLastUpdateTime());
 
+            map.put(NODE_TYPE, storageData.getNodeType());
+            map.put(MAPPING_SERVICE_ID, storageData.getMappingServiceInstanceId());
+
             map.put(NAME, storageData.getName());
             map.put(INSTANCE_UUID, storageData.getInstanceUUID());
             map.put(PROPERTIES, storageData.getProp());

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInstanceInventoryRegister.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.core.register.service;
 
 import com.google.gson.JsonObject;
+import org.apache.skywalking.oap.server.core.register.NodeType;
 import org.apache.skywalking.oap.server.library.module.Service;
 
 /**
@@ -29,7 +30,23 @@ public interface IServiceInstanceInventoryRegister extends Service {
     int getOrCreate(int serviceId, String serviceInstanceName, String uuid, long registerTime,
         JsonObject properties);
 
-    int getOrCreate(int serviceId, int addressId, long registerTime);
+    int getOrCreate(int serviceId, String serviceInstanceName, int addressId, long registerTime);
+
+    void update(int serviceInstanceId, NodeType nodeType, JsonObject properties);
 
     void heartbeat(int serviceInstanceId, long heartBeatTime);
+
+    void updateMapping(int serviceInstanceId, int mappingServiceInstanceId);
+
+    /**
+     * Reset the {@link org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory#mappingServiceInstanceId}
+     * of a given service id.
+     *
+     * There are cases when the mapping service id needs to be reset to {@code 0}, for example, when an
+     * uninstrumented gateway joins, the mapping service id of the services that are delegated by this gateway
+     * should be reset to {@code 0}, allowing the gateway to appear in the topology, see #3308 for more detail.
+     *
+     * @param serviceInstanceId id of the service whose mapping service id is to be reset
+     */
+    void resetMapping(int serviceInstanceId);
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java

@@ -73,7 +73,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory
             int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress, properties);
 
             if (serviceId != Const.NONE) {
-                int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, addressId, System.currentTimeMillis());
+                int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, networkAddress, addressId, System.currentTimeMillis());
 
                 if (serviceInstanceId != Const.NONE) {
                     return addressId;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInstanceInventoryRegister.java

@@ -19,9 +19,9 @@
 package org.apache.skywalking.oap.server.core.register.service;
 
 import com.google.gson.JsonObject;
-import java.util.Objects;
 import org.apache.skywalking.oap.server.core.*;
 import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
+import org.apache.skywalking.oap.server.core.register.NodeType;
 import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
 import org.apache.skywalking.oap.server.core.register.worker.InventoryStreamProcessor;
 import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder;
@@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.library.util.BooleanUtils;
 import org.slf4j.*;
 
 import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
 
 /**
  * @author peng-yongsheng
@@ -66,6 +67,7 @@ public class ServiceInstanceInventoryRegister implements IServiceInstanceInvento
             serviceInstanceInventory.setInstanceUUID(uuid);
             serviceInstanceInventory.setIsAddress(BooleanUtils.FALSE);
             serviceInstanceInventory.setAddressId(Const.NONE);
+            serviceInstanceInventory.setMappingServiceInstanceId(Const.NONE);
 
             serviceInstanceInventory.setRegisterTime(registerTime);
             serviceInstanceInventory.setHeartbeatTime(registerTime);
@@ -77,7 +79,7 @@ public class ServiceInstanceInventoryRegister implements IServiceInstanceInvento
         return serviceInstanceId;
     }
 
-    @Override public int getOrCreate(int serviceId, int addressId, long registerTime) {
+    @Override public int getOrCreate(int serviceId, String serviceInstanceName, int addressId, long registerTime) {
         if (logger.isDebugEnabled()) {
             logger.debug("get or create service instance by getAddress id, service id: {}, getAddress id: {}, registerTime: {}", serviceId, addressId, registerTime);
         }
@@ -87,9 +89,10 @@ public class ServiceInstanceInventoryRegister implements IServiceInstanceInvento
         if (serviceInstanceId == Const.NONE) {
             ServiceInstanceInventory serviceInstanceInventory = new ServiceInstanceInventory();
             serviceInstanceInventory.setServiceId(serviceId);
-            serviceInstanceInventory.setName(Const.EMPTY_STRING);
+            serviceInstanceInventory.setName(serviceInstanceName);
             serviceInstanceInventory.setIsAddress(BooleanUtils.TRUE);
             serviceInstanceInventory.setAddressId(addressId);
+            serviceInstanceInventory.setMappingServiceInstanceId(Const.NONE);
 
             serviceInstanceInventory.setRegisterTime(registerTime);
             serviceInstanceInventory.setHeartbeatTime(registerTime);
@@ -99,13 +102,63 @@ public class ServiceInstanceInventoryRegister implements IServiceInstanceInvento
         return serviceInstanceId;
     }
 
+    @Override
+    public void update(int serviceInstanceId, NodeType nodeType, JsonObject properties) {
+        ServiceInstanceInventory instanceInventory = getServiceInstanceInventoryCache().get(serviceInstanceId);
+        if (nonNull(instanceInventory)) {
+            if (properties != null || !compare(instanceInventory, nodeType)) {
+                instanceInventory = instanceInventory.getClone();
+                instanceInventory.setServiceInstanceNodeType(nodeType);
+                instanceInventory.setProperties(properties);
+                instanceInventory.setLastUpdateTime(System.currentTimeMillis());
+
+                InventoryStreamProcessor.getInstance().in(instanceInventory);
+            }
+        } else {
+            logger.warn("ServiceInstance {} nodeType/properties update, but not found in storage.", serviceInstanceId);
+        }
+    }
+
     @Override public void heartbeat(int serviceInstanceId, long heartBeatTime) {
         ServiceInstanceInventory serviceInstanceInventory = getServiceInstanceInventoryCache().get(serviceInstanceId);
-        if (Objects.nonNull(serviceInstanceInventory)) {
+        if (nonNull(serviceInstanceInventory)) {
             serviceInstanceInventory.setHeartbeatTime(heartBeatTime);
             InventoryStreamProcessor.getInstance().in(serviceInstanceInventory);
         } else {
             logger.warn("Service instance {} heartbeat, but not found in storage.", serviceInstanceId);
         }
     }
+
+    @Override public void updateMapping(int serviceInstanceId, int mappingServiceInstanceId) {
+        ServiceInstanceInventory instanceInventory = getServiceInstanceInventoryCache().get(serviceInstanceId);
+        if (nonNull(instanceInventory)) {
+            instanceInventory = instanceInventory.getClone();
+            instanceInventory.setMappingServiceInstanceId(mappingServiceInstanceId);
+            instanceInventory.setLastUpdateTime(System.currentTimeMillis());
+
+            InventoryStreamProcessor.getInstance().in(instanceInventory);
+        } else {
+            logger.warn("ServiceInstance {} mapping update, but not found in storage.", serviceInstanceId);
+        }
+    }
+
+    @Override public void resetMapping(int serviceInstanceId) {
+        ServiceInstanceInventory instanceInventory = getServiceInstanceInventoryCache().get(serviceInstanceId);
+        if (nonNull(instanceInventory) && instanceInventory.getMappingServiceInstanceId() != Const.NONE) {
+            instanceInventory = instanceInventory.getClone();
+            instanceInventory.setLastUpdateTime(System.currentTimeMillis());
+            instanceInventory.setResetServiceInstanceMapping(true);
+
+            InventoryStreamProcessor.getInstance().in(instanceInventory);
+        } else {
+            logger.warn("ServiceInstance {} mapping update, but not found in storage.", serviceInstanceId);
+        }
+    }
+
+    private boolean compare(ServiceInstanceInventory newServiceInstanceInventory, NodeType nodeType) {
+        if (nonNull(newServiceInstanceInventory)) {
+            return nodeType.equals(newServiceInstanceInventory.getServiceInstanceNodeType());
+        }
+        return true;
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/cache/IServiceInstanceInventoryCacheDAO.java

@@ -21,6 +21,8 @@ package org.apache.skywalking.oap.server.core.storage.cache;
 import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
 import org.apache.skywalking.oap.server.core.storage.DAO;
 
+import java.util.List;
+
 /**
  * @author peng-yongsheng
  */
@@ -31,4 +33,6 @@ public interface IServiceInstanceInventoryCacheDAO extends DAO {
     int getServiceInstanceId(int serviceId, String uuid);
 
     int getServiceInstanceId(int serviceId, int addressId);
+
+    List<ServiceInstanceInventory> loadLastUpdate(long lastUpdateTime);
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java

@@ -37,5 +37,9 @@ public interface ITopologyQueryDAO extends Service {
 
     List<Call.CallDetail> loadClientSideServiceRelations(Downsampling downsampling, long startTB, long endTB) throws IOException;
 
+    List<Call.CallDetail> loadServerSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException;
+
+    List<Call.CallDetail> loadClientSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException;
+
     List<Call.CallDetail> loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, long startTB, long endTB, int destEndpointId) throws IOException;
 }

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java

@@ -19,13 +19,17 @@
 package org.apache.skywalking.oap.query.graphql.resolver;
 
 import com.coxautodev.graphql.tools.GraphQLQueryResolver;
-import java.io.IOException;
 import org.apache.skywalking.oap.query.graphql.type.Duration;
 import org.apache.skywalking.oap.server.core.CoreModule;
-import org.apache.skywalking.oap.server.core.query.*;
+import org.apache.skywalking.oap.server.core.query.DurationUtils;
+import org.apache.skywalking.oap.server.core.query.StepToDownsampling;
+import org.apache.skywalking.oap.server.core.query.TopologyQueryService;
+import org.apache.skywalking.oap.server.core.query.entity.ServiceInstanceTopology;
 import org.apache.skywalking.oap.server.core.query.entity.Topology;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 
+import java.io.IOException;
+
 /**
  * @author peng-yongsheng
  */
@@ -59,6 +63,13 @@ public class TopologyQuery implements GraphQLQueryResolver {
         return getQueryService().getServiceTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, serviceId);
     }
 
+    public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId, final Duration duration) throws IOException {
+        long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
+        long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
+
+        return getQueryService().getServiceInstanceTopology(clientServiceId, serverServiceId, StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket);
+    }
+
     public Topology getEndpointTopology(final int endpointId, final Duration duration) throws IOException {
         long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
         long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());

oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol

@@ -1 +1 @@
-Subproject commit c970b1972ad69d2622eb0b41a067c230d46b3da8
+Subproject commit e47462fd6af92d42d1c161cf1cec975661148ab0

oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/TraceModuleProvider.java

@@ -40,6 +40,7 @@ import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentPa
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.SegmentParserServiceImpl;
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.endpoint.MultiScopesSpanListener;
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.segment.SegmentSpanListener;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.service.ServiceInstanceMappingSpanListener;
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.service.ServiceMappingSpanListener;
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.standardization.SegmentStandardizationWorker;
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
@@ -92,6 +93,7 @@ public class TraceModuleProvider extends ModuleProvider {
         if (moduleConfig.isTraceAnalysis()) {
             listenerManager.add(new MultiScopesSpanListener.Factory());
             listenerManager.add(new ServiceMappingSpanListener.Factory());
+            listenerManager.add(new ServiceInstanceMappingSpanListener.Factory());
         }
         listenerManager.add(new SegmentSpanListener.Factory(moduleConfig.getSampleRate()));
 

oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/endpoint/MultiScopesSpanListener.java

@@ -159,17 +159,22 @@ public class MultiScopesSpanListener implements EntrySpanListener, ExitSpanListe
         int destServiceId = serviceInventoryCache.getServiceId(peerId);
         int mappingServiceId = serviceInventoryCache.get(destServiceId).getMappingServiceId();
         int destInstanceId = instanceInventoryCache.getServiceInstanceId(destServiceId, peerId);
+        int mappingServiceInstanceId = instanceInventoryCache.get(destInstanceId).getMappingServiceInstanceId();
 
         sourceBuilder.setSourceEndpointId(Const.USER_ENDPOINT_ID);
         sourceBuilder.setSourceServiceInstanceId(segmentCoreInfo.getServiceInstanceId());
         sourceBuilder.setSourceServiceId(segmentCoreInfo.getServiceId());
         sourceBuilder.setDestEndpointId(spanDecorator.getOperationNameId());
-        sourceBuilder.setDestServiceInstanceId(destInstanceId);
         if (Const.NONE == mappingServiceId) {
             sourceBuilder.setDestServiceId(destServiceId);
         } else {
             sourceBuilder.setDestServiceId(mappingServiceId);
         }
+        if (Const.NONE == mappingServiceInstanceId) {
+            sourceBuilder.setDestServiceInstanceId(destInstanceId);
+        } else {
+            sourceBuilder.setDestServiceInstanceId(mappingServiceInstanceId);
+        }
         sourceBuilder.setDetectPoint(DetectPoint.CLIENT);
         sourceBuilder.setComponentId(spanDecorator.getComponentId());
         setPublicAttrs(sourceBuilder, spanDecorator);

oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/service/ServiceInstanceMappingSpanListener.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.service;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.apm.network.language.agent.SpanLayer;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.cache.NetworkAddressInventoryCache;
+import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
+import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
+import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
+import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister;
+import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.receiver.trace.provider.TraceServiceModuleConfig;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator.SegmentCoreInfo;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator.SpanDecorator;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.EntrySpanListener;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.SpanListener;
+import org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.SpanListenerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author zhangwei
+ */
+public class ServiceInstanceMappingSpanListener implements EntrySpanListener {
+
+    private static final Logger logger = LoggerFactory.getLogger(ServiceInstanceMappingSpanListener.class);
+
+    private final IServiceInstanceInventoryRegister serviceInstanceInventoryRegister;
+    private final TraceServiceModuleConfig config;
+    private final ServiceInventoryCache serviceInventoryCache;
+    private final ServiceInstanceInventoryCache serviceInstanceInventoryCache;
+    private final NetworkAddressInventoryCache networkAddressInventoryCache;
+    private final List<ServiceInstanceMapping> serviceInstanceMappings = new ArrayList<>();
+    private final List<Integer> serviceInstancesToResetMapping = new ArrayList<>();
+
+    public ServiceInstanceMappingSpanListener(ModuleManager moduleManager, TraceServiceModuleConfig config) {
+        this.serviceInstanceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
+        this.serviceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
+        this.serviceInstanceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInstanceInventoryRegister.class);
+        this.networkAddressInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(NetworkAddressInventoryCache.class);
+        this.config = config;
+    }
+
+    @Override
+    public void parseEntry(SpanDecorator spanDecorator, SegmentCoreInfo segmentCoreInfo) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("service instance mapping listener parse reference");
+        }
+        if (!spanDecorator.getSpanLayer().equals(SpanLayer.MQ)) {
+            if (spanDecorator.getRefsCount() > 0) {
+                for (int i = 0; i < spanDecorator.getRefsCount(); i++) {
+                    int networkAddressId = spanDecorator.getRefs(i).getNetworkAddressId();
+                    String address = networkAddressInventoryCache.get(networkAddressId).getName();
+                    int serviceInstanceId = serviceInstanceInventoryCache.getServiceInstanceId(serviceInventoryCache.getServiceId(networkAddressId), networkAddressId);
+
+                    if (config.getUninstrumentedGatewaysConfig().isAddressConfiguredAsGateway(address)) {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("{} is configured as gateway, will reset its mapping service instance id", serviceInstanceId);
+                        }
+                        ServiceInstanceInventory instanceInventory = serviceInstanceInventoryCache.get(serviceInstanceId);
+                        if (instanceInventory.getMappingServiceInstanceId() != Const.NONE && !serviceInstancesToResetMapping.contains(serviceInstanceId)) {
+                            serviceInstancesToResetMapping.add(serviceInstanceId);
+                        }
+                    } else {
+                        ServiceInstanceMapping serviceMapping = new ServiceInstanceMapping();
+                        serviceMapping.setServiceInstanceId(serviceInstanceId);
+                        serviceMapping.setMappingServiceInstanceId(segmentCoreInfo.getServiceInstanceId());
+                        serviceInstanceMappings.add(serviceMapping);
+                    }
+                }
+            }
+        }
+    }
+
+    @Override
+    public void build() {
+        serviceInstanceMappings.forEach(instanceMapping -> {
+            if (logger.isDebugEnabled()) {
+                logger.debug("service instance mapping listener build, service id: {}, mapping service id: {}", instanceMapping.getServiceInstanceId(), instanceMapping.getMappingServiceInstanceId());
+            }
+            serviceInstanceInventoryRegister.updateMapping(instanceMapping.getServiceInstanceId(), instanceMapping.getMappingServiceInstanceId());
+        });
+        serviceInstancesToResetMapping.forEach(instanceId -> {
+            if (logger.isDebugEnabled()) {
+                logger.debug("service instance mapping listener build, reset mapping of service id: {}", instanceId);
+            }
+            serviceInstanceInventoryRegister.resetMapping(instanceId);
+        });
+    }
+
+    @Override
+    public boolean containsPoint(Point point) {
+        return Point.Entry.equals(point);
+    }
+
+    public static class Factory implements SpanListenerFactory {
+
+        @Override
+        public SpanListener create(ModuleManager moduleManager, TraceServiceModuleConfig config) {
+            return new ServiceInstanceMappingSpanListener(moduleManager, config);
+        }
+    }
+
+    @Getter
+    @Setter
+    private static class ServiceInstanceMapping {
+        private int serviceInstanceId;
+        private int mappingServiceInstanceId;
+    }
+}

oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java

@@ -20,18 +20,27 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.parser.standard
 
 import com.google.common.base.Strings;
 import com.google.gson.JsonObject;
-import java.util.List;
 import org.apache.skywalking.apm.network.common.KeyStringValuePair;
 import org.apache.skywalking.apm.network.language.agent.SpanLayer;
-import org.apache.skywalking.oap.server.core.*;
+import org.apache.skywalking.oap.server.core.Const;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
 import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
 import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService;
-import org.apache.skywalking.oap.server.core.register.*;
-import org.apache.skywalking.oap.server.core.register.service.*;
+import org.apache.skywalking.oap.server.core.register.NodeType;
+import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
+import org.apache.skywalking.oap.server.core.register.ServiceInventory;
+import org.apache.skywalking.oap.server.core.register.service.IEndpointInventoryRegister;
+import org.apache.skywalking.oap.server.core.register.service.INetworkAddressInventoryRegister;
+import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister;
+import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister;
 import org.apache.skywalking.oap.server.core.source.DetectPoint;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator.SpanDecorator;
-import org.slf4j.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
 
 /**
  * @author peng-yongsheng
@@ -43,6 +52,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
     private static SpanIdExchanger EXCHANGER;
     private final ServiceInventoryCache serviceInventoryCacheDAO;
     private final IServiceInventoryRegister serviceInventoryRegister;
+    private final ServiceInstanceInventoryCache serviceInstanceInventoryCacheDAO;
+    private final IServiceInstanceInventoryRegister serviceInstanceInventoryRegister;
     private final IEndpointInventoryRegister endpointInventoryRegister;
     private final INetworkAddressInventoryRegister networkAddressInventoryRegister;
     private final IComponentLibraryCatalogService componentLibraryCatalogService;
@@ -57,6 +68,8 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
     private SpanIdExchanger(ModuleManager moduleManager) {
         this.serviceInventoryCacheDAO = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
         this.serviceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInventoryRegister.class);
+        this.serviceInstanceInventoryCacheDAO = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
+        this.serviceInstanceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInstanceInventoryRegister.class);
         this.endpointInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IEndpointInventoryRegister.class);
         this.networkAddressInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(INetworkAddressInventoryRegister.class);
         this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class);
@@ -116,6 +129,9 @@ public class SpanIdExchanger implements IdExchanger<SpanDecorator> {
                 }
             }
             serviceInventoryRegister.update(newServiceInventory.getSequence(), nodeType, properties);
+
+            ServiceInstanceInventory newServiceInstanceInventory = serviceInstanceInventoryCacheDAO.get(serviceInstanceInventoryCacheDAO.getServiceInstanceId(newServiceInventory.getSequence(), peerId));
+            serviceInstanceInventoryRegister.update(newServiceInstanceInventory.getSequence(), nodeType, properties);
         }
 
         if (standardBuilder.getOperationNameId() == Const.NONE) {

oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/transform/SpringSleuthSegmentBuilderTest.java

@@ -96,7 +96,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
                 }
             }
 
-            @Override public int getOrCreate(int serviceId, int addressId, long registerTime) {
+            @Override public int getOrCreate(int serviceId, String serviceInstanceName, int addressId, long registerTime) {
                 String key = "VitualAppCode:" + serviceId + ",getAddress:" + addressId;
                 if (applicationInstRegister.containsKey(key)) {
                     return applicationInstRegister.get(key);
@@ -107,9 +107,24 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
                 }
             }
 
+            @Override
+            public void update(int serviceInstanceId, NodeType nodeType, JsonObject properties) {
+
+            }
+
             @Override public void heartbeat(int serviceInstanceId, long heartBeatTime) {
 
             }
+
+            @Override
+            public void updateMapping(int serviceInstanceId, int mappingServiceInstanceId) {
+
+            }
+
+            @Override
+            public void resetMapping(int serviceInstanceId) {
+
+            }
         };
 
         Whitebox.setInternalState(CoreRegisterLinker.class, "SERVICE_INVENTORY_REGISTER", applicationIDService);

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/StorageModuleElasticsearchProvider.java

@@ -111,7 +111,7 @@ public class StorageModuleElasticsearchProvider extends ModuleProvider {
         this.registerServiceImplementation(IHistoryDeleteDAO.class, new HistoryDeleteEsDAO(getManager(), elasticSearchClient, new ElasticsearchStorageTTL()));
 
         this.registerServiceImplementation(IServiceInventoryCacheDAO.class, new ServiceInventoryCacheEsDAO(elasticSearchClient, config.getResultWindowMaxSize()));
-        this.registerServiceImplementation(IServiceInstanceInventoryCacheDAO.class, new ServiceInstanceInventoryCacheDAO(elasticSearchClient));
+        this.registerServiceImplementation(IServiceInstanceInventoryCacheDAO.class, new ServiceInstanceInventoryCacheDAO(elasticSearchClient, config.getResultWindowMaxSize()));
         this.registerServiceImplementation(IEndpointInventoryCacheDAO.class, new EndpointInventoryCacheEsDAO(elasticSearchClient));
         this.registerServiceImplementation(INetworkAddressInventoryCacheDAO.class, new NetworkAddressInventoryCacheEsDAO(elasticSearchClient, config.getResultWindowMaxSize()));
 

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/cache/ServiceInstanceInventoryCacheDAO.java

@@ -19,16 +19,23 @@
 package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.cache;
 
 import org.apache.skywalking.oap.server.core.Const;
-import org.apache.skywalking.oap.server.core.register.*;
+import org.apache.skywalking.oap.server.core.register.RegisterSource;
+import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
 import org.apache.skywalking.oap.server.core.storage.cache.IServiceInstanceInventoryCacheDAO;
 import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
+import org.apache.skywalking.oap.server.library.util.BooleanUtils;
 import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.slf4j.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * @author peng-yongsheng
@@ -38,9 +45,11 @@ public class ServiceInstanceInventoryCacheDAO extends EsDAO implements IServiceI
     private static final Logger logger = LoggerFactory.getLogger(ServiceInstanceInventoryCacheDAO.class);
 
     protected final ServiceInstanceInventory.Builder builder = new ServiceInstanceInventory.Builder();
+    protected final int resultWindowMaxSize;
 
-    public ServiceInstanceInventoryCacheDAO(ElasticSearchClient client) {
+    public ServiceInstanceInventoryCacheDAO(ElasticSearchClient client, int resultWindowMaxSize) {
         super(client);
+        this.resultWindowMaxSize = resultWindowMaxSize;
     }
 
     @Override public ServiceInstanceInventory get(int serviceInstanceId) {
@@ -72,6 +81,30 @@ public class ServiceInstanceInventoryCacheDAO extends EsDAO implements IServiceI
         return get(id);
     }
 
+    @Override public List<ServiceInstanceInventory> loadLastUpdate(long lastUpdateTime) {
+        List<ServiceInstanceInventory> instanceInventories = new ArrayList<>();
+
+        try {
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+
+            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+            boolQuery.must().add(QueryBuilders.termQuery(ServiceInstanceInventory.IS_ADDRESS, BooleanUtils.TRUE));
+            boolQuery.must().add(QueryBuilders.rangeQuery(ServiceInstanceInventory.LAST_UPDATE_TIME).gte(lastUpdateTime));
+
+            searchSourceBuilder.query(boolQuery);
+            searchSourceBuilder.size(resultWindowMaxSize);
+
+            SearchResponse response = getClient().search(ServiceInstanceInventory.INDEX_NAME, searchSourceBuilder);
+
+            for (SearchHit searchHit : response.getHits().getHits()) {
+                instanceInventories.add(this.builder.map2Data(searchHit.getSourceAsMap()));
+            }
+        } catch (Throwable t) {
+            logger.error(t.getMessage(), t);
+        }
+        return instanceInventories;
+    }
+
     private int get(String id) {
         try {
             GetResponse response = getClient().get(ServiceInstanceInventory.INDEX_NAME, id);

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java

@@ -18,13 +18,14 @@
 
 package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query;
 
-import java.io.IOException;
-import java.util.*;
 import org.apache.skywalking.oap.server.core.UnexpectedException;
 import org.apache.skywalking.oap.server.core.analysis.Downsampling;
 import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
 import org.apache.skywalking.oap.server.core.analysis.manual.endpointrelation.EndpointRelationServerSideMetrics;
-import org.apache.skywalking.oap.server.core.analysis.manual.servicerelation.*;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.query.entity.Call;
 import org.apache.skywalking.oap.server.core.source.DetectPoint;
@@ -34,11 +35,16 @@ import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSear
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
 import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * @author peng-yongsheng
  */
@@ -111,6 +117,48 @@ public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO {
         return load(sourceBuilder, indexName, DetectPoint.CLIENT);
     }
 
+    @Override
+    public List<Call.CallDetail> loadServerSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException {
+        String indexName = ModelName.build(downsampling, ServiceInstanceRelationServerSideMetrics.INDEX_NAME);
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+        sourceBuilder.size(0);
+        setInstanceQueryCondition(sourceBuilder, startTB, endTB, clientServiceId, serverServiceId);
+
+        return load(sourceBuilder, indexName, DetectPoint.SERVER);
+    }
+
+    @Override
+    public List<Call.CallDetail> loadClientSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException {
+        String indexName = ModelName.build(downsampling, ServiceInstanceRelationClientSideMetrics.INDEX_NAME);
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+        sourceBuilder.size(0);
+        setInstanceQueryCondition(sourceBuilder, startTB, endTB, clientServiceId, serverServiceId);
+
+        return load(sourceBuilder, indexName, DetectPoint.CLIENT);
+    }
+
+    private void setInstanceQueryCondition(SearchSourceBuilder sourceBuilder, long startTB, long endTB, int clientServiceId, int serverServiceId) {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        boolQuery.must().add(QueryBuilders.rangeQuery(EndpointRelationServerSideMetrics.TIME_BUCKET).gte(startTB).lte(endTB));
+
+        BoolQueryBuilder serviceIdBoolQuery = new BoolQueryBuilder();
+        boolQuery.must(serviceIdBoolQuery);
+
+        BoolQueryBuilder serverRelationBoolQuery = new BoolQueryBuilder();
+        serviceIdBoolQuery.should(serverRelationBoolQuery);
+
+        serverRelationBoolQuery.must(QueryBuilders.termQuery(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, clientServiceId));
+        serverRelationBoolQuery.must(QueryBuilders.termQuery(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, serverServiceId));
+
+        BoolQueryBuilder clientRelationBoolQuery = new BoolQueryBuilder();
+        serviceIdBoolQuery.should(clientRelationBoolQuery);
+
+        clientRelationBoolQuery.must(QueryBuilders.termQuery(ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, clientServiceId));
+        clientRelationBoolQuery.must(QueryBuilders.termQuery(ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, serverServiceId));
+
+        sourceBuilder.query(boolQuery);
+    }
+
     @Override
     public List<Call.CallDetail> loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, long startTB, long endTB, int destEndpointId) throws IOException {
         String indexName = ModelName.build(downsampling, EndpointRelationServerSideMetrics.INDEX_NAME);

oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/StorageModuleElasticsearch7Provider.java

@@ -113,7 +113,7 @@ public class StorageModuleElasticsearch7Provider extends ModuleProvider {
         this.registerServiceImplementation(IHistoryDeleteDAO.class, new HistoryDeleteEsDAO(getManager(), elasticSearch7Client, new ElasticsearchStorageTTL()));
 
         this.registerServiceImplementation(IServiceInventoryCacheDAO.class, new ServiceInventoryCacheEs7DAO(elasticSearch7Client, config.getResultWindowMaxSize()));
-        this.registerServiceImplementation(IServiceInstanceInventoryCacheDAO.class, new ServiceInstanceInventoryCacheEs7DAO(elasticSearch7Client));
+        this.registerServiceImplementation(IServiceInstanceInventoryCacheDAO.class, new ServiceInstanceInventoryCacheEs7DAO(elasticSearch7Client, config.getResultWindowMaxSize()));
         this.registerServiceImplementation(IEndpointInventoryCacheDAO.class, new EndpointInventoryCacheEs7DAO(elasticSearch7Client));
         this.registerServiceImplementation(INetworkAddressInventoryCacheDAO.class, new NetworkAddressInventoryCacheEs7DAO(elasticSearch7Client, config.getResultWindowMaxSize()));
 

oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/cache/ServiceInstanceInventoryCacheEs7DAO.java

@@ -36,8 +36,8 @@ public class ServiceInstanceInventoryCacheEs7DAO extends ServiceInstanceInventor
 
     private static final Logger logger = LoggerFactory.getLogger(ServiceInstanceInventoryCacheEs7DAO.class);
 
-    public ServiceInstanceInventoryCacheEs7DAO(ElasticSearchClient client) {
-        super(client);
+    public ServiceInstanceInventoryCacheEs7DAO(ElasticSearchClient client, int resultWindowMaxSize) {
+        super(client, resultWindowMaxSize);
     }
 
     @Override

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ServiceInstanceInventoryCacheDAO.java

@@ -19,9 +19,16 @@
 package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao;
 
 import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
 import org.apache.skywalking.oap.server.core.storage.cache.IServiceInstanceInventoryCacheDAO;
 import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
+import org.apache.skywalking.oap.server.library.util.BooleanUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,6 +62,36 @@ public class H2ServiceInstanceInventoryCacheDAO extends H2SQLExecutor implements
         return getByID(id);
     }
 
+    @Override
+    public List<ServiceInstanceInventory> loadLastUpdate(long lastUpdateTime) {
+        List<ServiceInstanceInventory> instanceInventories = new ArrayList<>();
+
+        try {
+            StringBuilder sql = new StringBuilder("select * from ");
+            sql.append(ServiceInstanceInventory.INDEX_NAME);
+            sql.append(" where ").append(ServiceInstanceInventory.IS_ADDRESS).append("=? ");
+            sql.append(" and ").append(ServiceInstanceInventory.LAST_UPDATE_TIME).append(">?");
+
+            try (Connection connection = h2Client.getConnection()) {
+                try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), BooleanUtils.TRUE, lastUpdateTime)) {
+                    ServiceInstanceInventory serviceInstanceInventory;
+                    do {
+                        serviceInstanceInventory = (ServiceInstanceInventory) toStorageData(resultSet, ServiceInstanceInventory.INDEX_NAME, new ServiceInstanceInventory.Builder());
+                        if (serviceInstanceInventory != null) {
+                            instanceInventories.add(serviceInstanceInventory);
+                        }
+                    }
+                    while (serviceInstanceInventory != null);
+                }
+            } catch (SQLException e) {
+                throw new IOException(e);
+            }
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+        }
+        return instanceInventories;
+    }
+
     private int getByID(String id) {
         return getEntityIDByID(h2Client, ServiceInstanceInventory.SEQUENCE, ServiceInstanceInventory.INDEX_NAME, id);
     }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java

@@ -24,7 +24,10 @@ import java.util.*;
 import org.apache.skywalking.oap.server.core.analysis.Downsampling;
 import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
 import org.apache.skywalking.oap.server.core.analysis.manual.endpointrelation.EndpointRelationServerSideMetrics;
-import org.apache.skywalking.oap.server.core.analysis.manual.servicerelation.*;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.query.entity.*;
 import org.apache.skywalking.oap.server.core.source.DetectPoint;
@@ -64,6 +67,24 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO {
         return loadServiceCalls(tableName, startTB, endTB, ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, ServiceRelationServerSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0), true);
     }
 
+    @Override
+    public List<Call.CallDetail> loadServerSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException {
+        String tableName = ModelName.build(downsampling, ServiceInstanceRelationServerSideMetrics.INDEX_NAME);
+        return loadServiceInstanceCalls(tableName, startTB, endTB,
+                ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID,
+                ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID,
+                clientServiceId, serverServiceId, false);
+    }
+
+    @Override
+    public List<Call.CallDetail> loadClientSideServiceInstanceRelations(int clientServiceId, int serverServiceId, Downsampling downsampling, long startTB, long endTB) throws IOException {
+        String tableName = ModelName.build(downsampling, ServiceInstanceRelationClientSideMetrics.INDEX_NAME);
+        return loadServiceInstanceCalls(tableName, startTB, endTB,
+                ServiceInstanceRelationClientSideMetrics.SOURCE_SERVICE_ID,
+                ServiceInstanceRelationClientSideMetrics.DEST_SERVICE_ID,
+                clientServiceId, serverServiceId, true);
+    }
+
     @Override
     public List<Call.CallDetail> loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, long startTB, long endTB, int destEndpointId) throws IOException {
         String tableName = ModelName.build(downsampling, EndpointRelationServerSideMetrics.INDEX_NAME);
@@ -108,6 +129,29 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO {
         return calls;
     }
 
+    private List<Call.CallDetail> loadServiceInstanceCalls(String tableName, long startTB, long endTB, String sourceCName,
+                                                           String descCName, int sourceServiceId, int destServiceId, boolean isClientSide) throws IOException {
+        Object[] conditions = new Object[]{startTB, endTB, sourceServiceId, destServiceId, destServiceId, sourceServiceId};
+        StringBuilder serviceIdMatchSql = new StringBuilder("and ((")
+                .append(sourceCName).append("=? and ").append(descCName).append("=?").append(") or (")
+                .append(sourceCName).append("=? and ").append(descCName).append("=?").append("))");
+        List<Call.CallDetail> calls = new ArrayList<>();
+        try (Connection connection = h2Client.getConnection()) {
+            try (ResultSet resultSet = h2Client.executeQuery(connection, "select "
+                            + Metrics.ENTITY_ID
+                            + " from " + tableName + " where "
+                            + Metrics.TIME_BUCKET + ">= ? and " + Metrics.TIME_BUCKET + "<=? "
+                            + serviceIdMatchSql.toString()
+                            + " group by " + Metrics.ENTITY_ID,
+                    conditions)) {
+                buildCalls(resultSet, calls, isClientSide);
+            }
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+        return calls;
+    }
+
     private List<Call.CallDetail> loadEndpointFromSide(String tableName, long startTB, long endTB, String sourceCName,
         String destCName, int id, boolean isSourceId) throws IOException {
         Object[] conditions = new Object[3];

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java

@@ -29,9 +29,7 @@ import org.apache.skywalking.e2e.service.endpoint.EndpointQuery;
 import org.apache.skywalking.e2e.service.endpoint.Endpoints;
 import org.apache.skywalking.e2e.service.instance.Instances;
 import org.apache.skywalking.e2e.service.instance.InstancesQuery;
-import org.apache.skywalking.e2e.topo.TopoData;
-import org.apache.skywalking.e2e.topo.TopoQuery;
-import org.apache.skywalking.e2e.topo.TopoResponse;
+import org.apache.skywalking.e2e.topo.*;
 import org.apache.skywalking.e2e.trace.Trace;
 import org.apache.skywalking.e2e.trace.TracesData;
 import org.apache.skywalking.e2e.trace.TracesQuery;
@@ -179,6 +177,30 @@ public class SimpleQueryClient {
         return Objects.requireNonNull(responseEntity.getBody()).getData().getTopo();
     }
 
+    public ServiceInstanceTopoData serviceInstanceTopo(final ServiceInstanceTopoQuery query) throws Exception {
+        final URL queryFileUrl = Resources.getResource("instanceTopo.gql");
+        final String queryString = Resources.readLines(queryFileUrl, Charset.forName("UTF8"))
+                .stream()
+                .filter(it -> !it.startsWith("#"))
+                .collect(Collectors.joining())
+                .replace("{step}", query.step())
+                .replace("{start}", query.start())
+                .replace("{end}", query.end())
+                .replace("{clientServiceId}", query.clientServiceId())
+                .replace("{serverServiceId}", query.serverServiceId());
+        final ResponseEntity<GQLResponse<ServiceInstanceTopoResponse>> responseEntity = restTemplate.exchange(
+                new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),
+                new ParameterizedTypeReference<GQLResponse<ServiceInstanceTopoResponse>>() {
+                }
+        );
+
+        if (responseEntity.getStatusCode() != HttpStatus.OK) {
+            throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode());
+        }
+
+        return Objects.requireNonNull(responseEntity.getBody()).getData().getTopo();
+    }
+
     public Metrics metrics(final MetricsQuery query) throws Exception {
         final URL queryFileUrl = Resources.getResource("metrics.gql");
         final String queryString = Resources.readLines(queryFileUrl, Charset.forName("UTF8"))

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/assertor/VariableExpressParser.java

@@ -31,18 +31,18 @@ import static java.util.Objects.isNull;
 public class VariableExpressParser {
 
 
-    public static <T> T parse(String express, List<T> actual, Function<T, String> getFiled) {
-        express = express.trim();
-        if (!(express.startsWith("${") && express.endsWith("}"))) {
+    public static <T> T parse(final String express, List<T> actual, Function<T, String> getFiled) {
+        String variable = express.trim();
+        if (!(variable.startsWith("${") && variable.endsWith("}"))) {
             return null;
         }
 
-        express = express.substring(2, express.length() - 1);
+        variable = variable.substring(2, variable.length() - 1);
 
-        int startIndexOfIndex = express.lastIndexOf("[");
-        String regex = express.substring(0, startIndexOfIndex);
-        int endIndexOfIndex = express.indexOf("]", startIndexOfIndex);
-        int expectedIndex = Integer.parseInt(express.substring(startIndexOfIndex + 1, endIndexOfIndex));
+        int startIndexOfIndex = variable.lastIndexOf("[");
+        String regex = variable.substring(0, startIndexOfIndex);
+        int endIndexOfIndex = variable.indexOf("]", startIndexOfIndex);
+        int expectedIndex = Integer.parseInt(variable.substring(startIndexOfIndex + 1, endIndexOfIndex));
         int mappingIndex = 0;
 
         T mapping = null;

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceNode.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+@Accessors(chain = true)
+@ToString
+public class ServiceInstanceNode {
+
+    private String id;
+    private String name;
+    private String serviceId;
+    private String serviceName;
+    private String type;
+    private String isReal;
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceNodeMatcher.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+import static java.util.Objects.nonNull;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+@ToString
+public class ServiceInstanceNodeMatcher extends AbstractMatcher<ServiceInstanceNode> {
+
+    private String id;
+    private String name;
+    private String serviceId;
+    private String serviceName;
+    private String type;
+    private String isReal;
+
+    @Override
+    public void verify(ServiceInstanceNode node) {
+        if (nonNull(getId())) {
+            final String expected = this.getId();
+            final String actual = node.getId();
+
+            doVerify(expected, actual);
+        }
+
+        if (nonNull(getName())) {
+            final String expected = this.getName();
+            final String actual = node.getName();
+
+            doVerify(expected, actual);
+        }
+
+        if (nonNull(getServiceId())) {
+            final String expected = this.getServiceId();
+            final String actual = node.getServiceId();
+
+            doVerify(expected, actual);
+        }
+
+        if (nonNull(getServiceName())) {
+            final String expected = this.getServiceName();
+            final String actual = node.getServiceName();
+
+            doVerify(expected, actual);
+        }
+
+        if (nonNull(getType())) {
+            final String expected = this.getType();
+            final String actual = node.getType();
+
+            doVerify(expected, actual);
+        }
+
+        if (nonNull(getIsReal())) {
+            final String expected = this.getIsReal();
+            final String actual = String.valueOf(node.getIsReal());
+
+            doVerify(expected, actual);
+        }
+    }
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceTopoData.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+@Accessors(chain = true)
+@ToString
+public class ServiceInstanceTopoData {
+
+    private List<ServiceInstanceNode> nodes;
+    private List<Call> calls;
+
+    public ServiceInstanceTopoData() {
+        this.nodes = new ArrayList<>();
+        this.calls = new ArrayList<>();
+    }
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceTopoMatcher.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.apache.skywalking.e2e.assertor.VariableExpressParser;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+import java.util.List;
+
+import static java.util.Objects.nonNull;
+import static org.assertj.core.api.Assertions.fail;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+@ToString
+public class ServiceInstanceTopoMatcher extends AbstractMatcher<ServiceInstanceTopoData> {
+
+    private List<ServiceInstanceNodeMatcher> nodes;
+    private List<CallMatcher> calls;
+
+    @Override
+    public void verify(ServiceInstanceTopoData topoData) {
+        if (nonNull(getNodes())) {
+            verifyNodes(topoData);
+        }
+
+        if (nonNull(getCalls())) {
+            convertNodeId(getCalls(), topoData.getNodes());
+            verifyCalls(topoData);
+        }
+    }
+
+    private void verifyNodes(ServiceInstanceTopoData topoData) {
+        for (int i = 0; i < getNodes().size(); i++) {
+            boolean matched = false;
+            for (int j = 0; j < topoData.getNodes().size(); j++) {
+                try {
+                    getNodes().get(i).verify(topoData.getNodes().get(j));
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\nActual: %s", getNodes(), topoData.getNodes());
+            }
+        }
+    }
+
+    private void verifyCalls(ServiceInstanceTopoData topoData) {
+        for (int i = 0; i < getCalls().size(); i++) {
+            boolean matched = false;
+            for (int j = 0; j < topoData.getCalls().size(); j++) {
+                try {
+                    getCalls().get(i).verify(topoData.getCalls().get(j));
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\nActual: %s", getCalls(), topoData.getCalls());
+            }
+        }
+    }
+
+    private static void convertNodeId(List<CallMatcher> callMatchers, List<ServiceInstanceNode> nodes) {
+        for (CallMatcher callMatcher : callMatchers) {
+            ServiceInstanceNode sourceNode = VariableExpressParser.parse(callMatcher.getSource(), nodes, ServiceInstanceNode::getName);
+            ServiceInstanceNode targetNode = VariableExpressParser.parse(callMatcher.getTarget(), nodes, ServiceInstanceNode::getName);
+
+            boolean convert = false;
+            if (nonNull(sourceNode)) {
+                callMatcher.setSource(sourceNode.getId());
+                convert = true;
+            }
+            if (nonNull(targetNode)) {
+                callMatcher.setTarget(targetNode.getId());
+                convert = true;
+            }
+
+            if (convert) {
+                callMatcher.setId(String.join("_", callMatcher.getSource(), callMatcher.getTarget()));
+            }
+        }
+    }
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceTopoQuery.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import org.apache.skywalking.e2e.AbstractQuery;
+
+/**
+ * @author zhangwei
+ */
+public class ServiceInstanceTopoQuery extends AbstractQuery<ServiceInstanceTopoQuery> {
+
+    private String clientServiceId;
+    private String serverServiceId;
+
+    public ServiceInstanceTopoQuery clientServiceId(String clientServiceId) {
+        this.clientServiceId = clientServiceId;
+        return this;
+    }
+
+    public ServiceInstanceTopoQuery serverServiceId(String serverServiceId) {
+        this.serverServiceId = serverServiceId;
+        return this;
+    }
+
+    public String clientServiceId() {
+        return clientServiceId;
+    }
+
+    public String serverServiceId() {
+        return serverServiceId;
+    }
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/ServiceInstanceTopoResponse.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.topo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @author zhangwei
+ */
+@Setter
+@Getter
+public class ServiceInstanceTopoResponse {
+
+    private ServiceInstanceTopoData topo;
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/TopoData.java

@@ -50,4 +50,12 @@ public class TopoData {
         this.calls = calls;
         return this;
     }
+
+    @Override
+    public String toString() {
+        return "TopoData{" +
+                "nodes=" + nodes +
+                ", calls=" + calls +
+                '}';
+    }
 }

test/e2e/e2e-base/src/main/resources/instanceTopo.gql

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+    "query": "query queryInstanceTopo($clientServiceId: ID!, $serverServiceId: ID!, $duration: Duration!) {
+        topo: getServiceInstanceTopology(clientServiceId: $clientServiceId, serverServiceId: $serverServiceId, duration: $duration) {
+            nodes {
+                id
+                name
+                serviceId
+                serviceName
+                type
+                isReal
+            }
+            calls {
+                id
+                source
+                detectPoints
+                target
+            }
+        }
+    }",
+    "variables": {
+        "duration": {
+            "start": "{start}",
+            "end": "{end}",
+            "step": "{step}"
+        },
+        "clientServiceId": "{clientServiceId}",
+        "serverServiceId": "{serverServiceId}"
+    }
+}

test/e2e/e2e-base/src/test/java/org/apache/skywalking/e2e/TestServiceInstanceTopoMatcher.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.skywalking.e2e;
+
+import org.apache.skywalking.e2e.assertor.exception.VariableNotFoundException;
+import org.apache.skywalking.e2e.topo.*;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.core.io.ClassPathResource;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author zhangwei
+ */
+public class TestServiceInstanceTopoMatcher {
+
+    private ServiceInstanceTopoMatcher topoMatcher;
+
+    @Before
+    public void setUp() throws IOException {
+        try (InputStream expectedInputStream = new ClassPathResource("serviceInstanceTopo.yml").getInputStream()) {
+            topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
+        }
+    }
+
+    @Test
+    public void shouldSuccess() {
+        final List<ServiceInstanceNode> nodes = new ArrayList<>();
+        nodes.add(new ServiceInstanceNode().setId("2").setName("e2e-cluster-provider-pid:1582@2ffd0ee4eeb1").setServiceId("2").setServiceName("e2e-cluster-provider").setType("Tomcat").setIsReal("true"));
+        nodes.add(new ServiceInstanceNode().setId("3").setName("e2e-cluster-provider-pid:1583@2ffd0ee4eeb1").setServiceId("2").setServiceName("e2e-cluster-provider").setType("Tomcat").setIsReal("true"));
+        nodes.add(new ServiceInstanceNode().setId("4").setName("e2e-cluster-consumer-pid:1591@2ffd0ee4eeb1").setServiceId("3").setServiceName("e2e-cluster-consumer").setIsReal("true"));
+
+
+        final List<Call> calls = new ArrayList<>();
+        calls.add(new Call().setId("4_3").setSource("4").setTarget("3"));
+        calls.add(new Call().setId("4_2").setSource("4").setTarget("2"));
+
+        final ServiceInstanceTopoData topoData = new ServiceInstanceTopoData().setNodes(nodes).setCalls(calls);
+
+        topoMatcher.verify(topoData);
+    }
+
+    @Test(expected = VariableNotFoundException.class)
+    public void shouldVariableNotFound() {
+        final List<ServiceInstanceNode> nodes = new ArrayList<>();
+        nodes.add(new ServiceInstanceNode().setId("2").setName("e2e-cluster-provider-pid:1582@2ffd0ee4eeb1").setServiceId("2").setServiceName("e2e-cluster-provider").setType("Tomcat").setIsReal("true"));
+        nodes.add(new ServiceInstanceNode().setId("3").setName("e2e-cluster-Aprovider-pid:1583@2ffd0ee4eeb1").setServiceId("2").setServiceName("e2e-cluster-provider").setType("Tomcat").setIsReal("true"));
+        nodes.add(new ServiceInstanceNode().setId("4").setName("e2e-cluster-consumer-pid:1591@2ffd0ee4eeb1").setServiceId("3").setServiceName("e2e-cluster-consumer").setIsReal("true"));
+
+
+        final List<Call> calls = new ArrayList<>();
+        calls.add(new Call().setId("4_3").setSource("4").setTarget("3"));
+        calls.add(new Call().setId("4_2").setSource("4").setTarget("2"));
+
+        final ServiceInstanceTopoData topoData = new ServiceInstanceTopoData().setNodes(nodes).setCalls(calls);
+
+        topoMatcher.verify(topoData);
+    }
+}

test/e2e/e2e-base/src/test/resources/serviceInstanceTopo.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+nodes:
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-consumer
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-provider
+    type: Tomcat
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-provider
+    type: Tomcat
+    isReal: true
+calls:
+  - id: not null
+    source: ${e2e-cluster-consumer(.*)[0]}
+    target: ${e2e-cluster-provider(.*)[0]}
+  - id: not null
+    source: ${e2e-cluster-consumer(.*)[0]}
+    target: ${e2e-cluster-provider(.*)[1]}

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-consumer/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster-with-gateway</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+
+    <artifactId>e2e-cluster-with-gateway-consumer</artifactId>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring.boot.version}</version>
+                <configuration>
+                    <executable>true</executable>
+                    <addResources>true</addResources>
+                    <excludeDevtools>true</excludeDevtools>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-consumer/src/main/java/org/apache/skywalking/e2e/cluster/TestController.java

@@ -42,9 +42,12 @@ public class TestController {
     @PostMapping("/users")
     public User createAuthor(@RequestBody final User user) throws InterruptedException {
         Thread.sleep(1000L);
-        final ResponseEntity<User> response = restTemplate.postForEntity(
-            "http://127.0.0.1:9099/e2e/users", user, User.class
-        );
+        ResponseEntity<User> response = null;
+        for (int i = 0; i < 2; i++) {
+            response = restTemplate.postForEntity(
+                    "http://127.0.0.1:9099/e2e/users", user, User.class
+            );
+        }
         return response.getBody();
     }
 }

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-consumer/src/main/resources/application.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 server:
-  port: 9091
+  port: 9092
 
 spring:
   main:

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-gateway/pom.xml

@@ -20,13 +20,13 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>e2e-cluster</artifactId>
+        <artifactId>e2e-cluster-with-gateway</artifactId>
         <groupId>org.apache.skywalking</groupId>
         <version>1.0.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>gateway</artifactId>
+    <artifactId>e2e-cluster-with-gateway-gateway</artifactId>
 
     <dependencies>
         <dependency>

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-gateway/src/main/resources/application.yml

@@ -24,5 +24,13 @@ zuul:
   ignoredServices: '*'
   routes:
     api:
+      serviceId: api
       path: /e2e/users
-      url: http://127.0.0.1:9090
+
+ribbon:
+  eureka:
+    enabled: false
+
+api:
+  ribbon:
+    listOfServers: http://127.0.0.1:9090,http://127.0.0.1:9091

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-provider/pom.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster-with-gateway</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+
+    <artifactId>e2e-cluster-with-gateway-provider</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+            <version>${spring.boot.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>${h2.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring.boot.version}</version>
+                <configuration>
+                    <executable>true</executable>
+                    <addResources>true</addResources>
+                    <excludeDevtools>true</excludeDevtools>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/pom.xml

@@ -0,0 +1,220 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster-with-gateway</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>e2e-cluster-with-gateway-test-runner</artifactId>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>e2e-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>e2e-cluster-with-gateway-gateway</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>e2e-cluster-with-gateway-provider</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>e2e-cluster-with-gateway-consumer</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+    </dependencies>
+
+    <properties>
+        <provider.name>e2e-cluster-with-gateway-provider</provider.name>
+        <consumer.name>e2e-cluster-with-gateway-consumer</consumer.name>
+        <gateway.name>e2e-cluster-with-gateway-gateway</gateway.name>
+        <e2e.container.version>1.1</e2e.container.version>
+        <e2e.container.name.prefix>skywalking-e2e-container-${build.id}-cluster-with-gateway</e2e.container.name.prefix>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.fabric8</groupId>
+                <artifactId>docker-maven-plugin</artifactId>
+                <configuration>
+                    <containerNamePattern>%a-%t-%i</containerNamePattern>
+                    <imagePullPolicy>Always</imagePullPolicy>
+                    <images>
+                        <image>
+                            <name>elastic/elasticsearch:${elasticsearch.version}</name>
+                            <alias>${e2e.container.name.prefix}-elasticsearch</alias>
+                            <run>
+                                <ports>
+                                    <port>es.port:9200</port>
+                                </ports>
+                                <wait>
+                                    <http>
+                                        <url>http://localhost:${es.port}</url>
+                                        <method>GET</method>
+                                        <status>200</status>
+                                    </http>
+                                    <time>120000</time>
+                                </wait>
+                                <env>
+                                    <discovery.type>single-node</discovery.type>
+                                </env>
+                            </run>
+                        </image>
+                        <image>
+                            <name>zookeeper:${zookeeper.image.version}</name>
+                            <alias>${e2e.container.name.prefix}-zookeeper</alias>
+                            <run>
+                                <ports>
+                                    <port>zk.port:2181</port>
+                                </ports>
+                                <wait>
+                                    <log>binding to port</log>
+                                    <time>120000</time>
+                                </wait>
+                            </run>
+                        </image>
+                        <image>
+                            <name>skyapm/e2e-container:${e2e.container.version}</name>
+                            <alias>${e2e.container.name.prefix}</alias>
+                            <run>
+                                <env>
+                                    <MODE>cluster</MODE>
+                                    <ES_VERSION>${elasticsearch.version}</ES_VERSION>
+                                    <SW_STORAGE_ES_CLUSTER_NODES>
+                                        ${e2e.container.name.prefix}-elasticsearch:9200
+                                    </SW_STORAGE_ES_CLUSTER_NODES>
+                                    <SW_CLUSTER_ZK_HOST_PORT>
+                                        ${e2e.container.name.prefix}-zookeeper:2181
+                                    </SW_CLUSTER_ZK_HOST_PORT>
+
+                                    <INSTRUMENTED_SERVICE_1>
+                                        ${provider.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_1>
+                                    <INSTRUMENTED_SERVICE_1_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800
+                                        -DSW_AGENT_NAME=${provider.name}
+                                        -Dserver.port=9090
+                                    </INSTRUMENTED_SERVICE_1_OPTS>
+
+                                    <INSTRUMENTED_SERVICE_2>
+                                        ${provider.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_2>
+                                    <INSTRUMENTED_SERVICE_2_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801
+                                        -DSW_AGENT_NAME=${provider.name}
+                                        -Dserver.port=9091
+                                    </INSTRUMENTED_SERVICE_2_OPTS>
+
+                                    <INSTRUMENTED_SERVICE_3>
+                                        ${consumer.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_3>
+                                    <INSTRUMENTED_SERVICE_3_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801
+                                        -DSW_AGENT_NAME=${consumer.name}
+                                        -Dserver.port=9092
+                                    </INSTRUMENTED_SERVICE_3_OPTS>
+                                </env>
+                                <dependsOn>
+                                    <container>${e2e.container.name.prefix}-elasticsearch</container>
+                                    <container>${e2e.container.name.prefix}-zookeeper</container>
+                                </dependsOn>
+                                <ports>
+                                    <port>+webapp.host:webapp.port:8081</port>
+                                    <port>+service.host:service.port:9092</port>
+                                </ports>
+                                <links>
+                                    <link>${e2e.container.name.prefix}-elasticsearch</link>
+                                    <link>${e2e.container.name.prefix}-zookeeper</link>
+                                </links>
+                                <volumes>
+                                    <bind>
+                                        <volume>${sw.home}:/sw</volume>
+                                        <volume>
+                                            ../${gateway.name}/target/${gateway.name}-${project.version}.jar:/home/${gateway.name}-${project.version}.jar
+                                        </volume>
+                                        <volume>
+                                            ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar
+                                        </volume>
+                                        <volume>
+                                            ../${consumer.name}/target/${consumer.name}-${project.version}.jar:/home/${consumer.name}-${project.version}.jar
+                                        </volume>
+                                        <volume>
+                                            ${project.basedir}/src/docker/rc.d:/rc.d:ro
+                                        </volume>
+                                        <volume>
+                                            ${project.basedir}/src/docker/clusterize.awk:/clusterize.awk
+                                        </volume>
+                                    </bind>
+                                </volumes>
+                                <wait>
+                                    <log>SkyWalking e2e container is ready for tests</log>
+                                    <time>3000000</time>
+                                </wait>
+                            </run>
+                        </image>
+                    </images>
+                </configuration>
+            </plugin>
+
+            <!-- set the system properties that can be used in test codes -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <configuration>
+                    <systemPropertyVariables>
+                        <sw.webapp.host>${webapp.host}</sw.webapp.host>
+                        <sw.webapp.port>${webapp.port}</sw.webapp.port>
+                        <service.host>${service.host}</service.host>
+                        <service.port>${service.port}</service.port>
+                        <provider.name>${provider.name}</provider.name>
+                        <consumer.name>${consumer.name}</consumer.name>
+                        <gateway.name>127.0.0.1:9099</gateway.name>
+                    </systemPropertyVariables>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/docker/rc.d/rc1-startup.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Licensed to the SkyAPM under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo 'starting gateway service...' \
+    && java -jar /home/e2e-cluster-with-gateway-gateway-1.0.0.jar 2>&1 > /tmp/gateway.log &
+
+check_tcp 127.0.0.1 \
+          9099 \
+          60 \
+          10 \
+          'waiting for the gateway service to be ready'
+
+if [[ $? -ne 0 ]]; then
+    echo "gateway service failed to start in 60 * 10 seconds: "
+    cat /tmp/gateway.log
+    exit 1
+fi
+
+echo 'starting OAP server...' \
+    && SW_STORAGE_ES_BULK_ACTIONS=1 \
+    SW_STORAGE_ES_FLUSH_INTERVAL=1 \
+    SW_RECEIVER_BUFFER_PATH=/tmp/oap/trace_buffer1 \
+    SW_SERVICE_MESH_BUFFER_PATH=/tmp/oap/mesh_buffer1 \
+    start_oap 'init'
+
+echo 'starting Web app...' \
+    && start_webapp '0.0.0.0' 8081
+
+if test "${MODE}" = "cluster"; then
+    # start another OAP server in a different port
+    echo 'starting OAP server...' \
+        && SW_CORE_GRPC_PORT=11801 \
+        SW_CORE_REST_PORT=12801 \
+        SW_STORAGE_ES_BULK_ACTIONS=1 \
+        SW_STORAGE_ES_FLUSH_INTERVAL=1 \
+        SW_RECEIVER_BUFFER_PATH=/tmp/oap/trace_buffer2 \
+        SW_SERVICE_MESH_BUFFER_PATH=/tmp/oap/mesh_buffer2 \
+        start_oap 'no-init'
+fi
+
+echo 'starting instrumented services...' && start_instrumented_services
+
+check_tcp 127.0.0.1 \
+          9090 \
+          60 \
+          10 \
+          "waiting for the instrumented service 0 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 0 failed to start in 30 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+check_tcp 127.0.0.1 \
+          9091 \
+          60 \
+          10 \
+          "waiting for the instrumented service 1 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 1 failed to start in 24 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+check_tcp 127.0.0.1 \
+          9092 \
+          60 \
+          10 \
+          "waiting for the instrumented service 2 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 2 failed to start in 24 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+echo "SkyWalking e2e container is ready for tests"
+
+tail -f ${OAP_LOG_DIR}/* \
+        ${WEBAPP_LOG_DIR}/* \
+        ${SERVICE_LOG}/* \
+        ${ES_HOME}/logs/elasticsearch.log \
+        ${ES_HOME}/logs/stdout.log

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java

@@ -33,9 +33,7 @@ import org.apache.skywalking.e2e.service.instance.Instance;
 import org.apache.skywalking.e2e.service.instance.Instances;
 import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
 import org.apache.skywalking.e2e.service.instance.InstancesQuery;
-import org.apache.skywalking.e2e.topo.TopoData;
-import org.apache.skywalking.e2e.topo.TopoMatcher;
-import org.apache.skywalking.e2e.topo.TopoQuery;
+import org.apache.skywalking.e2e.topo.*;
 import org.apache.skywalking.e2e.trace.Trace;
 import org.apache.skywalking.e2e.trace.TracesMatcher;
 import org.apache.skywalking.e2e.trace.TracesQuery;
@@ -61,6 +59,7 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
+import static org.assertj.core.api.Assertions.fail;
 
 /**
  * @author kezhenxu94
@@ -76,8 +75,15 @@ public class ClusterVerificationITCase {
     private String instrumentedServiceUrl;
     private long retryInterval = TimeUnit.SECONDS.toMillis(30);
 
+    private String providerName;
+    private String gateWayName;
+
+
     @Before
     public void setUp() {
+        providerName = System.getProperty("provider.name", "e2e-cluster-provider");
+        gateWayName = System.getProperty("gateway.name", "127.0.0.1:9099");
+
         final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1");
         final String swWebappPort = System.getProperty("sw.webapp.port", "32791");
         final String instrumentedServiceHost = System.getProperty("service.host", "127.0.0.1");
@@ -114,6 +120,7 @@ public class ClusterVerificationITCase {
     }
 
     private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
+        String clientServiceId = null, serverServiceId = null;
         boolean valid = false;
         while (!valid) {
             try {
@@ -131,12 +138,51 @@ public class ClusterVerificationITCase {
                 final TopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, TopoMatcher.class);
                 topoMatcher.verify(topoData);
                 valid = true;
+                for (Node node : topoData.getNodes()) {
+                    if (gateWayName.equals(node.getName())) {
+                        clientServiceId = node.getId();
+                    } else if (providerName.equals(node.getName())) {
+                        serverServiceId = node.getId();
+                    }
+                }
             } catch (Throwable t) {
                 LOGGER.warn(t.getMessage(), t);
                 generateTraffic();
                 Thread.sleep(retryInterval);
             }
         }
+        verifyServiceInstanceTopo(minutesAgo, clientServiceId, serverServiceId);
+    }
+
+    private void verifyServiceInstanceTopo(LocalDateTime minutesAgo, String clientServiceId, String serverServiceId) throws Exception {
+        if (clientServiceId == null || serverServiceId == null) {
+            fail("clientService or serverService not found");
+        }
+        boolean valid = false;
+        while (!valid) {
+            try {
+                final ServiceInstanceTopoData topoData = queryClient.serviceInstanceTopo(
+                        new ServiceInstanceTopoQuery()
+                                .stepByMinute()
+                                .start(minutesAgo.minusDays(1))
+                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                                .clientServiceId(clientServiceId)
+                                .serverServiceId(serverServiceId)
+                );
+
+                LOGGER.info("Actual service instance topology: {}", topoData);
+
+                InputStream expectedInputStream =
+                        new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml").getInputStream();
+                final ServiceInstanceTopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
+                topoMatcher.verify(topoData);
+                valid = true;
+            }catch (Throwable t){
+                LOGGER.warn(t.getMessage(), t);
+                generateTraffic();
+                Thread.sleep(retryInterval);
+            }
+        }
     }
 
     private void verifyServices(LocalDateTime minutesAgo) throws Exception {
@@ -194,8 +240,16 @@ public class ClusterVerificationITCase {
                     .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
             );
         }
-        InputStream expectedInputStream =
-            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml").getInputStream();
+
+        InputStream expectedInputStream;
+        if (providerName.equals(service.getLabel())) {
+            expectedInputStream =
+                    new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.providerInstances.yml").getInputStream();
+        } else {
+            expectedInputStream =
+                    new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml").getInputStream();
+        }
+
         final InstancesMatcher instancesMatcher = yaml.loadAs(expectedInputStream, InstancesMatcher.class);
         instancesMatcher.verify(instances);
         return instances;

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.providerInstances.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+instances:
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+nodes:
+  - id: not null
+    name: 127.0.0.1:9099
+    type: Unknown
+    serviceId: not null
+    serviceName: 127.0.0.1:9099
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-with-gateway-provider
+    type: Tomcat
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-with-gateway-provider
+    type: Tomcat
+    isReal: true
+calls:
+  - id: not null
+    source: ${127.0.0.1:9099[0]}
+    target: ${e2e-cluster-with-gateway-provider(.*)[0]}
+  - id: not null
+    source: ${127.0.0.1:9099[0]}
+    target: ${e2e-cluster-with-gateway-provider(.*)[1]}

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.services.yml

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+services:
+  - key: not null
+    label: e2e-cluster-with-gateway-provider
+
+  - key: not null
+    label: e2e-cluster-with-gateway-consumer

test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.topo.yml

@@ -19,10 +19,10 @@ nodes:
     name: User
     type: USER
   - id: not null
-    name: provider
+    name: e2e-cluster-with-gateway-provider
     type: Tomcat
   - id: not null
-    name: consumer
+    name: e2e-cluster-with-gateway-consumer
     type: Tomcat
   - id: not null
     name: "localhost:-1"
@@ -33,13 +33,13 @@ nodes:
 calls:
   - id: not null
     source: ${User[0]}
-    target: ${consumer[0]}
+    target: ${e2e-cluster-with-gateway-consumer[0]}
   - id: not null
     source: ${127.0.0.1:9099[0]}
-    target: ${provider[0]}
+    target: ${e2e-cluster-with-gateway-provider[0]}
   - id: not null
-    source: ${consumer[0]}
+    source: ${e2e-cluster-with-gateway-consumer[0]}
     target: ${127.0.0.1:9099[0]}
   - id: not null
-    source: ${provider[0]}
+    source: ${e2e-cluster-with-gateway-provider[0]}
     target: ${localhost:-1[0]}

test/e2e/e2e-cluster-with-gateway/pom.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>apache-skywalking-e2e</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>e2e-cluster-with-gateway</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>e2e-cluster-with-gateway-provider</module>
+        <module>e2e-cluster-with-gateway-consumer</module>
+        <module>e2e-cluster-with-gateway-gateway</module>
+        <module>e2e-cluster-with-gateway-test-runner</module>
+    </modules>
+</project>

test/e2e/e2e-cluster/e2e-cluster-consumer/pom.xml

@@ -29,7 +29,7 @@
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>
 
-    <artifactId>consumer</artifactId>
+    <artifactId>e2e-cluster-consumer</artifactId>
 
     <build>
         <plugins>

test/e2e/e2e-cluster/e2e-cluster-consumer/src/main/java/org/apache/skywalking/e2e/cluster/Service1Application.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ * @author kezhenxu94
+ */
+@SpringBootApplication
+public class Service1Application {
+    public static void main(String[] args) {
+        SpringApplication.run(Service1Application.class, args);
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-consumer/src/main/java/org/apache/skywalking/e2e/cluster/TestController.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.client.RestTemplate;
+
+/**
+ * @author kezhenxu94
+ */
+@RestController
+@RequestMapping("/e2e")
+public class TestController {
+    private final RestTemplate restTemplate = new RestTemplate();
+
+    private static final String[] URL = {"http://127.0.0.1:9090/e2e/users", "http://127.0.0.1:9091/e2e/users"};
+
+    @GetMapping("/health-check")
+    public String hello() {
+        return "healthy";
+    }
+
+    @PostMapping("/users")
+    public User createAuthor(@RequestBody final User user) throws InterruptedException {
+        Thread.sleep(1000L);
+        ResponseEntity<User> response = null;
+        for (String url : URL) {
+            response = restTemplate.postForEntity(
+                    url, user, User.class
+            );
+        }
+        return response.getBody();
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-consumer/src/main/java/org/apache/skywalking/e2e/cluster/User.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+/**
+ * @author kezhenxu94
+ */
+public class User {
+    public User() {
+    }
+
+    private Long id;
+
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(final Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(final String name) {
+        this.name = name;
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-consumer/src/main/resources/application.yml

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  port: 9092
+
+spring:
+  main:
+    banner-mode: 'off'

test/e2e/e2e-cluster/e2e-cluster-provider/pom.xml

@@ -29,7 +29,7 @@
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>
 
-    <artifactId>provider</artifactId>
+    <artifactId>e2e-cluster-provider</artifactId>
 
     <dependencies>
         <dependency>

test/e2e/e2e-cluster/e2e-cluster-provider/src/main/java/org/apache/skywalking/e2e/cluster/Service0Application.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+
+/**
+ * @author kezhenxu94
+ */
+@EnableJpaRepositories
+@SpringBootApplication
+public class Service0Application {
+    public static void main(String[] args) {
+        SpringApplication.run(Service0Application.class, args);
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-provider/src/main/java/org/apache/skywalking/e2e/cluster/TestController.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @author kezhenxu94
+ */
+@RestController
+@RequestMapping("/e2e")
+public class TestController {
+    private final UserRepo userRepo;
+
+    public TestController(final UserRepo userRepo) {
+        this.userRepo = userRepo;
+    }
+
+    @GetMapping("/health-check")
+    public String hello() {
+        return "healthy";
+    }
+
+    @PostMapping("/users")
+    public User createAuthor(@RequestBody final User user) {
+        return userRepo.save(user);
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-provider/src/main/java/org/apache/skywalking/e2e/cluster/User.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+/**
+ * @author kezhenxu94
+ */
+@Entity
+public class User {
+    public User() {
+    }
+
+    @Id
+    @GeneratedValue
+    private Long id;
+
+    @Column
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(final Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(final String name) {
+        this.name = name;
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-provider/src/main/java/org/apache/skywalking/e2e/cluster/UserRepo.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+/**
+ * @author kezhenxu94
+ */
+public interface UserRepo extends JpaRepository<User, Long> {
+}

test/e2e/e2e-cluster/e2e-cluster-provider/src/main/resources/application.yml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  port: 9090
+
+spring:
+  main:
+    banner-mode: 'off'
+  datasource:
+    url: jdbc:h2:mem:testdb
+    driver-class-name: org.h2.Driver
+    data-username: sa
+    password: sa
+    platform: org.hibernate.dialect.H2Dialect
+  jpa:
+    generate-ddl: true
+    hibernate:
+      ddl-auto: create-drop
+    properties:
+      hibernate.format_sql: true
+    show-sql: true

test/e2e/e2e-cluster/e2e-cluster-test-runner/pom.xml

@@ -28,7 +28,7 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>test-runner</artifactId>
+    <artifactId>e2e-cluster-test-runner</artifactId>
 
     <dependencies>
 
@@ -40,28 +40,21 @@
 
         <dependency>
             <groupId>org.apache.skywalking</groupId>
-            <artifactId>gateway</artifactId>
+            <artifactId>e2e-cluster-provider</artifactId>
             <version>${project.version}</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.skywalking</groupId>
-            <artifactId>provider</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.skywalking</groupId>
-            <artifactId>consumer</artifactId>
+            <artifactId>e2e-cluster-consumer</artifactId>
             <version>${project.version}</version>
         </dependency>
 
     </dependencies>
 
     <properties>
-        <provider.name>provider</provider.name>
-        <consumer.name>consumer</consumer.name>
-        <gateway.name>gateway</gateway.name>
+        <provider.name>e2e-cluster-provider</provider.name>
+        <consumer.name>e2e-cluster-consumer</consumer.name>
         <e2e.container.version>1.1</e2e.container.version>
         <e2e.container.name.prefix>skywalking-e2e-container-${build.id}-cluster</e2e.container.name.prefix>
     </properties>
@@ -128,21 +121,26 @@
                                     <INSTRUMENTED_SERVICE_1_OPTS>
                                         -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800
                                         -DSW_AGENT_NAME=${provider.name}
+                                        -Dserver.port=9090
                                     </INSTRUMENTED_SERVICE_1_OPTS>
-                                    <INSTRUMENTED_SERVICE_1_ARGS>
-                                        --server.port=9090
-                                    </INSTRUMENTED_SERVICE_1_ARGS>
 
                                     <INSTRUMENTED_SERVICE_2>
-                                        ${consumer.name}-${project.version}.jar
+                                        ${provider.name}-${project.version}.jar
                                     </INSTRUMENTED_SERVICE_2>
                                     <INSTRUMENTED_SERVICE_2_OPTS>
                                         -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801
-                                        -DSW_AGENT_NAME=${consumer.name}
+                                        -DSW_AGENT_NAME=${provider.name}
+                                        -Dserver.port=9091
                                     </INSTRUMENTED_SERVICE_2_OPTS>
-                                    <INSTRUMENTED_SERVICE_2_ARGS>
-                                        --server.port=9091
-                                    </INSTRUMENTED_SERVICE_2_ARGS>
+
+                                    <INSTRUMENTED_SERVICE_3>
+                                        ${consumer.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_3>
+                                    <INSTRUMENTED_SERVICE_3_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801
+                                        -DSW_AGENT_NAME=${consumer.name}
+                                        -Dserver.port=9092
+                                    </INSTRUMENTED_SERVICE_3_OPTS>
                                 </env>
                                 <dependsOn>
                                     <container>${e2e.container.name.prefix}-elasticsearch</container>
@@ -150,7 +148,7 @@
                                 </dependsOn>
                                 <ports>
                                     <port>+webapp.host:webapp.port:8081</port>
-                                    <port>+service.host:service.port:9091</port>
+                                    <port>+service.host:service.port:9092</port>
                                 </ports>
                                 <links>
                                     <link>${e2e.container.name.prefix}-elasticsearch</link>
@@ -159,9 +157,6 @@
                                 <volumes>
                                     <bind>
                                         <volume>${sw.home}:/sw</volume>
-                                        <volume>
-                                            ../${gateway.name}/target/${gateway.name}-${project.version}.jar:/home/${gateway.name}-${project.version}.jar
-                                        </volume>
                                         <volume>
                                             ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar
                                         </volume>
@@ -196,6 +191,8 @@
                         <sw.webapp.port>${webapp.port}</sw.webapp.port>
                         <service.host>${service.host}</service.host>
                         <service.port>${service.port}</service.port>
+                        <provider.name>${provider.name}</provider.name>
+                        <consumer.name>${consumer.name}</consumer.name>
                     </systemPropertyVariables>
                 </configuration>
                 <executions>

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/clusterize.awk

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/awk -f
+
+BEGIN {
+    in_cluster_section=0;
+    in_cluster_zk_section=0;
+
+    in_storage_section=0;
+    in_storage_es_section=0;
+    in_storage_h2_section=0;
+}
+
+{
+    if (in_cluster_section == 0) {
+        in_cluster_section=$0 ~ /^cluster:$/
+    } else {
+        in_cluster_section=$0 ~ /^(#|\s{2})/
+    }
+    if (in_storage_section == 0) {
+        in_storage_section=$0 ~ /^storage:$/
+    } else {
+        in_storage_section=$0 ~ /^(#|\s{2})/
+    }
+
+    if (in_cluster_section == 1) {
+        # in the cluster: section now
+        # disable standalone module
+        if ($0 ~ /^  standalone:$/) {
+            print "#" $0
+        } else {
+            if (in_cluster_zk_section == 0) {
+                in_cluster_zk_section=$0 ~ /^#?\s+zookeeper:$/
+            } else {
+                in_cluster_zk_section=$0 ~ /^(#\s{4}|\s{2})/
+            }
+            if (in_cluster_zk_section == 1) {
+                # in the cluster.zookeeper section now
+                # uncomment zk config
+                gsub("^#", "", $0)
+                print
+            } else {
+                print
+            }
+        }
+    } else if (in_storage_section == 1) {
+        # in the storage: section now
+        # disable h2 module
+        if (in_storage_es_section == 0) {
+            if (ENVIRON["ES_VERSION"] ~ /^6.+/) {
+                in_storage_es_section=$0 ~ /^#?\s+elasticsearch:$/
+            } else if (ENVIRON["ES_VERSION"] ~ /^7.+/) {
+                in_storage_es_section=$0 ~ /^#?\s+elasticsearch7:$/
+            }
+        } else {
+            in_storage_es_section=$0 ~ /^#?\s{4}/
+        }
+        if (in_storage_h2_section == 0) {
+            in_storage_h2_section=$0 ~ /^#?\s+h2:$/
+        } else {
+            in_storage_h2_section=$0 ~ /^#?\s{4}/
+        }
+        if (in_storage_es_section == 1) {
+            # in the storage.elasticsearch section now
+            # uncomment es config
+            gsub("^#", "", $0)
+            print
+        } else if (in_storage_h2_section == 1) {
+            # comment out h2 config
+            if ($0 !~ /^#/) {
+                print "#" $0
+            } else {
+                print
+            }
+        } else {
+            print
+        }
+    } else {
+        print
+    }
+}
+

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc0-prepare.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the SkyAPM under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apt-get update && apt-get install -y gawk
+
+if test "${MODE}" = "cluster"; then
+    original_wd=$(pwd)
+
+    # substitute application.yml to be capable of cluster mode
+    cd ${SW_HOME}/config \
+        && gawk -f /clusterize.awk application.yml > clusterized_app.yml \
+        && mv clusterized_app.yml application.yml \
+        && sed '/<Loggers>/a<logger name="org.apache.skywalking.oap.server.receiver.trace.provider.UninstrumentedGatewaysConfig" level="DEBUG"/>\
+        \n<logger name="org.apache.skywalking.oap.server.receiver.trace.provider.parser.listener.service.ServiceMappingSpanListener" level="DEBUG"/>' log4j2.xml > log4j2debuggable.xml \
+        && mv log4j2debuggable.xml log4j2.xml
+
+    cd ${SW_HOME}/webapp \
+        && gawk '/^\s+listOfServers:/ {gsub("listOfServers:.*", "listOfServers: 127.0.0.1:12800,127.0.0.1:12801", $0)} {print}' webapp.yml > clusterized_webapp.yml \
+        && mv clusterized_webapp.yml webapp.yml
+
+    cd ${original_wd}
+fi

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc1-startup.sh

@@ -15,21 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-echo 'starting gateway service...' \
-    && java -jar /home/gateway-1.0.0.jar 2>&1 > /tmp/gateway.log &
-
-check_tcp 127.0.0.1 \
-          9099 \
-          60 \
-          10 \
-          'waiting for the gateway service to be ready'
-
-if [[ $? -ne 0 ]]; then
-    echo "gateway service failed to start in 60 * 10 seconds: "
-    cat /tmp/gateway.log
-    exit 1
-fi
-
 echo 'starting OAP server...' \
     && SW_STORAGE_ES_BULK_ACTIONS=1 \
     SW_STORAGE_ES_FLUSH_INTERVAL=1 \
@@ -78,10 +63,22 @@ if [[ $? -ne 0 ]]; then
     exit 1
 fi
 
+check_tcp 127.0.0.1 \
+          9092 \
+          60 \
+          10 \
+          "waiting for the instrumented service 2 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 2 failed to start in 24 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
 echo "SkyWalking e2e container is ready for tests"
 
 tail -f ${OAP_LOG_DIR}/* \
         ${WEBAPP_LOG_DIR}/* \
         ${SERVICE_LOG}/* \
         ${ES_HOME}/logs/elasticsearch.log \
-        ${ES_HOME}/logs/stdout.log
+        ${ES_HOME}/logs/stdout.log

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java

@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e;
+
+import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
+import org.apache.skywalking.e2e.metrics.Metrics;
+import org.apache.skywalking.e2e.metrics.MetricsQuery;
+import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
+import org.apache.skywalking.e2e.service.Service;
+import org.apache.skywalking.e2e.service.ServicesMatcher;
+import org.apache.skywalking.e2e.service.ServicesQuery;
+import org.apache.skywalking.e2e.service.endpoint.Endpoint;
+import org.apache.skywalking.e2e.service.endpoint.EndpointQuery;
+import org.apache.skywalking.e2e.service.endpoint.Endpoints;
+import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher;
+import org.apache.skywalking.e2e.service.instance.Instance;
+import org.apache.skywalking.e2e.service.instance.Instances;
+import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
+import org.apache.skywalking.e2e.service.instance.InstancesQuery;
+import org.apache.skywalking.e2e.topo.*;
+import org.apache.skywalking.e2e.trace.Trace;
+import org.apache.skywalking.e2e.trace.TracesMatcher;
+import org.apache.skywalking.e2e.trace.TracesQuery;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.web.client.RestTemplate;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.InputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.*;
+import static org.assertj.core.api.Assertions.fail;
+
+/**
+ * @author kezhenxu94
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+public class ClusterVerificationITCase {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ClusterVerificationITCase.class);
+
+    private final RestTemplate restTemplate = new RestTemplate();
+    private final Yaml yaml = new Yaml();
+
+    private SimpleQueryClient queryClient;
+    private String instrumentedServiceUrl;
+    private long retryInterval = TimeUnit.SECONDS.toMillis(30);
+
+    private String providerName;
+    private String consumerName;
+
+    @Before
+    public void setUp() {
+        providerName = System.getProperty("provider.name", "e2e-cluster-provider");
+        consumerName = System.getProperty("consumer.name", "e2e-cluster-consumer");
+
+        final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1");
+        final String swWebappPort = System.getProperty("sw.webapp.port", "32791");
+        final String instrumentedServiceHost = System.getProperty("service.host", "127.0.0.1");
+        final String instrumentedServicePort = System.getProperty("service.port", "32790");
+        queryClient = new SimpleQueryClient(swWebappHost, swWebappPort);
+        instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort;
+    }
+
+    @Test(timeout = 1200000)
+    @DirtiesContext
+    public void verify() throws Exception {
+        LocalDateTime startTime = LocalDateTime.now(ZoneOffset.UTC);
+
+        // minimum guarantee that the instrumented services registered
+        // which is the prerequisite of following verifications(service instance, service metrics, etc.)
+        List<Service> services = Collections.emptyList();
+        while (services.size() < 2) {
+            try {
+                services = queryClient.services(
+                        new ServicesQuery()
+                                .start(startTime)
+                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                );
+                Thread.sleep(500); // take a nap to avoid high payload
+            } catch (Throwable ignored) {
+            }
+        }
+
+        verifyTraces(startTime);
+
+        verifyServices(startTime);
+
+        verifyTopo(startTime);
+    }
+
+    private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
+        String clientServiceId = null, serverServiceId = null;
+        boolean valid = false;
+        while (!valid) {
+            try {
+                final TopoData topoData = queryClient.topo(
+                        new TopoQuery()
+                                .stepByMinute()
+                                .start(minutesAgo)
+                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                );
+                LOGGER.info("Actual topology: {}", topoData);
+
+                InputStream expectedInputStream =
+                        new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.topo.yml").getInputStream();
+
+                final TopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, TopoMatcher.class);
+                topoMatcher.verify(topoData);
+                valid = true;
+                for (Node node : topoData.getNodes()) {
+                    if (node.getName().equals(providerName)) {
+                        serverServiceId = node.getId();
+                    } else if (node.getName().equals(consumerName)) {
+                        clientServiceId = node.getId();
+                    }
+                }
+            } catch (Throwable t) {
+                LOGGER.warn(t.getMessage(), t);
+                generateTraffic();
+                Thread.sleep(retryInterval);
+            }
+        }
+        verifyServiceInstanceTopo(minutesAgo, clientServiceId, serverServiceId);
+    }
+
+    private void verifyServiceInstanceTopo(LocalDateTime minutesAgo, String clientServiceId, String serverServiceId) throws Exception {
+        if (clientServiceId == null || serverServiceId == null) {
+            fail("clientService or serverService not found");
+        }
+        boolean valid = false;
+        while (!valid) {
+            try {
+                final ServiceInstanceTopoData topoData = queryClient.serviceInstanceTopo(
+                        new ServiceInstanceTopoQuery()
+                                .stepByMinute()
+                                .start(minutesAgo.minusDays(1))
+                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                                .clientServiceId(clientServiceId)
+                                .serverServiceId(serverServiceId)
+                );
+
+                LOGGER.info("Actual service instance topology: {}", topoData);
+                InputStream expectedInputStream =
+                        new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml").getInputStream();
+                final ServiceInstanceTopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
+                topoMatcher.verify(topoData);
+                valid = true;
+            } catch (Throwable t) {
+                LOGGER.warn(t.getMessage(), t);
+                generateTraffic();
+                Thread.sleep(retryInterval);
+            }
+        }
+    }
+
+    private void verifyServices(LocalDateTime minutesAgo) throws Exception {
+        List<Service> services = queryClient.services(
+                new ServicesQuery()
+                        .start(minutesAgo)
+                        .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+        );
+        while (services.isEmpty()) {
+            LOGGER.warn("services is null, will retry to query");
+            services = queryClient.services(
+                    new ServicesQuery()
+                            .start(minutesAgo)
+                            .end(LocalDateTime.now(ZoneOffset.UTC))
+            );
+            Thread.sleep(retryInterval);
+        }
+
+        InputStream expectedInputStream =
+                new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.services.yml").getInputStream();
+
+        final ServicesMatcher servicesMatcher = yaml.loadAs(expectedInputStream, ServicesMatcher.class);
+        servicesMatcher.verify(services);
+
+        for (Service service : services) {
+            LOGGER.info("verifying service instances: {}", service);
+
+            verifyServiceMetrics(service, minutesAgo);
+
+            Instances instances = verifyServiceInstances(minutesAgo, service);
+
+            verifyInstancesMetrics(instances, minutesAgo);
+
+            Endpoints endpoints = verifyServiceEndpoints(minutesAgo, service);
+
+            verifyEndpointsMetrics(endpoints, minutesAgo);
+        }
+    }
+
+    private Instances verifyServiceInstances(LocalDateTime minutesAgo, Service service) throws Exception {
+        Instances instances = queryClient.instances(
+                new InstancesQuery()
+                        .serviceId(service.getKey())
+                        .start(minutesAgo)
+                        .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+        );
+        while (instances == null) {
+            LOGGER.warn("instances is null, will send traffic data and retry to query");
+            generateTraffic();
+            Thread.sleep(retryInterval);
+            instances = queryClient.instances(
+                    new InstancesQuery()
+                            .serviceId(service.getKey())
+                            .start(minutesAgo)
+                            .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+            );
+        }
+
+        InputStream expectedInputStream;
+        if (providerName.equals(service.getLabel())) {
+            expectedInputStream =
+                    new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.providerInstances.yml").getInputStream();
+        } else {
+            expectedInputStream =
+                    new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml").getInputStream();
+        }
+
+        final InstancesMatcher instancesMatcher = yaml.loadAs(expectedInputStream, InstancesMatcher.class);
+        instancesMatcher.verify(instances);
+        return instances;
+    }
+
+    private Endpoints verifyServiceEndpoints(LocalDateTime minutesAgo, Service service) throws Exception {
+        Endpoints endpoints = queryClient.endpoints(
+                new EndpointQuery().serviceId(service.getKey())
+        );
+        while (endpoints == null) {
+            LOGGER.warn("endpoints is null, will send traffic data and retry to query");
+            generateTraffic();
+            Thread.sleep(retryInterval);
+            endpoints = queryClient.endpoints(
+                    new EndpointQuery().serviceId(service.getKey())
+            );
+        }
+        InputStream expectedInputStream =
+                new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.endpoints.yml").getInputStream();
+        final EndpointsMatcher endpointsMatcher = yaml.loadAs(expectedInputStream, EndpointsMatcher.class);
+        endpointsMatcher.verify(endpoints);
+        return endpoints;
+    }
+
+    private void verifyInstancesMetrics(Instances instances, final LocalDateTime minutesAgo) throws Exception {
+        for (Instance instance : instances.getInstances()) {
+            for (String metricsName : ALL_INSTANCE_METRICS) {
+                LOGGER.info("verifying service instance response time: {}", instance);
+
+                boolean valid = false;
+                while (!valid) {
+                    LOGGER.warn("instanceMetrics is null, will retry to query");
+                    Metrics instanceMetrics = queryClient.metrics(
+                            new MetricsQuery()
+                                    .stepByMinute()
+                                    .metricsName(metricsName)
+                                    .start(minutesAgo)
+                                    .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                                    .id(instance.getKey())
+                    );
+                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                    greaterThanZero.setValue("gt 0");
+                    instanceRespTimeMatcher.setValue(greaterThanZero);
+                    try {
+                        instanceRespTimeMatcher.verify(instanceMetrics);
+                        valid = true;
+                    } catch (Throwable ignored) {
+                        generateTraffic();
+                        Thread.sleep(retryInterval);
+                    }
+                    LOGGER.info("{}: {}", metricsName, instanceMetrics);
+                }
+            }
+        }
+    }
+
+    private void verifyEndpointsMetrics(Endpoints endpoints, final LocalDateTime minutesAgo) throws Exception {
+        for (Endpoint endpoint : endpoints.getEndpoints()) {
+            if (!endpoint.getLabel().equals("/e2e/users")) {
+                continue;
+            }
+            for (String metricName : ALL_ENDPOINT_METRICS) {
+                LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName);
+
+                boolean valid = false;
+                while (!valid) {
+                    Metrics endpointMetrics = queryClient.metrics(
+                            new MetricsQuery()
+                                    .stepByMinute()
+                                    .metricsName(metricName)
+                                    .start(minutesAgo)
+                                    .end(LocalDateTime.now(ZoneOffset.UTC))
+                                    .id(endpoint.getKey())
+                    );
+                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                    greaterThanZero.setValue("gt 0");
+                    instanceRespTimeMatcher.setValue(greaterThanZero);
+                    try {
+                        instanceRespTimeMatcher.verify(endpointMetrics);
+                        valid = true;
+                    } catch (Throwable ignored) {
+                        generateTraffic();
+                        Thread.sleep(retryInterval);
+                    }
+                    LOGGER.info("{}: {}", metricName, endpointMetrics);
+                }
+            }
+        }
+    }
+
+    private void verifyServiceMetrics(Service service, final LocalDateTime minutesAgo) throws Exception {
+        for (String metricName : ALL_SERVICE_METRICS) {
+            LOGGER.info("verifying service {}, metrics: {}", service, metricName);
+
+            boolean valid = false;
+            while (!valid) {
+                Metrics serviceMetrics = queryClient.metrics(
+                        new MetricsQuery()
+                                .stepByMinute()
+                                .metricsName(metricName)
+                                .start(minutesAgo)
+                                .end(LocalDateTime.now(ZoneOffset.UTC))
+                                .id(service.getKey())
+                );
+                AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                greaterThanZero.setValue("gt 0");
+                instanceRespTimeMatcher.setValue(greaterThanZero);
+                try {
+                    instanceRespTimeMatcher.verify(serviceMetrics);
+                    valid = true;
+                } catch (Throwable ignored) {
+                    generateTraffic();
+                    Thread.sleep(retryInterval);
+                }
+                LOGGER.info("{}: {}", metricName, serviceMetrics);
+            }
+        }
+    }
+
+    private void verifyTraces(LocalDateTime minutesAgo) throws Exception {
+        final TracesQuery query = new TracesQuery()
+                .stepBySecond()
+                .start(minutesAgo)
+                .orderByStartTime();
+
+        List<Trace> traces = queryClient.traces(query.end(LocalDateTime.now(ZoneOffset.UTC)));
+        while (traces.isEmpty()) {
+            LOGGER.warn("traces is empty, will generate traffic data and retry");
+            generateTraffic();
+            Thread.sleep(retryInterval);
+            traces = queryClient.traces(query.end(LocalDateTime.now(ZoneOffset.UTC)));
+        }
+
+        InputStream expectedInputStream =
+                new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.traces.yml").getInputStream();
+
+        final TracesMatcher tracesMatcher = yaml.loadAs(expectedInputStream, TracesMatcher.class);
+        tracesMatcher.verifyLoosely(traces);
+    }
+
+    private void generateTraffic() {
+        try {
+            final Map<String, String> user = new HashMap<>();
+            user.put("name", "SkyWalking");
+            final ResponseEntity<String> responseEntity = restTemplate.postForEntity(
+                    instrumentedServiceUrl + "/e2e/users",
+                    user,
+                    String.class
+            );
+            LOGGER.info("responseEntity: {}, {}", responseEntity.getStatusCode(), responseEntity.getBody());
+        } catch (Throwable t) {
+            LOGGER.warn(t.getMessage(), t);
+        }
+    }
+}

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.endpoints.yml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+endpoints:
+  - key: not null
+    label: /e2e/users

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+instances:
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.providerInstances.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+instances:
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.serviceInstanceTopo.yml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+nodes:
+  - id: not null
+    name: not null
+    type: null
+    serviceId: not null
+    serviceName: e2e-cluster-consumer
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-provider
+    type: Tomcat
+    isReal: true
+  - id: not null
+    name: not null
+    serviceId: not null
+    serviceName: e2e-cluster-provider
+    type: Tomcat
+    isReal: true
+calls:
+  - id: not null
+    source: ${e2e-cluster-consumer(.*)[0]}
+    target: ${e2e-cluster-provider(.*)[0]}
+  - id: not null
+    source: ${e2e-cluster-consumer(.*)[0]}
+    target: ${e2e-cluster-provider(.*)[1]}

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.services.yml

@@ -22,7 +22,7 @@
 
 services:
   - key: not null
-    label: provider
+    label: e2e-cluster-provider
 
   - key: not null
-    label: consumer
+    label: e2e-cluster-consumer

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.topo.yml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+  - id: not null
+    name: User
+    type: USER
+  - id: not null
+    name: e2e-cluster-provider
+    type: Tomcat
+  - id: not null
+    name: e2e-cluster-consumer
+    type: Tomcat
+  - id: not null
+    name: "localhost:-1"
+    type: H2
+calls:
+  - id: not null
+    source: ${User[0]}
+    target: ${e2e-cluster-consumer[0]}
+  - id: not null
+    source: ${e2e-cluster-consumer[0]}
+    target: ${e2e-cluster-provider[0]}
+  - id: not null
+    source: ${e2e-cluster-provider[0]}
+    target: ${localhost:-1[0]}

test/e2e/e2e-cluster/e2e-cluster-test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.traces.yml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+traces:
+  - key: not null
+    endpointNames:
+      - /e2e/users
+    duration: ge 0
+    start: gt 0
+    isError: false
+    traceIds:
+      - not null

test/e2e/e2e-cluster/pom.xml

@@ -31,9 +31,8 @@
     <packaging>pom</packaging>
 
     <modules>
-        <module>provider</module>
-        <module>consumer</module>
-        <module>gateway</module>
-        <module>test-runner</module>
+        <module>e2e-cluster-provider</module>
+        <module>e2e-cluster-consumer</module>
+        <module>e2e-cluster-test-runner</module>
     </modules>
 </project>

test/e2e/e2e-mysql/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java

@@ -33,9 +33,7 @@ import org.apache.skywalking.e2e.service.instance.Instance;
 import org.apache.skywalking.e2e.service.instance.Instances;
 import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
 import org.apache.skywalking.e2e.service.instance.InstancesQuery;
-import org.apache.skywalking.e2e.topo.TopoData;
-import org.apache.skywalking.e2e.topo.TopoMatcher;
-import org.apache.skywalking.e2e.topo.TopoQuery;
+import org.apache.skywalking.e2e.topo.*;
 import org.apache.skywalking.e2e.trace.Trace;
 import org.apache.skywalking.e2e.trace.TracesMatcher;
 import org.apache.skywalking.e2e.trace.TracesQuery;
@@ -87,7 +85,7 @@ public class SampleVerificationITCase {
         instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort;
     }
 
-    @Test
+    @Test(timeout = 1200000)
     @DirtiesContext
     public void verify() throws Exception {
         final LocalDateTime minutesAgo = LocalDateTime.now(ZoneOffset.UTC);
@@ -140,6 +138,14 @@ public class SampleVerificationITCase {
                 LOGGER.warn(e.getMessage(), e);
             }
         });
+
+        doRetryableVerification(() -> {
+            try{
+                verifyServiceInstanceTopo(minutesAgo);
+            }catch (Exception e) {
+                LOGGER.warn(e.getMessage(), e);
+            }
+        });
     }
 
     private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
@@ -160,6 +166,26 @@ public class SampleVerificationITCase {
         topoMatcher.verify(topoData);
     }
 
+    private void verifyServiceInstanceTopo(LocalDateTime minutesAgo) throws Exception {
+        final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
+
+        final ServiceInstanceTopoData topoData = queryClient.serviceInstanceTopo(
+                new ServiceInstanceTopoQuery()
+                        .stepByMinute()
+                        .start(minutesAgo.minusDays(1))
+                        .end(now)
+                        .clientServiceId("1")
+                        .serverServiceId("2")
+        );
+        LOGGER.info("instanceTopoData: {}", topoData);
+
+        InputStream expectedInputStream =
+                new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml").getInputStream();
+
+        final ServiceInstanceTopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
+        topoMatcher.verify(topoData);
+    }
+
     private void verifyServices(LocalDateTime minutesAgo) throws Exception {
         final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
 

test/e2e/e2e-mysql/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+nodes:
+  - id: 1
+    name: User
+    type: USER
+    serviceId: 1
+    serviceName: User
+    isReal: false
+  - id: 2
+    name: not null
+    serviceId: 2
+    serviceName: Your_ApplicationName
+    type: Tomcat
+    isReal: true
+calls:
+  - id: 1_2
+    source: 1
+    detectPoints:
+      - SERVER
+    target: 2

test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java

@@ -33,9 +33,7 @@ import org.apache.skywalking.e2e.service.instance.Instance;
 import org.apache.skywalking.e2e.service.instance.Instances;
 import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
 import org.apache.skywalking.e2e.service.instance.InstancesQuery;
-import org.apache.skywalking.e2e.topo.TopoData;
-import org.apache.skywalking.e2e.topo.TopoMatcher;
-import org.apache.skywalking.e2e.topo.TopoQuery;
+import org.apache.skywalking.e2e.topo.*;
 import org.apache.skywalking.e2e.trace.Trace;
 import org.apache.skywalking.e2e.trace.TracesMatcher;
 import org.apache.skywalking.e2e.trace.TracesQuery;
@@ -87,7 +85,7 @@ public class SampleVerificationITCase {
         instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort;
     }
 
-    @Test
+    @Test(timeout = 1200000)
     @DirtiesContext
     public void verify() throws Exception {
         final LocalDateTime minutesAgo = LocalDateTime.now(ZoneOffset.UTC);
@@ -140,6 +138,14 @@ public class SampleVerificationITCase {
                 LOGGER.warn(e.getMessage(), e);
             }
         });
+
+        doRetryableVerification(() -> {
+            try{
+                verifyServiceInstanceTopo(minutesAgo);
+            }catch (Exception e) {
+                LOGGER.warn(e.getMessage(), e);
+            }
+        });
     }
 
     private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
@@ -160,6 +166,26 @@ public class SampleVerificationITCase {
         topoMatcher.verify(topoData);
     }
 
+    private void verifyServiceInstanceTopo(LocalDateTime minutesAgo) throws Exception {
+        final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);
+
+        final ServiceInstanceTopoData topoData = queryClient.serviceInstanceTopo(
+                new ServiceInstanceTopoQuery()
+                        .stepByMinute()
+                        .start(minutesAgo.minusDays(1))
+                        .end(now)
+                        .clientServiceId("1")
+                        .serverServiceId("2")
+        );
+        LOGGER.info("instanceTopoData: {}", topoData);
+
+        InputStream expectedInputStream =
+                new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml").getInputStream();
+
+        final ServiceInstanceTopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class);
+        topoMatcher.verify(topoData);
+    }
+
     private void verifyServices(LocalDateTime minutesAgo) throws Exception {
         final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC);