From d63f2c9427d7b23e55dbe7127a99662338dfb482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Mon, 24 Feb 2020 23:01:13 +0800 Subject: [PATCH] Support topology with multiple selected services (#4408) * 1. Format codes 2. Add comments for storage interface of topology 3. Narrow the scope of secondary query to get the component name. Only query the nodes as client out of the given service Ids. * Support topology with multiple selected services * Sync the typo fix in the protocol --- .../server/core/query/TopologyBuilder.java | 6 +- .../core/query/TopologyQueryService.java | 63 +++++++++++++------ .../core/storage/query/ITopologyQueryDAO.java | 56 +++++++++++++---- .../query/graphql/resolver/TopologyQuery.java | 26 ++++++-- .../src/main/resources/query-protocol | 2 +- .../jdbc/h2/dao/H2TopologyQueryDAO.java | 4 +- 6 files changed, 115 insertions(+), 42 deletions(-) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyBuilder.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyBuilder.java index 5c6f51bcf..8aeb8ad11 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyBuilder.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyBuilder.java @@ -82,7 +82,8 @@ class TopologyBuilder { nodes.put(target.getSequence(), buildNode(target)); if (BooleanUtils.valueToBoolean(target.getIsAddress())) { nodes.get(target.getSequence()) - .setType(componentLibraryCatalogService.getServerNameBasedOnComponent(clientCall.getComponentId())); + .setType( + componentLibraryCatalogService.getServerNameBasedOnComponent(clientCall.getComponentId())); } } @@ -129,7 +130,8 @@ class TopologyBuilder { Node conjecturalNode = new Node(); conjecturalNode.setId(source.getSequence()); conjecturalNode.setName(source.getName()); - conjecturalNode.setType(componentLibraryCatalogService.getServerNameBasedOnComponent(serverCall.getComponentId())); + conjecturalNode.setType( + componentLibraryCatalogService.getServerNameBasedOnComponent(serverCall.getComponentId())); conjecturalNode.setReal(true); nodes.put(source.getSequence(), conjecturalNode); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java index e384d7c66..980a577f0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java @@ -90,30 +90,44 @@ public class TopologyQueryService implements Service { } public Topology getGlobalTopology(final Downsampling downsampling, final long startTB, - final long endTB) throws IOException { + final long endTB) throws IOException { logger.debug("Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", downsampling, startTB, endTB); - List serviceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceRelations(downsampling, startTB, endTB); - List serviceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceRelations(downsampling, startTB, endTB); + List serviceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceRelations( + downsampling, startTB, endTB); + List serviceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceRelations( + downsampling, startTB, endTB); TopologyBuilder builder = new TopologyBuilder(moduleManager); return builder.build(serviceRelationClientCalls, serviceRelationServerCalls); } public Topology getServiceTopology(final Downsampling downsampling, final long startTB, final long endTB, - final int serviceId) throws IOException { - List serviceIds = new ArrayList<>(); - serviceIds.add(serviceId); - - List serviceRelationClientCalls = getTopologyQueryDAO().loadSpecifiedClientSideServiceRelations(downsampling, startTB, endTB, serviceIds); - List serviceRelationServerCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(downsampling, startTB, endTB, serviceIds); + final List serviceIds) throws IOException { + List serviceRelationClientCalls = getTopologyQueryDAO().loadSpecifiedClientSideServiceRelations( + downsampling, startTB, endTB, serviceIds); + List serviceRelationServerCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations( + downsampling, startTB, endTB, serviceIds); TopologyBuilder builder = new TopologyBuilder(moduleManager); Topology topology = builder.build(serviceRelationClientCalls, serviceRelationServerCalls); - List sourceServiceIds = new ArrayList<>(); - serviceRelationClientCalls.forEach(call -> sourceServiceIds.add(call.getSource())); - if (CollectionUtils.isNotEmpty(sourceServiceIds)) { - List sourceCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(downsampling, startTB, endTB, sourceServiceIds); + /** + * The topology built above is complete. + * There is a special case, there may be a node of the `serviceIds` call these services as and only as a client, so it is included in the topology, + * its component name could be missed as not being queried before. We add another query about this. + */ + List outScopeSourceServiceIds = new ArrayList<>(); + serviceRelationClientCalls.forEach(call -> { + // Client side relationships exclude the given services(#serviceIds) + // The given services(#serviceIds)'s component names have been included inside `serviceRelationServerCalls` + if (!serviceIds.contains(call.getSource())) { + outScopeSourceServiceIds.add(call.getSource()); + } + }); + if (CollectionUtils.isNotEmpty(outScopeSourceServiceIds)) { + // If exist, query them as the server side to get the target's component. + List sourceCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations( + downsampling, startTB, endTB, outScopeSourceServiceIds); topology.getNodes().forEach(node -> { if (Strings.isNullOrEmpty(node.getType())) { for (Call.CallDetail call : sourceCalls) { @@ -129,20 +143,29 @@ public class TopologyQueryService implements Service { return topology; } - public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId, - final Downsampling downsampling, final long startTB, final long endTB) throws IOException { - logger.debug("ClientServiceId: {}, ServerServiceId: {}, Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", clientServiceId, serverServiceId, downsampling, startTB, endTB); + public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, + final int serverServiceId, + final Downsampling downsampling, + final long startTB, + final long endTB) throws IOException { + logger.debug( + "ClientServiceId: {}, ServerServiceId: {}, Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", + clientServiceId, serverServiceId, downsampling, startTB, endTB + ); - List serviceInstanceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB); - List serviceInstanceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB); + List serviceInstanceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceInstanceRelations( + clientServiceId, serverServiceId, downsampling, startTB, endTB); + List serviceInstanceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceInstanceRelations( + clientServiceId, serverServiceId, downsampling, startTB, endTB); ServiceInstanceTopologyBuilder builder = new ServiceInstanceTopologyBuilder(moduleManager); return builder.build(serviceInstanceRelationClientCalls, serviceInstanceRelationServerCalls); } public Topology getEndpointTopology(final Downsampling downsampling, final long startTB, final long endTB, - final int endpointId) throws IOException { - List serverSideCalls = getTopologyQueryDAO().loadSpecifiedDestOfServerSideEndpointRelations(downsampling, startTB, endTB, endpointId); + final int endpointId) throws IOException { + List serverSideCalls = getTopologyQueryDAO().loadSpecifiedDestOfServerSideEndpointRelations( + downsampling, startTB, endTB, endpointId); Topology topology = new Topology(); serverSideCalls.forEach(callDetail -> { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java index 1f9689e1f..ba2618f7c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java @@ -21,29 +21,63 @@ package org.apache.skywalking.oap.server.core.storage.query; import java.io.IOException; import java.util.List; import org.apache.skywalking.oap.server.core.analysis.Downsampling; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; import org.apache.skywalking.oap.server.core.query.entity.Call; import org.apache.skywalking.oap.server.library.module.Service; public interface ITopologyQueryDAO extends Service { - + /** + * Query {@link ServiceRelationServerSideMetrics} through the given conditions + */ List loadSpecifiedServerSideServiceRelations(Downsampling downsampling, long startTB, long endTB, - List serviceIds) throws IOException; + List serviceIds) throws IOException; + /** + * Query {@link ServiceRelationClientSideMetrics} through the given conditions + */ List loadSpecifiedClientSideServiceRelations(Downsampling downsampling, long startTB, long endTB, - List serviceIds) throws IOException; + List serviceIds) throws IOException; + /** + * Query {@link ServiceRelationServerSideMetrics} globally, without given serviceIds + */ List loadServerSideServiceRelations(Downsampling downsampling, long startTB, - long endTB) throws IOException; + long endTB) throws IOException; + /** + * Query {@link ServiceRelationClientSideMetrics} globally, without given serviceIds + */ List loadClientSideServiceRelations(Downsampling downsampling, long startTB, - long endTB) throws IOException; + long endTB) throws IOException; - List loadServerSideServiceInstanceRelations(int clientServiceId, int serverServiceId, - Downsampling downsampling, long startTB, long endTB) throws IOException; + /** + * Query {@link ServiceInstanceRelationServerSideMetrics} through given conditions, including the specific + * clientServiceId and serverServiceId + */ + List loadServerSideServiceInstanceRelations(int clientServiceId, + int serverServiceId, + Downsampling downsampling, + long startTB, + long endTB) throws IOException; - List loadClientSideServiceInstanceRelations(int clientServiceId, int serverServiceId, - Downsampling downsampling, long startTB, long endTB) throws IOException; + /** + * Query {@link ServiceInstanceRelationClientSideMetrics} through given conditions, including the specific + * clientServiceId and serverServiceId + */ + List loadClientSideServiceInstanceRelations(int clientServiceId, + int serverServiceId, + Downsampling downsampling, + long startTB, + long endTB) throws IOException; - List loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, long startTB, - long endTB, int destEndpointId) throws IOException; + /** + * Query the endpoint relationship. Endpoint dependency is not detected from server side agent. + */ + List loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, + long startTB, + long endTB, + int destEndpointId) throws IOException; } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java index 596cb9d89..37ff28cb2 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java @@ -20,6 +20,8 @@ package org.apache.skywalking.oap.query.graphql.resolver; import com.coxautodev.graphql.tools.GraphQLQueryResolver; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.skywalking.oap.query.graphql.type.Duration; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.query.DurationUtils; @@ -49,29 +51,41 @@ public class TopologyQuery implements GraphQLQueryResolver { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getQueryService().getGlobalTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); + return getQueryService().getGlobalTopology( + StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket); } public Topology getServiceTopology(final int serviceId, final Duration duration) throws IOException { + List selectedServiceList = new ArrayList<>(1); + selectedServiceList.add(serviceId); + return this.getServicesTopology(selectedServiceList, duration); + } + + public Topology getServicesTopology(final List serviceIds, final Duration duration) throws IOException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getQueryService().getServiceTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, serviceId); + return getQueryService().getServiceTopology( + StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, serviceIds); } public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId, - final Duration duration) throws IOException { + final Duration duration) throws IOException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getQueryService().getServiceInstanceTopology(clientServiceId, serverServiceId, StepToDownsampling.transform(duration - .getStep()), startTimeBucket, endTimeBucket); + return getQueryService().getServiceInstanceTopology( + clientServiceId, serverServiceId, StepToDownsampling.transform(duration + .getStep()), startTimeBucket, + endTimeBucket + ); } public Topology getEndpointTopology(final int endpointId, final Duration duration) throws IOException { long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart()); long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd()); - return getQueryService().getEndpointTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, endpointId); + return getQueryService().getEndpointTopology( + StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, endpointId); } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 6104df8f0..1018b7950 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 6104df8f0debce4c81142e5b1fc696d61bb673c5 +Subproject commit 1018b795021e0d96e6c131c8695fb9ddc9d66916 diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java index 02ed57f27..ebf68e824 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java @@ -56,7 +56,7 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO { public List loadSpecifiedClientSideServiceRelations(Downsampling downsampling, long startTB, long endTB, List serviceIds) throws IOException { String tableName = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME); - return loadServiceCalls(tableName, startTB, endTB, ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceIds, true); + return loadServiceCalls(tableName, startTB, endTB, ServiceRelationClientSideMetrics.SOURCE_SERVICE_ID, ServiceRelationClientSideMetrics.DEST_SERVICE_ID, serviceIds, true); } @Override @@ -70,7 +70,7 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO { public List loadClientSideServiceRelations(Downsampling downsampling, long startTB, long endTB) throws IOException { String tableName = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME); - return loadServiceCalls(tableName, startTB, endTB, ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, ServiceRelationServerSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0), true); + return loadServiceCalls(tableName, startTB, endTB, ServiceRelationClientSideMetrics.SOURCE_SERVICE_ID, ServiceRelationClientSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0), true); } @Override