From c9202af1ee4f48acaa20814788da8b224bfcfe0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Tue, 17 Nov 2020 22:45:32 +0800 Subject: [PATCH] Support group in the service traffic and service list query. (#5851) * Support group in the service traffic and service list query. --- CHANGES.md | 2 + docs/en/setup/backend/backend-setup.md | 1 + .../en/setup/backend/service-auto-grouping.md | 16 ++++ .../skywalking/oap/server/core/Const.java | 1 + .../manual/service/ServiceTraffic.java | 19 +++- .../core/query/MetadataQueryService.java | 10 +-- .../oap/server/core/query/type/Service.java | 1 + .../core/storage/query/IMetadataQueryDAO.java | 39 +++++++-- .../manual/service/ServiceTrafficTest.java | 44 ++++++++++ .../query/graphql/resolver/MetadataQuery.java | 8 +- .../src/main/resources/query-protocol | 2 +- .../query/MetadataQueryEsDAO.java | 11 ++- .../plugin/influxdb/InfluxConstants.java | 2 + .../plugin/influxdb/TableMetaInfo.java | 3 +- .../plugin/influxdb/query/MetadataQuery.java | 86 ++++++++----------- .../jdbc/h2/dao/H2MetadataQueryDAO.java | 12 ++- 16 files changed, 184 insertions(+), 73 deletions(-) create mode 100644 docs/en/setup/backend/service-auto-grouping.md create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTrafficTest.java diff --git a/CHANGES.md b/CHANGES.md index b0b3c0836..40031986c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,8 @@ Release Notes. * Support `sideCar.internalErrorCode` in the Service, ServiceInstance, Endpoint, ServiceRelation, and ServiceInstanceRelation sources. * Improve Kubernetes service registry for ALS analysis. * Add health checker for cluster management +* Support the service auto grouping. +* Support query service list by the group name. * Improve the queryable tags generation. Remove the duplicated tags to reduce the storage payload. * Fix the threads of the Kafka fetcher exit if some unexpected exceptions happen. * Fix the excessive timeout period set by the kubernetes-client. diff --git a/docs/en/setup/backend/backend-setup.md b/docs/en/setup/backend/backend-setup.md index c41db9efc..4ff27a431 100755 --- a/docs/en/setup/backend/backend-setup.md +++ b/docs/en/setup/backend/backend-setup.md @@ -108,6 +108,7 @@ or 3rd party configuration management system. 1. [Uninstrumented Gateways](uninstrumented-gateways.md). Configure gateways/proxies that are not supported by SkyWalking agent plugins, to reflect the delegation in topology graph. 1. [Apdex threshold](apdex-threshold.md). Configure the thresholds for different services if Apdex calculation is activated in the OAL. +1. [Service Grouping](service-auto-grouping.md). An automatic grouping mechanism for all services based on name. 1. [Group Parameterized Endpoints](endpoint-grouping-rules.md). Configure the grouping rules for parameterized endpoints, to improve the meaning of the metrics. 1. [Meter Analysis](backend-meter.md). Set up the backend analysis rules, when use [SkyWalking Meter System Toolkit](../service-agent/java-agent/README.md#advanced-features) diff --git a/docs/en/setup/backend/service-auto-grouping.md b/docs/en/setup/backend/service-auto-grouping.md new file mode 100644 index 000000000..9530dc140 --- /dev/null +++ b/docs/en/setup/backend/service-auto-grouping.md @@ -0,0 +1,16 @@ +# Service Auto Grouping +SkyWalking supports various default and customized dashboard templates. +Each template provides the reasonable layout for the services in the particular field. +Such as, services with a language agent installed +could have different metrics with service detected by the service mesh observability solution, +and different with SkyWalking's self-observability metrics dashboard. + +Therefore, since 8.3.0, SkyWalking OAP would generate the group based on this simple naming format. + +### ${service name} = [${group name}::]${logic name} + +Once the service name includes double colons(`::`), the literal string before the colons would be considered as the group name. +In the latest GraphQL query, the group name has been provided as an option parameter. +> getAllServices(duration: Duration!, group: String): [Service!]! + +RocketBot UI dashboards(`Standard` type) support the `group name` for default and custom configurations. \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/Const.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/Const.java index 3592bc85e..499bc1884 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/Const.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/Const.java @@ -39,4 +39,5 @@ public class Const { public static final String UNKNOWN = "Unknown"; public static final String EMPTY_STRING = ""; public static final String POINT = "."; + public static final String DOUBLE_COLONS_SPLIT = "::"; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java index 2f635667c..0841f8e53 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTraffic.java @@ -35,6 +35,8 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.StorageBuilder; import org.apache.skywalking.oap.server.core.storage.annotation.Column; +import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT; + @Stream(name = ServiceTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE, builder = ServiceTraffic.Builder.class, processor = MetricsStreamProcessor.class) @MetricsExtension(supportDownSampling = false, supportUpdate = false) @@ -47,6 +49,7 @@ public class ServiceTraffic extends Metrics { public static final String NAME = "name"; public static final String NODE_TYPE = "node_type"; + public static final String GROUP = "service_group"; @Setter @Getter @@ -58,6 +61,11 @@ public class ServiceTraffic extends Metrics { @Column(columnName = NODE_TYPE) private NodeType nodeType; + @Setter + @Getter + @Column(columnName = GROUP) + private String group; + @Override public String id() { return IDManager.ServiceID.buildId(name, nodeType); @@ -93,14 +101,23 @@ public class ServiceTraffic extends Metrics { ServiceTraffic serviceTraffic = new ServiceTraffic(); serviceTraffic.setName((String) dbMap.get(NAME)); serviceTraffic.setNodeType(NodeType.valueOf(((Number) dbMap.get(NODE_TYPE)).intValue())); + serviceTraffic.setGroup((String) dbMap.get(GROUP)); return serviceTraffic; } @Override public Map data2Map(final ServiceTraffic storageData) { + final String serviceName = storageData.getName(); + if (NodeType.Normal.equals(storageData.getNodeType())) { + int groupIdx = serviceName.indexOf(DOUBLE_COLONS_SPLIT); + if (groupIdx > 0) { + storageData.setGroup(serviceName.substring(0, groupIdx)); + } + } Map map = new HashMap<>(); - map.put(NAME, storageData.getName()); + map.put(NAME, serviceName); map.put(NODE_TYPE, storageData.getNodeType().value()); + map.put(GROUP, storageData.getGroup()); return map; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java index b373fe427..696dcb195 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java @@ -46,12 +46,12 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li return metadataQueryDAO; } - public List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException { - return getMetadataQueryDAO().getAllServices(startTimestamp, endTimestamp); + public List getAllServices(final String group) throws IOException { + return getMetadataQueryDAO().getAllServices(group); } - public List getAllBrowserServices(final long startTimestamp, final long endTimestamp) throws IOException { - return getMetadataQueryDAO().getAllBrowserServices(startTimestamp, endTimestamp); + public List getAllBrowserServices() throws IOException { + return getMetadataQueryDAO().getAllBrowserServices(); } public List getAllDatabases() throws IOException { @@ -60,7 +60,7 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li public List searchServices(final long startTimestamp, final long endTimestamp, final String keyword) throws IOException { - return getMetadataQueryDAO().searchServices(startTimestamp, endTimestamp, keyword); + return getMetadataQueryDAO().searchServices(keyword); } public List getServiceInstances(final long startTimestamp, final long endTimestamp, diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Service.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Service.java index a844cc9e3..b517f4b81 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Service.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Service.java @@ -26,4 +26,5 @@ import lombok.Setter; public class Service { private String id; private String name; + private String group; } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java index 91400311e..ad792e033 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java @@ -27,19 +27,48 @@ import org.apache.skywalking.oap.server.core.query.type.ServiceInstance; import org.apache.skywalking.oap.server.core.storage.DAO; public interface IMetadataQueryDAO extends DAO { - List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException; + /** + * @param group group name for filtering. + * @return list of the all available normal services + */ + List getAllServices(final String group) throws IOException; - List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException; + /** + * @return list of the all available browser services + */ + List getAllBrowserServices() throws IOException; + /** + * @return list of all conjecture database services. + */ List getAllDatabases() throws IOException; - List searchServices(final long startTimestamp, final long endTimestamp, - final String keyword) throws IOException; + /** + * @param keyword to filter the normal service + * @return the list of normal services matching the given keyword + */ + List searchServices(final String keyword) throws IOException; + /** + * @param serviceCode to literal match + * @return the service matching the given full name. + */ Service searchService(final String serviceCode) throws IOException; + /** + * @param keyword to filter the endpoints + * @param serviceId the owner of the endpoints + * @param limit max match size. + * @return list of services matching the given conditions. + */ List searchEndpoint(final String keyword, final String serviceId, final int limit) throws IOException; + /** + * @param startTimestamp The instance is required to be live after this timestamp + * @param endTimestamp The instance is required to be live before this timestamp. + * @param serviceId the owner of the instances. + * @return list of instances matching the given conditions. + */ List getServiceInstances(final long startTimestamp, final long endTimestamp, - final String serviceId) throws IOException; + final String serviceId) throws IOException; } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTrafficTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTrafficTest.java new file mode 100644 index 000000000..15e244894 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/manual/service/ServiceTrafficTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.analysis.manual.service; + +import java.util.Map; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.junit.Assert; +import org.junit.Test; + +public class ServiceTrafficTest { + @Test + public void testGrouping() { + ServiceTraffic traffic = new ServiceTraffic(); + traffic.setName("group-name::service-name"); + traffic.setNodeType(NodeType.Normal); + final Map stringObjectMap = new ServiceTraffic.Builder().data2Map(traffic); + Assert.assertEquals("group-name", stringObjectMap.get(ServiceTraffic.GROUP)); + } + + @Test + public void testNoGrouping() { + ServiceTraffic traffic = new ServiceTraffic(); + traffic.setName("group-name:service-name:no"); + traffic.setNodeType(NodeType.Normal); + final Map stringObjectMap = new ServiceTraffic.Builder().data2Map(traffic); + Assert.assertNull(stringObjectMap.get(ServiceTraffic.GROUP)); + } +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java index e76cabae6..c538a0ece 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java @@ -62,13 +62,13 @@ public class MetadataQuery implements GraphQLQueryResolver { return new ClusterBrief(); } - public List getAllServices(final Duration duration) throws IOException, ParseException { - return getMetadataQueryService().getAllServices(duration.getStartTimestamp(), duration.getEndTimestamp()); + public List getAllServices(final Duration duration, + final String group) throws IOException, ParseException { + return getMetadataQueryService().getAllServices(group); } public List getAllBrowserServices(final Duration duration) throws IOException, ParseException { - return getMetadataQueryService().getAllBrowserServices( - duration.getStartTimestamp(), duration.getEndTimestamp()); + return getMetadataQueryService().getAllBrowserServices(); } public List searchServices(final Duration duration, diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 92f62f13b..77afd8140 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 92f62f13be3a2bf3dfd007ee85c68d54af2c6149 +Subproject commit 77afd814040c5532e5e4a68f5ef1694a80dad0c4 diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java index 75894f0cf..c044818fc 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; @@ -57,11 +58,14 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { } @Override - public List getAllServices(long startTimestamp, long endTimestamp) throws IOException { + public List getAllServices(final String group) throws IOException { SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceTraffic.NODE_TYPE, NodeType.Normal.value())); + if (StringUtil.isNotEmpty(group)) { + boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceTraffic.GROUP, group)); + } sourceBuilder.query(boolQueryBuilder); sourceBuilder.size(queryMaxSize); @@ -72,7 +76,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { } @Override - public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { + public List getAllBrowserServices() throws IOException { SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); @@ -110,7 +114,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { } @Override - public List searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException { + public List searchServices(String keyword) throws IOException { SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); @@ -232,6 +236,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO { Service service = new Service(); service.setId(serviceTraffic.id()); service.setName(serviceTraffic.getName()); + service.setGroup(serviceTraffic.getGroup()); services.add(service); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java index 33bc7dbc0..3743a24b5 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java @@ -43,6 +43,8 @@ public interface InfluxConstants { String NODE_TYPE = "_node_type"; + String SERVICE_GROUP = "_service_group"; + String SERVICE_ID = "_service_id"; } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java index dfd4476d9..ce4e805f4 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java @@ -62,8 +62,9 @@ public class TableMetaInfo { || EndpointTraffic.INDEX_NAME.equals(model.getName())) { storageAndTagMap.put(EndpointTraffic.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); } else { - // service_traffic name, node_type + // service_traffic name, node_type, group storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); + storageAndTagMap.put(ServiceTraffic.GROUP, InfluxConstants.TagName.SERVICE_GROUP); } } else { diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index ea9a636e0..bdfbb26be 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Objects; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; @@ -50,7 +51,6 @@ import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.SelectSubQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; -import org.influxdb.querybuilder.WhereSubQueryImpl; import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ID_COLUMN; import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.NAME; @@ -67,21 +67,20 @@ public class MetadataQuery implements IMetadataQueryDAO { private final InfluxClient client; @Override - public List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException { - SelectSubQueryImpl subQuery = select() - .fromSubQuery(client.getDatabase()) - .column(ID_COLUMN).column(NAME) - .from(ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))) - .groupBy(TagName.NAME, TagName.NODE_TYPE); - SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); - query.setSubQuery(subQuery); - return buildServices(query); + public List getAllServices(final String group) throws IOException { + final WhereQueryImpl where = select( + ID_COLUMN, NAME, ServiceTraffic.GROUP) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); + if (StringUtil.isNotEmpty(group)) { + where.and(eq(TagName.SERVICE_GROUP, group)); + } + return buildServices(where); } @Override - public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { - WhereQueryImpl query = select(ID_COLUMN, NAME) + public List getAllBrowserServices() throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME, ServiceTraffic.GROUP) .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Browser.value()))); return buildServices(query); @@ -89,14 +88,10 @@ public class MetadataQuery implements IMetadataQueryDAO { @Override public List getAllDatabases() throws IOException { - SelectSubQueryImpl subQuery = select() - .fromSubQuery(client.getDatabase()) - .column(ID_COLUMN).column(NAME) - .from(ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Database.value())) - .groupBy(TagName.NAME, TagName.NODE_TYPE); - SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); - query.setSubQuery(subQuery); + WhereQueryImpl query = select(ID_COLUMN, NAME, ServiceTraffic.GROUP) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Database.value()))); + QueryResult.Series series = client.queryForSingleSeries(query); if (log.isDebugEnabled()) { log.debug("SQL: {} result: {}", query.getCommand(), series); @@ -115,54 +110,44 @@ public class MetadataQuery implements IMetadataQueryDAO { } @Override - public List searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException { - WhereSubQueryImpl, SelectQueryImpl> subQuery = select() - .fromSubQuery(client.getDatabase()) - .column(ID_COLUMN) - .column(NAME) - .from(ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); + public List searchServices(String keyword) throws IOException { + final WhereQueryImpl where = select( + ID_COLUMN, NAME, ServiceTraffic.GROUP) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); if (!Strings.isNullOrEmpty(keyword)) { - subQuery.and(contains(ServiceTraffic.NAME, keyword)); + where.and(contains(ServiceTraffic.NAME, keyword)); } - subQuery.groupBy(TagName.NAME, TagName.NODE_TYPE); - - SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); - query.setSubQuery(subQuery); - return buildServices(query); + return buildServices(where); } @Override public Service searchService(String serviceCode) throws IOException { - WhereQueryImpl query = select(ID_COLUMN, NAME) + WhereQueryImpl where = select( + ID_COLUMN, NAME, ServiceTraffic.GROUP) .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); - query.and(eq(ServiceTraffic.NAME, serviceCode)); - return buildServices(query).get(0); + .where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))) + .and(eq(ServiceTraffic.NAME, serviceCode)); + return buildServices(where).get(0); } @Override public List searchEndpoint(final String keyword, final String serviceId, final int limit) throws IOException { - WhereSubQueryImpl, SelectQueryImpl> subQuery = select() - .fromSubQuery(client.getDatabase()) + final WhereQueryImpl where = select() .column(ID_COLUMN) .column(NAME) - .from(EndpointTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.SERVICE_ID, String.valueOf(serviceId))); + .from(client.getDatabase(), EndpointTraffic.INDEX_NAME) + .where(eq(TagName.SERVICE_ID, String.valueOf(serviceId))); if (!Strings.isNullOrEmpty(keyword)) { - subQuery.where(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/"))); + where.and(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/"))); } - subQuery.groupBy(TagName.NAME, TagName.SERVICE_ID); - SelectQueryImpl query = select(ID_COLUMN, NAME) - .from(client.getDatabase()); - query.setSubQuery(subQuery); - query.limit(limit); + where.limit(limit); - final QueryResult.Series series = client.queryForSingleSeries(query); + final QueryResult.Series series = client.queryForSingleSeries(where); if (log.isDebugEnabled()) { - log.debug("SQL: {} result: {}", query.getCommand(), series); + log.debug("SQL: {} result: {}", where.getCommand(), series); } List list = new ArrayList<>(limit); @@ -249,6 +234,7 @@ public class MetadataQuery implements IMetadataQueryDAO { Service service = new Service(); service.setId((String) values.get(1)); service.setName((String) values.get(2)); + service.setGroup((String) values.get(3)); services.add(service); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java index a22b14cd2..b7c25d461 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java @@ -29,6 +29,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; @@ -55,12 +56,16 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO { } @Override - public List getAllServices(long startTimestamp, long endTimestamp) throws IOException { + public List getAllServices(final String group) throws IOException { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(5); sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where "); sql.append(ServiceTraffic.NODE_TYPE).append("=?"); condition.add(NodeType.Normal.value()); + if (StringUtil.isNotEmpty(group)) { + sql.append(ServiceTraffic.GROUP).append("=?"); + condition.add(group); + } sql.append(" limit ").append(metadataQueryMaxSize); try (Connection connection = h2Client.getConnection()) { @@ -74,7 +79,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO { } @Override - public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { + public List getAllBrowserServices() throws IOException { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(5); sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where "); @@ -118,7 +123,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO { } @Override - public List searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException { + public List searchServices(String keyword) throws IOException { StringBuilder sql = new StringBuilder(); List condition = new ArrayList<>(5); sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where "); @@ -255,6 +260,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO { Service service = new Service(); service.setId(resultSet.getString(H2TableInstaller.ID_COLUMN)); service.setName(resultSet.getString(ServiceTraffic.NAME)); + service.setGroup(resultSet.getString(ServiceTraffic.GROUP)); services.add(service); }