Support group in the service traffic and service list query. (#5851)

* Support group in the service traffic and service list query.
This commit is contained in:
吴晟 Wu Sheng 2020-11-17 22:45:32 +08:00 committed by GitHub
parent 9c486a05ee
commit c9202af1ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 184 additions and 73 deletions

View File

@ -32,6 +32,8 @@ Release Notes.
* Support `sideCar.internalErrorCode` in the Service, ServiceInstance, Endpoint, ServiceRelation, and ServiceInstanceRelation sources.
* Improve Kubernetes service registry for ALS analysis.
* Add health checker for cluster management
* Support the service auto grouping.
* Support query service list by the group name.
* Improve the queryable tags generation. Remove the duplicated tags to reduce the storage payload.
* Fix the threads of the Kafka fetcher exit if some unexpected exceptions happen.
* Fix the excessive timeout period set by the kubernetes-client.

View File

@ -108,6 +108,7 @@ or 3rd party configuration management system.
1. [Uninstrumented Gateways](uninstrumented-gateways.md). Configure gateways/proxies that are not supported by SkyWalking agent plugins,
to reflect the delegation in topology graph.
1. [Apdex threshold](apdex-threshold.md). Configure the thresholds for different services if Apdex calculation is activated in the OAL.
1. [Service Grouping](service-auto-grouping.md). An automatic grouping mechanism for all services based on name.
1. [Group Parameterized Endpoints](endpoint-grouping-rules.md). Configure the grouping rules for parameterized endpoints,
to improve the meaning of the metrics.
1. [Meter Analysis](backend-meter.md). Set up the backend analysis rules, when use [SkyWalking Meter System Toolkit](../service-agent/java-agent/README.md#advanced-features)

View File

@ -0,0 +1,16 @@
# Service Auto Grouping
SkyWalking supports various default and customized dashboard templates.
Each template provides the reasonable layout for the services in the particular field.
Such as, services with a language agent installed
could have different metrics with service detected by the service mesh observability solution,
and different with SkyWalking's self-observability metrics dashboard.
Therefore, since 8.3.0, SkyWalking OAP would generate the group based on this simple naming format.
### ${service name} = [${group name}::]${logic name}
Once the service name includes double colons(`::`), the literal string before the colons would be considered as the group name.
In the latest GraphQL query, the group name has been provided as an option parameter.
> getAllServices(duration: Duration!, group: String): [Service!]!
RocketBot UI dashboards(`Standard` type) support the `group name` for default and custom configurations.

View File

@ -39,4 +39,5 @@ public class Const {
public static final String UNKNOWN = "Unknown";
public static final String EMPTY_STRING = "";
public static final String POINT = ".";
public static final String DOUBLE_COLONS_SPLIT = "::";
}

View File

@ -35,6 +35,8 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
import org.apache.skywalking.oap.server.core.storage.StorageBuilder;
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
import static org.apache.skywalking.oap.server.core.Const.DOUBLE_COLONS_SPLIT;
@Stream(name = ServiceTraffic.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE,
builder = ServiceTraffic.Builder.class, processor = MetricsStreamProcessor.class)
@MetricsExtension(supportDownSampling = false, supportUpdate = false)
@ -47,6 +49,7 @@ public class ServiceTraffic extends Metrics {
public static final String NAME = "name";
public static final String NODE_TYPE = "node_type";
public static final String GROUP = "service_group";
@Setter
@Getter
@ -58,6 +61,11 @@ public class ServiceTraffic extends Metrics {
@Column(columnName = NODE_TYPE)
private NodeType nodeType;
@Setter
@Getter
@Column(columnName = GROUP)
private String group;
@Override
public String id() {
return IDManager.ServiceID.buildId(name, nodeType);
@ -93,14 +101,23 @@ public class ServiceTraffic extends Metrics {
ServiceTraffic serviceTraffic = new ServiceTraffic();
serviceTraffic.setName((String) dbMap.get(NAME));
serviceTraffic.setNodeType(NodeType.valueOf(((Number) dbMap.get(NODE_TYPE)).intValue()));
serviceTraffic.setGroup((String) dbMap.get(GROUP));
return serviceTraffic;
}
@Override
public Map<String, Object> data2Map(final ServiceTraffic storageData) {
final String serviceName = storageData.getName();
if (NodeType.Normal.equals(storageData.getNodeType())) {
int groupIdx = serviceName.indexOf(DOUBLE_COLONS_SPLIT);
if (groupIdx > 0) {
storageData.setGroup(serviceName.substring(0, groupIdx));
}
}
Map<String, Object> map = new HashMap<>();
map.put(NAME, storageData.getName());
map.put(NAME, serviceName);
map.put(NODE_TYPE, storageData.getNodeType().value());
map.put(GROUP, storageData.getGroup());
return map;
}
}

View File

@ -46,12 +46,12 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li
return metadataQueryDAO;
}
public List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException {
return getMetadataQueryDAO().getAllServices(startTimestamp, endTimestamp);
public List<Service> getAllServices(final String group) throws IOException {
return getMetadataQueryDAO().getAllServices(group);
}
public List<Service> getAllBrowserServices(final long startTimestamp, final long endTimestamp) throws IOException {
return getMetadataQueryDAO().getAllBrowserServices(startTimestamp, endTimestamp);
public List<Service> getAllBrowserServices() throws IOException {
return getMetadataQueryDAO().getAllBrowserServices();
}
public List<Database> getAllDatabases() throws IOException {
@ -60,7 +60,7 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li
public List<Service> searchServices(final long startTimestamp, final long endTimestamp,
final String keyword) throws IOException {
return getMetadataQueryDAO().searchServices(startTimestamp, endTimestamp, keyword);
return getMetadataQueryDAO().searchServices(keyword);
}
public List<ServiceInstance> getServiceInstances(final long startTimestamp, final long endTimestamp,

View File

@ -26,4 +26,5 @@ import lombok.Setter;
public class Service {
private String id;
private String name;
private String group;
}

View File

@ -27,19 +27,48 @@ import org.apache.skywalking.oap.server.core.query.type.ServiceInstance;
import org.apache.skywalking.oap.server.core.storage.DAO;
public interface IMetadataQueryDAO extends DAO {
List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException;
/**
* @param group group name for filtering.
* @return list of the all available normal services
*/
List<Service> getAllServices(final String group) throws IOException;
List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException;
/**
* @return list of the all available browser services
*/
List<Service> getAllBrowserServices() throws IOException;
/**
* @return list of all conjecture database services.
*/
List<Database> getAllDatabases() throws IOException;
List<Service> searchServices(final long startTimestamp, final long endTimestamp,
final String keyword) throws IOException;
/**
* @param keyword to filter the normal service
* @return the list of normal services matching the given keyword
*/
List<Service> searchServices(final String keyword) throws IOException;
/**
* @param serviceCode to literal match
* @return the service matching the given full name.
*/
Service searchService(final String serviceCode) throws IOException;
/**
* @param keyword to filter the endpoints
* @param serviceId the owner of the endpoints
* @param limit max match size.
* @return list of services matching the given conditions.
*/
List<Endpoint> searchEndpoint(final String keyword, final String serviceId, final int limit) throws IOException;
/**
* @param startTimestamp The instance is required to be live after this timestamp
* @param endTimestamp The instance is required to be live before this timestamp.
* @param serviceId the owner of the instances.
* @return list of instances matching the given conditions.
*/
List<ServiceInstance> getServiceInstances(final long startTimestamp, final long endTimestamp,
final String serviceId) throws IOException;
final String serviceId) throws IOException;
}

View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis.manual.service;
import java.util.Map;
import org.apache.skywalking.oap.server.core.analysis.NodeType;
import org.junit.Assert;
import org.junit.Test;
public class ServiceTrafficTest {
@Test
public void testGrouping() {
ServiceTraffic traffic = new ServiceTraffic();
traffic.setName("group-name::service-name");
traffic.setNodeType(NodeType.Normal);
final Map<String, Object> stringObjectMap = new ServiceTraffic.Builder().data2Map(traffic);
Assert.assertEquals("group-name", stringObjectMap.get(ServiceTraffic.GROUP));
}
@Test
public void testNoGrouping() {
ServiceTraffic traffic = new ServiceTraffic();
traffic.setName("group-name:service-name:no");
traffic.setNodeType(NodeType.Normal);
final Map<String, Object> stringObjectMap = new ServiceTraffic.Builder().data2Map(traffic);
Assert.assertNull(stringObjectMap.get(ServiceTraffic.GROUP));
}
}

View File

@ -62,13 +62,13 @@ public class MetadataQuery implements GraphQLQueryResolver {
return new ClusterBrief();
}
public List<Service> getAllServices(final Duration duration) throws IOException, ParseException {
return getMetadataQueryService().getAllServices(duration.getStartTimestamp(), duration.getEndTimestamp());
public List<Service> getAllServices(final Duration duration,
final String group) throws IOException, ParseException {
return getMetadataQueryService().getAllServices(group);
}
public List<Service> getAllBrowserServices(final Duration duration) throws IOException, ParseException {
return getMetadataQueryService().getAllBrowserServices(
duration.getStartTimestamp(), duration.getEndTimestamp());
return getMetadataQueryService().getAllBrowserServices();
}
public List<Service> searchServices(final Duration duration,

@ -1 +1 @@
Subproject commit 92f62f13be3a2bf3dfd007ee85c68d54af2c6149
Subproject commit 77afd814040c5532e5e4a68f5ef1694a80dad0c4

View File

@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.analysis.NodeType;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
@ -57,11 +58,14 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> getAllServices(long startTimestamp, long endTimestamp) throws IOException {
public List<Service> getAllServices(final String group) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceTraffic.NODE_TYPE, NodeType.Normal.value()));
if (StringUtil.isNotEmpty(group)) {
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceTraffic.GROUP, group));
}
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(queryMaxSize);
@ -72,7 +76,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
public List<Service> getAllBrowserServices() throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
@ -110,7 +114,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException {
public List<Service> searchServices(String keyword) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
@ -232,6 +236,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
Service service = new Service();
service.setId(serviceTraffic.id());
service.setName(serviceTraffic.getName());
service.setGroup(serviceTraffic.getGroup());
services.add(service);
}

View File

@ -43,6 +43,8 @@ public interface InfluxConstants {
String NODE_TYPE = "_node_type";
String SERVICE_GROUP = "_service_group";
String SERVICE_ID = "_service_id";
}
}

View File

@ -62,8 +62,9 @@ public class TableMetaInfo {
|| EndpointTraffic.INDEX_NAME.equals(model.getName())) {
storageAndTagMap.put(EndpointTraffic.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID);
} else {
// service_traffic name, node_type
// service_traffic name, node_type, group
storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE);
storageAndTagMap.put(ServiceTraffic.GROUP, InfluxConstants.TagName.SERVICE_GROUP);
}
} else {

View File

@ -31,6 +31,7 @@ import java.util.Map;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.analysis.NodeType;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
@ -50,7 +51,6 @@ import org.influxdb.dto.QueryResult;
import org.influxdb.querybuilder.SelectQueryImpl;
import org.influxdb.querybuilder.SelectSubQueryImpl;
import org.influxdb.querybuilder.WhereQueryImpl;
import org.influxdb.querybuilder.WhereSubQueryImpl;
import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ID_COLUMN;
import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.NAME;
@ -67,21 +67,20 @@ public class MetadataQuery implements IMetadataQueryDAO {
private final InfluxClient client;
@Override
public List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException {
SelectSubQueryImpl<SelectQueryImpl> subQuery = select()
.fromSubQuery(client.getDatabase())
.column(ID_COLUMN).column(NAME)
.from(ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())))
.groupBy(TagName.NAME, TagName.NODE_TYPE);
SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase());
query.setSubQuery(subQuery);
return buildServices(query);
public List<Service> getAllServices(final String group) throws IOException {
final WhereQueryImpl<SelectQueryImpl> where = select(
ID_COLUMN, NAME, ServiceTraffic.GROUP)
.from(client.getDatabase(), ServiceTraffic.INDEX_NAME)
.where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())));
if (StringUtil.isNotEmpty(group)) {
where.and(eq(TagName.SERVICE_GROUP, group));
}
return buildServices(where);
}
@Override
public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
WhereQueryImpl<SelectQueryImpl> query = select(ID_COLUMN, NAME)
public List<Service> getAllBrowserServices() throws IOException {
WhereQueryImpl<SelectQueryImpl> query = select(ID_COLUMN, NAME, ServiceTraffic.GROUP)
.from(client.getDatabase(), ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Browser.value())));
return buildServices(query);
@ -89,14 +88,10 @@ public class MetadataQuery implements IMetadataQueryDAO {
@Override
public List<Database> getAllDatabases() throws IOException {
SelectSubQueryImpl<SelectQueryImpl> subQuery = select()
.fromSubQuery(client.getDatabase())
.column(ID_COLUMN).column(NAME)
.from(ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Database.value()))
.groupBy(TagName.NAME, TagName.NODE_TYPE);
SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase());
query.setSubQuery(subQuery);
WhereQueryImpl<SelectQueryImpl> query = select(ID_COLUMN, NAME, ServiceTraffic.GROUP)
.from(client.getDatabase(), ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Database.value())));
QueryResult.Series series = client.queryForSingleSeries(query);
if (log.isDebugEnabled()) {
log.debug("SQL: {} result: {}", query.getCommand(), series);
@ -115,54 +110,44 @@ public class MetadataQuery implements IMetadataQueryDAO {
}
@Override
public List<Service> searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException {
WhereSubQueryImpl<SelectSubQueryImpl<SelectQueryImpl>, SelectQueryImpl> subQuery = select()
.fromSubQuery(client.getDatabase())
.column(ID_COLUMN)
.column(NAME)
.from(ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())));
public List<Service> searchServices(String keyword) throws IOException {
final WhereQueryImpl<SelectQueryImpl> where = select(
ID_COLUMN, NAME, ServiceTraffic.GROUP)
.from(client.getDatabase(), ServiceTraffic.INDEX_NAME)
.where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())));
if (!Strings.isNullOrEmpty(keyword)) {
subQuery.and(contains(ServiceTraffic.NAME, keyword));
where.and(contains(ServiceTraffic.NAME, keyword));
}
subQuery.groupBy(TagName.NAME, TagName.NODE_TYPE);
SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase());
query.setSubQuery(subQuery);
return buildServices(query);
return buildServices(where);
}
@Override
public Service searchService(String serviceCode) throws IOException {
WhereQueryImpl<SelectQueryImpl> query = select(ID_COLUMN, NAME)
WhereQueryImpl<SelectQueryImpl> where = select(
ID_COLUMN, NAME, ServiceTraffic.GROUP)
.from(client.getDatabase(), ServiceTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())));
query.and(eq(ServiceTraffic.NAME, serviceCode));
return buildServices(query).get(0);
.where(eq(TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value())))
.and(eq(ServiceTraffic.NAME, serviceCode));
return buildServices(where).get(0);
}
@Override
public List<Endpoint> searchEndpoint(final String keyword,
final String serviceId,
final int limit) throws IOException {
WhereSubQueryImpl<SelectSubQueryImpl<SelectQueryImpl>, SelectQueryImpl> subQuery = select()
.fromSubQuery(client.getDatabase())
final WhereQueryImpl<SelectQueryImpl> where = select()
.column(ID_COLUMN)
.column(NAME)
.from(EndpointTraffic.INDEX_NAME)
.where(eq(InfluxConstants.TagName.SERVICE_ID, String.valueOf(serviceId)));
.from(client.getDatabase(), EndpointTraffic.INDEX_NAME)
.where(eq(TagName.SERVICE_ID, String.valueOf(serviceId)));
if (!Strings.isNullOrEmpty(keyword)) {
subQuery.where(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/")));
where.and(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/")));
}
subQuery.groupBy(TagName.NAME, TagName.SERVICE_ID);
SelectQueryImpl query = select(ID_COLUMN, NAME)
.from(client.getDatabase());
query.setSubQuery(subQuery);
query.limit(limit);
where.limit(limit);
final QueryResult.Series series = client.queryForSingleSeries(query);
final QueryResult.Series series = client.queryForSingleSeries(where);
if (log.isDebugEnabled()) {
log.debug("SQL: {} result: {}", query.getCommand(), series);
log.debug("SQL: {} result: {}", where.getCommand(), series);
}
List<Endpoint> list = new ArrayList<>(limit);
@ -249,6 +234,7 @@ public class MetadataQuery implements IMetadataQueryDAO {
Service service = new Service();
service.setId((String) values.get(1));
service.setName((String) values.get(2));
service.setGroup((String) values.get(3));
services.add(service);
}
}

View File

@ -29,6 +29,7 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oap.server.core.analysis.NodeType;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
@ -55,12 +56,16 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> getAllServices(long startTimestamp, long endTimestamp) throws IOException {
public List<Service> getAllServices(final String group) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where ");
sql.append(ServiceTraffic.NODE_TYPE).append("=?");
condition.add(NodeType.Normal.value());
if (StringUtil.isNotEmpty(group)) {
sql.append(ServiceTraffic.GROUP).append("=?");
condition.add(group);
}
sql.append(" limit ").append(metadataQueryMaxSize);
try (Connection connection = h2Client.getConnection()) {
@ -74,7 +79,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
public List<Service> getAllBrowserServices() throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where ");
@ -118,7 +123,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
@Override
public List<Service> searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException {
public List<Service> searchServices(String keyword) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceTraffic.INDEX_NAME).append(" where ");
@ -255,6 +260,7 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
Service service = new Service();
service.setId(resultSet.getString(H2TableInstaller.ID_COLUMN));
service.setName(resultSet.getString(ServiceTraffic.NAME));
service.setGroup(resultSet.getString(ServiceTraffic.GROUP));
services.add(service);
}