kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add service type from register and support browser as service type (#4157) 

* Add service type from register and support browser as service type

* Add service type at agent side.

* Fix format.

* Fix a test.

* Fix formats.

Co-authored-by: kezhenxu94 <kezhenxu94@163.com>
This commit is contained in:
吴晟 Wu Sheng 2020-01-01 18:01:41 +08:00 committed by GitHub
parent 9862fe023f
commit 6c20f4a41b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 123 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apm-protocol/apm-network/src/main/proto

@ -1 +1 @@
Subproject commit 7b244ff7ec350910295eee85633e02d92a6f6b1c
Subproject commit c331e840908905460c0d8df7573fa33ea6764873

3
apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/ServiceAndEndpointRegisterClient.java
View File

@ -41,6 +41,7 @@ import org.apache.skywalking.apm.agent.core.os.OSUtil;
import org.apache.skywalking.apm.network.common.Commands;
import org.apache.skywalking.apm.network.common.KeyIntValuePair;
import org.apache.skywalking.apm.network.common.KeyStringValuePair;
import org.apache.skywalking.apm.network.common.ServiceType;
import org.apache.skywalking.apm.network.register.v2.RegisterGrpc;
import org.apache.skywalking.apm.network.register.v2.Service;
import org.apache.skywalking.apm.network.register.v2.ServiceInstance;
@ -141,7 +142,7 @@ public class ServiceAndEndpointRegisterClient implements BootService, Runnable,
if (RemoteDownstreamConfig.Agent.SERVICE_ID == DictionaryUtil.nullValue()) {
if (registerBlockingStub != null) {
ServiceRegisterMapping serviceRegisterMapping = registerBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS).doServiceRegister(
Services.newBuilder().addServices(Service.newBuilder().setServiceName(Config.Agent.SERVICE_NAME)).build());
Services.newBuilder().addServices(Service.newBuilder().setServiceName(Config.Agent.SERVICE_NAME).setType(ServiceType.normal)).build());
if (serviceRegisterMapping != null) {
for (KeyIntValuePair registered : serviceRegisterMapping.getServicesList()) {
if (Config.Agent.SERVICE_NAME.equals(registered.getKey())) {

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java
View File

@ -77,6 +77,10 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li
return getMetadataQueryDAO().getAllServices(startTimestamp, endTimestamp);
}
public List<Service> getAllBrowserServices(final long startTimestamp, final long endTimestamp) throws IOException {
return getMetadataQueryDAO().getAllBrowserServices(startTimestamp, endTimestamp);
}
public List<Database> getAllDatabases() throws IOException {
return getMetadataQueryDAO().getAllDatabases();
}

26
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NodeType.java
View File

@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.core.register;
import org.apache.skywalking.apm.network.common.ServiceType;
import org.apache.skywalking.oap.server.core.UnexpectedException;
/**
@ -32,7 +33,7 @@ public enum NodeType {
/**
* <code>Unknown = 0;</code>
*/
Unknown(0),
Normal(0),
/**
* <code>Database = 1;</code>
*/
@ -53,6 +54,10 @@ public enum NodeType {
* <code>Cache = 5;</code>
*/
Cache(5),
/**
* <code>Browser = 6;</code>
*/
Browser(6),
UNRECOGNIZED(-1);
private final int value;
@ -68,7 +73,7 @@ public enum NodeType {
public static NodeType get(int value) {
switch (value) {
case 0:
return Unknown;
return Normal;
case 1:
return Database;
case 2:
@ -95,4 +100,21 @@ public enum NodeType {
public static NodeType fromSpanLayerValue(int spanLayerValue) {
return get(spanLayerValue);
}
public static NodeType fromRegisterServiceType(ServiceType serviceType) {
switch (serviceType) {
case normal:
return Normal;
case database:
return Database;
case cache:
return Cache;
case mq:
return MQ;
case browser:
return Browser;
default:
return Normal;
}
}
}

5
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java
View File

@ -58,7 +58,10 @@ public class ServiceInventory extends RegisterSource {
@Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING;
@Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress;
@Setter @Getter @Column(columnName = ADDRESS_ID) private int addressId;
@Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType;
/**
* Node type of the service, default {@link NodeType#Normal}
*/
@Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType = 0;
@Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId;
@Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop = Const.EMPTY_JSON_OBJECT_STRING;
@Getter private JsonObject properties;

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java
View File

@ -29,6 +29,8 @@ public interface IServiceInventoryRegister extends Service {
int getOrCreate(String serviceName, JsonObject properties);
int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties);
int getOrCreate(int addressId, String serviceName, JsonObject properties);
void update(int serviceId, NodeType nodeType, JsonObject properties);

7
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java
View File

@ -55,7 +55,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
return serviceInventoryCache;
}
@Override public int getOrCreate(String serviceName, JsonObject properties) {
@Override public int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties) {
int serviceId = getServiceInventoryCache().getServiceId(serviceName);
if (serviceId == Const.NONE) {
@ -67,6 +67,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
long now = System.currentTimeMillis();
serviceInventory.setRegisterTime(now);
serviceInventory.setHeartbeatTime(now);
serviceInventory.setServiceNodeType(nodeType);
serviceInventory.setMappingServiceId(Const.NONE);
serviceInventory.setLastUpdateTime(now);
serviceInventory.setProperties(properties);
@ -76,6 +77,10 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
return serviceId;
}
@Override public int getOrCreate(String serviceName, JsonObject properties) {
return getOrCreate(serviceName, NodeType.Normal, properties);
}
@Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
int serviceId = getServiceInventoryCache().getServiceId(addressId);

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java
View File

@ -36,6 +36,8 @@ public interface IMetadataQueryDAO extends DAO {
List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException;
List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException;
List<Database> getAllDatabases() throws IOException;
List<Service> searchServices(final long startTimestamp, final long endTimestamp,

7
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java
View File

@ -64,6 +64,13 @@ public class MetadataQuery implements GraphQLQueryResolver {
return getMetadataQueryService().getAllServices(startTimestamp, endTimestamp);
}
public List<Service> getAllBrowserServices(final Duration duration) throws IOException, ParseException {
long startTimestamp = DurationUtils.INSTANCE.startTimeToTimestamp(duration.getStep(), duration.getStart());
long endTimestamp = DurationUtils.INSTANCE.endTimeToTimestamp(duration.getStep(), duration.getEnd());
return getMetadataQueryService().getAllBrowserServices(startTimestamp, endTimestamp);
}
public List<Service> searchServices(final Duration duration, final String keyword)
throws IOException, ParseException {
long startTimestamp = DurationUtils.INSTANCE.startTimeToTimestamp(duration.getStep(), duration.getStart());

2
oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol

@ -1 +1 @@
Subproject commit a52c7a957f02a37b9409b52fc55db774f20291bf
Subproject commit a9b48130626e4b4dcf46bb8268c7125cc6f50814

11
oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import org.apache.skywalking.apm.network.common.Commands;
import org.apache.skywalking.apm.network.common.KeyIntValuePair;
import org.apache.skywalking.apm.network.common.KeyStringValuePair;
import org.apache.skywalking.apm.network.common.ServiceType;
import org.apache.skywalking.apm.network.register.v2.EndpointMapping;
import org.apache.skywalking.apm.network.register.v2.EndpointMappingElement;
import org.apache.skywalking.apm.network.register.v2.Endpoints;
@ -41,6 +42,7 @@ import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
import org.apache.skywalking.oap.server.core.register.NodeType;
import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
import org.apache.skywalking.oap.server.core.register.ServiceInventory;
import org.apache.skywalking.oap.server.core.register.service.IEndpointInventoryRegister;
@ -90,7 +92,14 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem
if (logger.isDebugEnabled()) {
logger.debug("Register service, service code: {}", serviceName);
}
int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null);
ServiceType serviceType = service.getType();
if (serviceType == null) {
// All service register from agents before 7.0.0, should be be null.
serviceType = ServiceType.normal;
}
int serviceId = serviceInventoryRegister.getOrCreate(serviceName, NodeType.fromRegisterServiceType(serviceType), null);
if (serviceId != Const.NONE) {
KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build();

4
oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/transform/SpringSleuthSegmentBuilderTest.java
View File

@ -56,6 +56,10 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
}
}
@Override public int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties) {
return getOrCreate(serviceName, properties);
}
@Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
String key = "Address:" + serviceName;
if (applicationRegister.containsKey(key)) {

21
oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java
View File

@ -76,6 +76,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(0);
@ -118,6 +119,25 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(queryMaxSize);
SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
return buildServices(response);
}
@Override
public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Browser.value()));
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(queryMaxSize);
@ -166,6 +186,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
if (!Strings.isNullOrEmpty(keyword)) {
String matchCName = MatchCNameBuilder.INSTANCE.build(ServiceInventory.NAME);

11
oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetadataQueryEs7DAO.java
View File

@ -18,7 +18,9 @@
package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
import java.io.IOException;
import org.apache.skywalking.oap.server.core.register.EndpointInventory;
import org.apache.skywalking.oap.server.core.register.NodeType;
import org.apache.skywalking.oap.server.core.register.ServiceInventory;
import org.apache.skywalking.oap.server.core.source.DetectPoint;
import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@ -29,8 +31,6 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
/**
* @author peng-yongsheng
*/
@ -48,12 +48,13 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(0);
SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
return (int) response.getHits().getTotalHits().value;
return (int)response.getHits().getTotalHits().value;
}
@Override
@ -68,7 +69,7 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
sourceBuilder.size(0);
SearchResponse response = getClient().search(EndpointInventory.INDEX_NAME, sourceBuilder);
return (int) response.getHits().getTotalHits().value;
return (int)response.getHits().getTotalHits().value;
}
@Override
@ -80,7 +81,7 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
return (int) response.getHits().getTotalHits().value;
return (int)response.getHits().getTotalHits().value;
}
}

31
oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java
View File

@ -71,7 +71,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
List<Object> condition = new ArrayList<>(5);
sql.append("select count(*) num from ").append(ServiceInventory.INDEX_NAME).append(" where ");
setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=0");
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=" + BooleanUtils.FALSE);
sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=" + NodeType.Normal.value());
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
@ -131,8 +132,32 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceInventory.INDEX_NAME).append(" where ");
setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=? limit ").append(metadataQueryMaxSize);
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
condition.add(BooleanUtils.FALSE);
sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
condition.add(NodeType.Normal.value());
sql.append(" limit ").append(metadataQueryMaxSize);
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
return buildServices(resultSet);
}
} catch (SQLException e) {
throw new IOException(e);
}
}
@Override
public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceInventory.INDEX_NAME).append(" where ");
setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
condition.add(BooleanUtils.FALSE);
sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
condition.add(NodeType.Browser.value());
sql.append(" limit ").append(metadataQueryMaxSize);
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
@ -185,6 +210,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
condition.add(BooleanUtils.FALSE);
sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
condition.add(NodeType.Normal.value());
if (!Strings.isNullOrEmpty(keyword)) {
sql.append(" and ").append(ServiceInventory.NAME).append(" like \"%").append(keyword).append("%\"");
}