From 9cc5e76d97230c4cb19acff43606091741e7a0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Fri, 11 Jan 2019 21:24:51 +0800 Subject: [PATCH] Support service inventory extension fields in json format (#2148) * Add properties for service inventory * Fix code bugs * Provide develop doc for inventory extension. --- docs/en/guides/README.md | 2 + docs/en/guides/inventory-extension.md | 31 ++++++++++ .../register/NetworkAddressInventory.java | 11 ++++ .../core/register/ServiceInventory.java | 29 +++++++++ .../INetworkAddressInventoryRegister.java | 3 +- .../service/IServiceInventoryRegister.java | 7 ++- .../NetworkAddressInventoryRegister.java | 9 ++- .../service/ServiceInventoryRegister.java | 22 ++++++- .../mesh/ServiceMeshMetricDataDecorator.java | 4 +- .../v5/grpc/ApplicationRegisterHandler.java | 2 +- .../NetworkAddressRegisterServiceHandler.java | 2 +- .../ApplicationRegisterServletHandler.java | 2 +- .../NetworkAddressRegisterServletHandler.java | 2 +- .../v6/grpc/RegisterServiceHandler.java | 6 +- .../parser/decorator/SpanDecorator.java | 26 +++++++- .../standardization/ReferenceIdExchanger.java | 2 +- .../standardization/SpanIdExchanger.java | 62 ++++++++++++++++--- .../zipkin/handler/SpanProcessor.java | 2 +- .../zipkin/transform/SegmentBuilder.java | 2 +- .../SpringSleuthSegmentBuilderTest.java | 10 ++- 20 files changed, 202 insertions(+), 34 deletions(-) create mode 100644 docs/en/guides/inventory-extension.md diff --git a/docs/en/guides/README.md b/docs/en/guides/README.md index 26f003366..af3c1926e 100644 --- a/docs/en/guides/README.md +++ b/docs/en/guides/README.md @@ -37,6 +37,8 @@ and private plugin developer should read this. - [Storage extension development guide](storage-extention.md). Help potential contributors to build a new storage implementor besides the official. - [Customize analysis by oal script](write-oal.md). Guide you to use oal script to make your own metric available. +- [Backend Inventory entity extension](inventory-extension.md). If you want to extend SkyWalking inventory entities, and +want to push upstream back to our Apache OSS repo, please read these principles. ### UI developer Our UI is constituted by static pages and web container. diff --git a/docs/en/guides/inventory-extension.md b/docs/en/guides/inventory-extension.md new file mode 100644 index 000000000..cee70eb28 --- /dev/null +++ b/docs/en/guides/inventory-extension.md @@ -0,0 +1,31 @@ +# Backend Inventory Entity Extension +SkyWalking includes four inventory entities. +- Service Inventory +- Service Instance Inventory +- Endpoint Inventory +- Network Address Inventory + +All metric, topology, trace and alarm are related to these entity IDs. + +For understanding the **Service**, **Service Instance** and **Endpoint** concepts, +please read [Project Overview](../concepts-and-designs/overview.md#why-use-skywalking). + +For **Network Address Inventory**, it represents all network address, in IP:port, hostname, domain name +formats, which are detected by language agents or other probes. + +## Extension +Right now, only **Service Inventory** extension is already supported in backend core. +Service provides field `properties` in Json format, which is usually used for specific service +rather than normal business services, such as Database, Cache, MQ, etc. + +For keeping code consistent and friendly in query and visualization, the Json properties +need to follow the rules. + +### Database +1. NodeType == **Database(1)** +1. Json properties include following keys. + - `database`. Database name, such as MySQL, PostgreSQL + - `db.type`. Database type, such as sql db, redis db. + - `db.instance`. Database instance name. + + diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java index 0647a2c38..3fc26cde1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java @@ -82,6 +82,17 @@ public class NetworkAddressInventory extends RegisterSource { return true; } + public NetworkAddressInventory getClone() { + NetworkAddressInventory inventory = new NetworkAddressInventory(); + inventory.setSequence(getSequence()); + inventory.setRegisterTime(getRegisterTime()); + inventory.setHeartbeatTime(getHeartbeatTime()); + inventory.setName(name); + inventory.setNodeType(nodeType); + + return inventory; + } + @Override public void combine(RegisterSource registerSource) { super.combine(registerSource); NetworkAddressInventory inventory = (NetworkAddressInventory)registerSource; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java index 068d7b44f..676ba52b5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register; +import com.google.gson.*; import java.util.*; import lombok.*; import org.apache.skywalking.oap.server.core.Const; @@ -46,6 +47,7 @@ public class ServiceInventory extends RegisterSource { public static final String NODE_TYPE = "node_type"; public static final String MAPPING_SERVICE_ID = "mapping_service_id"; public static final String MAPPING_LAST_UPDATE_TIME = "mapping_last_update_time"; + public static final String PROPERTIES = "properties"; @Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING; @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress; @@ -53,6 +55,8 @@ public class ServiceInventory extends RegisterSource { @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType; @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId; @Setter @Getter @Column(columnName = MAPPING_LAST_UPDATE_TIME) private long mappingLastUpdateTime; + @Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop; + @Getter private JsonObject properties; public NodeType getServiceNodeType() { return NodeType.get(this.nodeType); @@ -86,6 +90,24 @@ public class ServiceInventory extends RegisterSource { return result; } + public void setProperties(JsonObject properties) { + this.properties = properties; + if (properties != null && properties.keySet().size() > 0) { + this.prop = properties.toString(); + } + } + + private void setProp(String prop) { + this.prop = prop; + if (!Strings.isNullOrEmpty(prop)) { + this.properties = new Gson().fromJson(prop, JsonObject.class); + } + } + + public boolean hasProperties() { + return prop != null && prop.length() > 0; + } + public ServiceInventory getClone() { ServiceInventory inventory = new ServiceInventory(); inventory.setSequence(getSequence()); @@ -97,6 +119,7 @@ public class ServiceInventory extends RegisterSource { inventory.setAddressId(addressId); inventory.setMappingLastUpdateTime(mappingLastUpdateTime); inventory.setMappingServiceId(mappingServiceId); + inventory.setProp(prop); return inventory; } @@ -133,6 +156,7 @@ public class ServiceInventory extends RegisterSource { remoteBuilder.addDataLongs(getMappingLastUpdateTime()); remoteBuilder.addDataStrings(Strings.isNullOrEmpty(name) ? Const.EMPTY_STRING : name); + remoteBuilder.addDataStrings(Strings.isNullOrEmpty(prop) ? Const.EMPTY_STRING : prop); return remoteBuilder; } @@ -148,6 +172,8 @@ public class ServiceInventory extends RegisterSource { setMappingLastUpdateTime(remoteData.getDataLongs(2)); setName(remoteData.getDataStrings(0)); + setProp(remoteData.getDataStrings(1)); + } @Override public int remoteHashCode() { @@ -158,6 +184,7 @@ public class ServiceInventory extends RegisterSource { super.combine(registerSource); ServiceInventory serviceInventory = (ServiceInventory)registerSource; nodeType = serviceInventory.nodeType; + setProp(serviceInventory.getProp()); if (Const.NONE != serviceInventory.getMappingServiceId() && serviceInventory.getMappingLastUpdateTime() >= this.getMappingLastUpdateTime()) { this.mappingServiceId = serviceInventory.getMappingServiceId(); this.mappingLastUpdateTime = serviceInventory.getMappingLastUpdateTime(); @@ -177,6 +204,7 @@ public class ServiceInventory extends RegisterSource { inventory.setRegisterTime((Long)dbMap.get(REGISTER_TIME)); inventory.setHeartbeatTime((Long)dbMap.get(HEARTBEAT_TIME)); inventory.setMappingLastUpdateTime((Long)dbMap.get(MAPPING_LAST_UPDATE_TIME)); + inventory.setProp((String)dbMap.get(PROPERTIES)); return inventory; } @@ -191,6 +219,7 @@ public class ServiceInventory extends RegisterSource { map.put(REGISTER_TIME, storageData.getRegisterTime()); map.put(HEARTBEAT_TIME, storageData.getHeartbeatTime()); map.put(MAPPING_LAST_UPDATE_TIME, storageData.getMappingLastUpdateTime()); + map.put(PROPERTIES, storageData.getProp()); return map; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java index d5f190ff6..32e78ea52 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import org.apache.skywalking.oap.server.core.register.NodeType; import org.apache.skywalking.oap.server.library.module.Service; @@ -25,7 +26,7 @@ import org.apache.skywalking.oap.server.library.module.Service; * @author peng-yongsheng */ public interface INetworkAddressInventoryRegister extends Service { - int getOrCreate(String networkAddress); + int getOrCreate(String networkAddress, JsonObject properties); int get(String networkAddress); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java index 398149c1e..c4d7ef02f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import org.apache.skywalking.oap.server.library.module.Service; /** @@ -25,9 +26,11 @@ import org.apache.skywalking.oap.server.library.module.Service; */ public interface IServiceInventoryRegister extends Service { - int getOrCreate(String serviceName); + int getOrCreate(String serviceName, JsonObject properties); - int getOrCreate(int addressId, String serviceName); + int getOrCreate(int addressId, String serviceName, JsonObject properties); + + void updateProperties(int serviceId, JsonObject properties); void heartbeat(int serviceId, long heartBeatTime); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java index 18d05b200..391029971 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import java.util.Objects; import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.cache.*; @@ -73,11 +74,11 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory return this.serviceInstanceInventoryRegister; } - @Override public int getOrCreate(String networkAddress) { + @Override public int getOrCreate(String networkAddress, JsonObject properties) { int addressId = getNetworkAddressInventoryCache().getAddressId(networkAddress); if (addressId != Const.NONE) { - int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress); + int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress, properties); if (serviceId != Const.NONE) { int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, addressId, System.currentTimeMillis()); @@ -107,6 +108,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory @Override public void heartbeat(int addressId, long heartBeatTime) { NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); if (Objects.nonNull(networkAddress)) { + networkAddress = networkAddress.getClone(); networkAddress.setHeartbeatTime(heartBeatTime); InventoryProcess.INSTANCE.in(networkAddress); @@ -119,7 +121,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); if (!this.compare(networkAddress, nodeType)) { - NetworkAddressInventory newNetworkAddress = getNetworkAddressInventoryCache().get(addressId); + NetworkAddressInventory newNetworkAddress = networkAddress.getClone(); newNetworkAddress.setNetworkAddressNodeType(nodeType); newNetworkAddress.setHeartbeatTime(System.currentTimeMillis()); @@ -128,6 +130,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ServiceInventory newServiceInventory = getServiceInventoryCache().get(getServiceInventoryCache().getServiceId(networkAddress.getSequence())); if (!this.compare(newServiceInventory, nodeType)) { + newServiceInventory = newServiceInventory.getClone(); newServiceInventory.setServiceNodeType(nodeType); newServiceInventory.setHeartbeatTime(System.currentTimeMillis()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java index 04b60dba7..25e06ab7d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java @@ -18,10 +18,11 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import java.util.Objects; import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; -import org.apache.skywalking.oap.server.core.register.ServiceInventory; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.worker.InventoryProcess; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.util.BooleanUtils; @@ -50,7 +51,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { return serviceInventoryCache; } - @Override public int getOrCreate(String serviceName) { + @Override public int getOrCreate(String serviceName, JsonObject properties) { int serviceId = getServiceInventoryCache().getServiceId(serviceName); if (serviceId == Const.NONE) { @@ -64,13 +65,14 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { serviceInventory.setHeartbeatTime(now); serviceInventory.setMappingServiceId(Const.NONE); serviceInventory.setMappingLastUpdateTime(now); + serviceInventory.setProperties(properties); InventoryProcess.INSTANCE.in(serviceInventory); } return serviceId; } - @Override public int getOrCreate(int addressId, String serviceName) { + @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) { int serviceId = getServiceInventoryCache().getServiceId(addressId); if (serviceId == Const.NONE) { @@ -89,9 +91,23 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { return serviceId; } + @Override public void updateProperties(int serviceId, JsonObject properties) { + ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId); + if (Objects.nonNull(serviceInventory)) { + serviceInventory = serviceInventory.getClone(); + serviceInventory.setProperties(properties); + serviceInventory.setMappingLastUpdateTime(System.currentTimeMillis()); + + InventoryProcess.INSTANCE.in(serviceInventory); + } else { + logger.warn("Service {} properties update, but not found in storage."); + } + } + @Override public void heartbeat(int serviceId, long heartBeatTime) { ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId); if (Objects.nonNull(serviceInventory)) { + serviceInventory = serviceInventory.getClone(); serviceInventory.setHeartbeatTime(heartBeatTime); InventoryProcess.INSTANCE.in(serviceInventory); diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java index 486195587..abdbb10be 100644 --- a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java +++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java @@ -44,7 +44,7 @@ public class ServiceMeshMetricDataDecorator { boolean isRegistered = true; sourceServiceId = origin.getSourceServiceId(); if (sourceServiceId == Const.NONE) { - sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName()); + sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName(), null); if (sourceServiceId != Const.NONE) { getNewDataBuilder().setSourceServiceId(sourceServiceId); } else { @@ -65,7 +65,7 @@ public class ServiceMeshMetricDataDecorator { } destServiceId = origin.getDestServiceId(); if (destServiceId == Const.NONE) { - destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName()); + destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName(), null); if (destServiceId != Const.NONE) { getNewDataBuilder().setDestServiceId(destServiceId); } else { diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java index 44eec839a..f0e2b5ef1 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java @@ -47,7 +47,7 @@ public class ApplicationRegisterHandler extends ApplicationRegisterServiceGrpc.A ApplicationMapping.Builder builder = ApplicationMapping.newBuilder(); String serviceName = request.getApplicationCode(); - int serviceId = serviceInventoryRegister.getOrCreate(serviceName); + int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null); if (serviceId != Const.NONE) { KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(serviceName).setValue(serviceId).build(); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java index 5c88d63fe..a1adb9e57 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java @@ -50,7 +50,7 @@ public class NetworkAddressRegisterServiceHandler extends NetworkAddressRegister NetworkAddressMappings.Builder builder = NetworkAddressMappings.newBuilder(); for (String networkAddress : addressesList) { - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); if (addressId != Const.NONE) { KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(networkAddress).setValue(addressId).build(); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java index 49f4f35fe..b08512bc9 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java @@ -57,7 +57,7 @@ public class ApplicationRegisterServletHandler extends JettyJsonHandler { JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class); for (int i = 0; i < applicationCodes.size(); i++) { String applicationCode = applicationCodes.get(i).getAsString(); - int applicationId = serviceInventoryRegister.getOrCreate(applicationCode); + int applicationId = serviceInventoryRegister.getOrCreate(applicationCode, null); JsonObject mapping = new JsonObject(); mapping.addProperty(APPLICATION_CODE, applicationCode); mapping.addProperty(APPLICATION_ID, applicationId); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java index d1a463213..b873d08e4 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java @@ -62,7 +62,7 @@ public class NetworkAddressRegisterServletHandler extends JettyJsonHandler { logger.debug("network getAddress register, network getAddress: {}", networkAddress); } - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); JsonObject mapping = new JsonObject(); mapping.addProperty(ADDRESS_ID, addressId); mapping.addProperty(NETWORK_ADDRESS, networkAddress); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java index 05c11d171..f31f27fae 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java @@ -63,7 +63,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem if (logger.isDebugEnabled()) { logger.debug("Register service, service code: {}", serviceName); } - int serviceId = serviceInventoryRegister.getOrCreate(serviceName); + int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null); if (serviceId != Const.NONE) { KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build(); @@ -149,7 +149,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem NetAddressMapping.Builder builder = NetAddressMapping.newBuilder(); request.getAddressesList().forEach(networkAddress -> { - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); if (addressId != Const.NONE) { builder.addAddressIds(KeyIntValuePair.newBuilder().setKey(networkAddress).setValue(addressId)); @@ -186,7 +186,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem return; } - networkAddressId = networkAddressInventoryRegister.getOrCreate(address); + networkAddressId = networkAddressInventoryRegister.getOrCreate(address, null); if (networkAddressId == Const.NONE) { return; } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java index fb4cf364c..76fefa6f1 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java @@ -18,9 +18,9 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator; -import org.apache.skywalking.apm.network.language.agent.SpanLayer; -import org.apache.skywalking.apm.network.language.agent.SpanObject; -import org.apache.skywalking.apm.network.language.agent.SpanType; +import java.util.*; +import org.apache.skywalking.apm.network.common.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.*; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; import static java.util.Objects.isNull; @@ -281,6 +281,14 @@ public class SpanDecorator implements StandardBuilder { return referenceDecorators[index]; } + public List getAllTags() { + if (isOrigin) { + return isV2 ? spanObjectV2.getTagsList() : convert(spanObject.getTagsList()); + } else { + return isV2 ? spanBuilderV2.getTagsList() : convert(spanBuilder.getTagsList()); + } + } + @Override public void toBuilder() { if (this.isOrigin) { this.isOrigin = false; @@ -292,4 +300,16 @@ public class SpanDecorator implements StandardBuilder { standardBuilder.toBuilder(); } } + + private List convert(List list) { + List result = new ArrayList<>(); + if (list != null) { + list.forEach(element -> { + result.add(KeyStringValuePair.newBuilder() + .setKey(element.getKey()) + .setValue(element.getValue()).build()); + }); + } + return result; + } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java index 31d6368bf..c41253c45 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java @@ -88,7 +88,7 @@ public class ReferenceIdExchanger implements IdExchanger { } if (standardBuilder.getNetworkAddressId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getNetworkAddress())) { - int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress()); + int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress(), null); if (networkAddressId == 0) { if (logger.isDebugEnabled()) { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java index 8a1af6b74..f9bcf9c87 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java @@ -19,9 +19,14 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.parser.standardization; import com.google.common.base.Strings; +import com.google.gson.JsonObject; +import java.util.List; +import org.apache.skywalking.apm.network.common.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.SpanLayer; import org.apache.skywalking.oap.server.core.*; +import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; -import org.apache.skywalking.oap.server.core.register.NodeType; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.service.*; import org.apache.skywalking.oap.server.core.source.DetectPoint; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -36,6 +41,8 @@ public class SpanIdExchanger implements IdExchanger { private static final Logger logger = LoggerFactory.getLogger(SpanIdExchanger.class); private static SpanIdExchanger EXCHANGER; + private final ServiceInventoryCache serviceInventoryCacheDAO; + private final IServiceInventoryRegister serviceInventoryRegister; private final IEndpointInventoryRegister endpointInventoryRegister; private final INetworkAddressInventoryRegister networkAddressInventoryRegister; private final IComponentLibraryCatalogService componentLibraryCatalogService; @@ -48,6 +55,8 @@ public class SpanIdExchanger implements IdExchanger { } private SpanIdExchanger(ModuleManager moduleManager) { + this.serviceInventoryCacheDAO = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class); + this.serviceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInventoryRegister.class); this.endpointInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IEndpointInventoryRegister.class); this.networkAddressInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(INetworkAddressInventoryRegister.class); this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class); @@ -69,10 +78,11 @@ public class SpanIdExchanger implements IdExchanger { } } + int peerId = 0; if (standardBuilder.getPeerId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getPeer())) { - int peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer()); + peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer(), buildServiceProperties(standardBuilder)); - if (peerId == 0) { + if (peerId == Const.NONE) { if (logger.isDebugEnabled()) { logger.debug("peer: {} in service: {} exchange failed", standardBuilder.getPeer(), serviceId); } @@ -81,13 +91,27 @@ public class SpanIdExchanger implements IdExchanger { standardBuilder.toBuilder(); standardBuilder.setPeerId(peerId); standardBuilder.setPeer(Const.EMPTY_STRING); - - int spanLayerValue = standardBuilder.getSpanLayerValue(); - networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue)); } } - if (standardBuilder.getOperationNameId() == 0) { + if (peerId != Const.NONE) { + int spanLayerValue = standardBuilder.getSpanLayerValue(); + networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue)); + + /** + * In some case, conjecture node, such as Database node, could be registered by agents. + * At here, if the target service properties need to be updated, + * it will only be updated at the first time for now. + */ + if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) { + ServiceInventory newServiceInventory = serviceInventoryCacheDAO.get(serviceInventoryCacheDAO.getServiceId(peerId)); + if (!newServiceInventory.hasProperties()) { + serviceInventoryRegister.updateProperties(newServiceInventory.getSequence(), buildServiceProperties(standardBuilder)); + } + } + } + + if (standardBuilder.getOperationNameId() == Const.NONE) { String endpointName = Strings.isNullOrEmpty(standardBuilder.getOperationName()) ? Const.DOMAIN_OPERATION_NAME : standardBuilder.getOperationName(); int endpointId = endpointInventoryRegister.getOrCreate(serviceId, endpointName, DetectPoint.fromSpanType(standardBuilder.getSpanType())); @@ -104,4 +128,28 @@ public class SpanIdExchanger implements IdExchanger { } return true; } + + private JsonObject buildServiceProperties(SpanDecorator standardBuilder) { + JsonObject properties = new JsonObject(); + if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) { + List tags = standardBuilder.getAllTags(); + tags.forEach(tag -> { + if ("db.type".equals(tag.getKey())) { + properties.addProperty("type", tag.getValue()); + } else if ("db.instance".equals(tag.getKey())) { + properties.addProperty("instance", tag.getValue()); + } + }); + String componentName; + int id = standardBuilder.getComponentId(); + if (id != Const.NONE) { + componentName = componentLibraryCatalogService.getServerNameBasedOnComponent(id); + } else { + componentName = "UNKNOWN"; + } + properties.addProperty("database", componentName); + } + + return properties; + } } diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java index f5e97be51..48774f923 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java @@ -48,7 +48,7 @@ public class SpanProcessor { // In Zipkin, the local service name represents the application owner. String applicationCode = span.localServiceName(); if (applicationCode != null) { - int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode); + int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode, null); if (applicationId != 0) { CoreRegisterLinker.getServiceInstanceInventoryRegister().getOrCreate(applicationId, applicationCode, applicationCode, span.timestampAsLong(), diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java index be1f1bb6e..3f687a6cc 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java @@ -301,7 +301,7 @@ public class SegmentBuilder { private Segment addApp(String serviceCode, long registerTime) throws Exception { int serviceId = waitForExchange(() -> - CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode), + CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode, null), 10 ); diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java index 1fbdeb3ea..553904605 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.receiver.zipkin.transform; +import com.google.gson.JsonObject; import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.LinkedList; @@ -27,7 +28,7 @@ import org.apache.skywalking.apm.network.language.agent.SpanType; import org.apache.skywalking.apm.network.language.agent.v2.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v2.SegmentReference; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; -import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister; import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister; import org.apache.skywalking.oap.server.receiver.zipkin.CoreRegisterLinker; @@ -52,7 +53,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { public void testTransform() throws Exception { IServiceInventoryRegister applicationIDService = new IServiceInventoryRegister() { - @Override public int getOrCreate(String serviceName) { + @Override public int getOrCreate(String serviceName, JsonObject properties) { String key = "AppCode:" + serviceName; if (applicationRegister.containsKey(key)) { return applicationRegister.get(key); @@ -63,7 +64,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { } } - @Override public int getOrCreate(int addressId, String serviceName) { + @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) { String key = "Address:" + serviceName; if (applicationRegister.containsKey(key)) { return applicationRegister.get(key); @@ -74,6 +75,9 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { } } + @Override public void updateProperties(int serviceId, JsonObject properties) { + } + @Override public void heartbeat(int serviceId, long heartBeatTime) { }