diff --git a/dist-material/application.yml b/dist-material/application.yml index 8012e4105..4067fefd4 100644 --- a/dist-material/application.yml +++ b/dist-material/application.yml @@ -75,6 +75,10 @@ core: # the metrics may not be accurate within that minute. enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute + # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query, + # and it will cause more load for memory, network of OAP and storage. + # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves. + activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} storage: # elasticsearch: # nameSpace: ${SW_NAMESPACE:""} diff --git a/docs/en/setup/backend/backend-setup.md b/docs/en/setup/backend/backend-setup.md index f006be26f..b755de910 100755 --- a/docs/en/setup/backend/backend-setup.md +++ b/docs/en/setup/backend/backend-setup.md @@ -115,3 +115,15 @@ which format process is timezone related. In default, SkyWalking OAP backend choose the OS default timezone. If you want to override it, please follow Java and OS documents to do so. + +#### How to query the storage directly from 3rd party tool? +SkyWalking provides browser UI, CLI and GraphQL ways to support extensions. But some users may have the idea to query data +directly from the storage. Such as in ElasticSearch case, Kibana is a great tool to do this. + +In default, due to reduce memory, network and storage space usages, SkyWalking saves id(s) only in the entity and metadata saved in the +`*_inventory` entities only. But these tools usually don't support nested query, or don't work conveniently. In this special case, +SkyWalking provide a config to add all necessary name column(s) into the final metrics entities with ID as a trade-off. + +Take a look at `core/default/activeExtraModelColumns` config in the `application.yaml`, and set it as `true` to open this feature. + +This feature wouldn't provide any new feature to the native SkyWalking scenarios, just for the 3rd party integration. diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index a97537b39..3d87b3e09 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -74,6 +74,10 @@ core: # the metrics may not be accurate within that minute. enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute + # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query, + # and it will cause more load for memory, network of OAP and storage. + # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves. + activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} storage: # elasticsearch: # nameSpace: ${SW_NAMESPACE:""} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index c9ceccb19..09cf78c2d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import lombok.Getter; import lombok.Setter; +import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn; import org.apache.skywalking.oap.server.library.module.ModuleConfig; @Getter @@ -103,6 +104,16 @@ public class CoreModuleConfig extends ModuleConfig { */ private int maxSizeOfAnalyzeProfileSnapshot = 12000; + /** + * Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} == + * true. These columns of model are not required logically in aggregation or further query, and it will cause more load for + * memory, network of OAP and storage. + * + * But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party + * tool, such as Kibana->ES, to query the data by themselves. + */ + private boolean activeExtraModelColumns = false; + CoreModuleConfig() { this.downsampling = new ArrayList<>(); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 2034a723e..4175877f5 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -138,6 +138,10 @@ public class CoreModuleProvider extends ModuleProvider { @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { + if (moduleConfig.isActiveExtraModelColumns()) { + DefaultScopeDefine.activeExtraModelColumns(); + } + StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager()); AnnotationScan scopeScan = new AnnotationScan(); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DatabaseAccess.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DatabaseAccess.java index 4c10689b8..1a382e7ab 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DatabaseAccess.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DatabaseAccess.java @@ -42,6 +42,7 @@ public class DatabaseAccess extends Source { private long id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java index cab4cc6db..f6314b231 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.Setter; import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.annotation.AnnotationListener; @@ -78,6 +79,13 @@ public class DefaultScopeDefine { private static final Map SERVICE_INSTANCE_CATALOG = new HashMap<>(); private static final Map ENDPOINT_CATALOG = new HashMap<>(); + @Setter + private static boolean ACTIVE_EXTRA_MODEL_COLUMNS = false; + + public static void activeExtraModelColumns() { + ACTIVE_EXTRA_MODEL_COLUMNS = true; + } + /** * Annotation scan listener */ @@ -99,7 +107,7 @@ public class DefaultScopeDefine { /** * Add a new scope based on the scan result * - * @param declaration includes the definition. + * @param declaration includes the definition. * @param originalClass represents the class having the {@link ScopeDeclaration} annotation */ private static final void addNewScope(ScopeDeclaration declaration, Class originalClass) { @@ -133,11 +141,13 @@ public class DefaultScopeDefine { ScopeDefaultColumn.DefinedByField definedByField = field.getAnnotation( ScopeDefaultColumn.DefinedByField.class); if (definedByField != null) { - scopeDefaultColumns.add( - new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(), - definedByField - .isID() - )); + if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) { + scopeDefaultColumns.add( + new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(), + definedByField + .isID() + )); + } } } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java index e2611d7e3..370907b62 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java @@ -42,6 +42,7 @@ public class Endpoint extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter @@ -49,6 +50,7 @@ public class Endpoint extends Source { private int serviceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java index 973103756..5a87d30f5 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java @@ -43,6 +43,7 @@ public class EndpointRelation extends Source { private int endpointId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name", requireDynamicActive = true) private String endpoint; @Getter @Setter @@ -50,10 +51,10 @@ public class EndpointRelation extends Source { private int serviceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "source_service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "service_instance_id") private int serviceInstanceId; @Getter @Setter @@ -64,6 +65,7 @@ public class EndpointRelation extends Source { private int childEndpointId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name", requireDynamicActive = true) private String childEndpoint; @Getter @Setter @@ -71,10 +73,10 @@ public class EndpointRelation extends Source { private int childServiceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true) private String childServiceName; @Getter @Setter - @ScopeDefaultColumn.DefinedByField(columnName = "child_service_instance_id") private int childServiceInstanceId; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EnvoyInstanceMetric.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EnvoyInstanceMetric.java index f00f8e0f9..60269db4c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EnvoyInstanceMetric.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EnvoyInstanceMetric.java @@ -54,6 +54,7 @@ public class EnvoyInstanceMetric extends Source { private int serviceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java index dfc770d26..45d5d743b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java @@ -51,6 +51,13 @@ public class ScopeDefaultColumn { String columnName(); boolean isID() default false; + + /** + * Dynamic active means this column is only activated through core setting explicitly. + * + * @return + */ + boolean requireDynamicActive() default false; } @Target({ElementType.TYPE}) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java index e997df6f7..5dc390e31 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java @@ -42,6 +42,7 @@ public class Service extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java index 8bb0c652d..851d30ccd 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java @@ -46,9 +46,11 @@ public class ServiceInstance extends Source { private int serviceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java index c73c7c0af..fe710003a 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java @@ -45,9 +45,11 @@ public class ServiceInstanceCLRCPU extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java index 6ee072869..303808697 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java @@ -45,9 +45,11 @@ public class ServiceInstanceCLRGC extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java index fbe04ecac..e3cdeaf8e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java @@ -45,9 +45,11 @@ public class ServiceInstanceCLRThread extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java index 44fc4114f..2ca1dd93c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java @@ -42,9 +42,11 @@ public class ServiceInstanceJVMCPU extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java index 65e513677..3cacd860d 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java @@ -42,9 +42,11 @@ public class ServiceInstanceJVMGC extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java index 80f8b7995..458262942 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java @@ -42,9 +42,11 @@ public class ServiceInstanceJVMMemory extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java index b1498712e..dd20db9e1 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java @@ -42,9 +42,11 @@ public class ServiceInstanceJVMMemoryPool extends Source { private int id; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true) private String serviceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java index 34b0906d6..9523c9a42 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java @@ -48,9 +48,11 @@ public class ServiceInstanceRelation extends Source { private int sourceServiceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "source_service_name", requireDynamicActive = true) private String sourceServiceName; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "source_service_instance_name", requireDynamicActive = true) private String sourceServiceInstanceName; @Getter @Setter @@ -62,9 +64,11 @@ public class ServiceInstanceRelation extends Source { private int destServiceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "dest_service_name", requireDynamicActive = true) private String destServiceName; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "dest_service_instance_name", requireDynamicActive = true) private String destServiceInstanceName; @Getter @Setter diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java index f2016bad0..7a8172b7b 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java @@ -43,6 +43,7 @@ public class ServiceRelation extends Source { private int sourceServiceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "source_name", requireDynamicActive = true) private String sourceServiceName; @Getter @Setter @@ -52,6 +53,7 @@ public class ServiceRelation extends Source { private int destServiceId; @Getter @Setter + @ScopeDefaultColumn.DefinedByField(columnName = "dest_name", requireDynamicActive = true) private String destServiceName; @Getter @Setter