Support name column(s) in the storage for 3rd party integration, like Kibana. Default OFF (#4439)

* Support dynamic column in the source.

* Support activeExtraModelColumns as a default OFF option.

* Fix words

* Update backend-setup.md

Fix format issue.

dist-material/application.yml

@@ -75,6 +75,10 @@ core:
     # the metrics may not be accurate within that minute.
     enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true}
     topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute
+    # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query,
+    # and it will cause more load for memory, network of OAP and storage.
+    # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
+    activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
 storage:
 #  elasticsearch:
 #    nameSpace: ${SW_NAMESPACE:""}

docs/en/setup/backend/backend-setup.md

@@ -115,3 +115,15 @@ which format process is timezone related.
   
 In default, SkyWalking OAP backend choose the OS default timezone.
 If you want to override it, please follow Java and OS documents to do so.
+
+#### How to query the storage directly from 3rd party tool?
+SkyWalking provides browser UI, CLI and GraphQL ways to support extensions. But some users may have the idea to query data 
+directly from the storage. Such as in ElasticSearch case, Kibana is a great tool to do this.
+
+In default, due to reduce memory, network and storage space usages, SkyWalking saves id(s) only in the entity and metadata saved in the
+`*_inventory` entities only. But these tools usually don't support nested query, or don't work conveniently. In this special case,
+SkyWalking provide a config to add all necessary name column(s) into the final metrics entities with ID as a trade-off.
+
+Take a look at `core/default/activeExtraModelColumns` config in the `application.yaml`, and set it as `true` to open this feature.
+
+This feature wouldn't provide any new feature to the native SkyWalking scenarios, just for the 3rd party integration.

oap-server/server-bootstrap/src/main/resources/application.yml

@@ -74,6 +74,10 @@ core:
     # the metrics may not be accurate within that minute.
     enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true}
     topNReportPeriod: ${SW_CORE_TOPN_REPORT_PERIOD:10} # top_n record worker report cycle, unit is minute
+    # Extra model column are the column defined by in the codes, These columns of model are not required logically in aggregation or further query,
+    # and it will cause more load for memory, network of OAP and storage.
+    # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
+    activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
 storage:
 #  elasticsearch:
 #    nameSpace: ${SW_NAMESPACE:""}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java

@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.source.ScopeDefaultColumn;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 
 @Getter
@@ -103,6 +104,16 @@ public class CoreModuleConfig extends ModuleConfig {
      */
     private int maxSizeOfAnalyzeProfileSnapshot = 12000;
 
+    /**
+     * Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} ==
+     * true. These columns of model are not required logically in aggregation or further query, and it will cause more load for
+     * memory, network of OAP and storage.
+     *
+     * But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party
+     * tool, such as Kibana->ES, to query the data by themselves.
+     */
+    private boolean activeExtraModelColumns = false;
+
     CoreModuleConfig() {
         this.downsampling = new ArrayList<>();
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java

@@ -138,6 +138,10 @@ public class CoreModuleProvider extends ModuleProvider {
 
     @Override
     public void prepare() throws ServiceNotProvidedException, ModuleStartException {
+        if (moduleConfig.isActiveExtraModelColumns()) {
+            DefaultScopeDefine.activeExtraModelColumns();
+        }
+
         StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager());
 
         AnnotationScan scopeScan = new AnnotationScan();

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DatabaseAccess.java

@@ -42,6 +42,7 @@ public class DatabaseAccess extends Source {
     private long id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java

@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import lombok.Setter;
 import org.apache.skywalking.oap.server.core.UnexpectedException;
 import org.apache.skywalking.oap.server.core.annotation.AnnotationListener;
 
@@ -78,6 +79,13 @@ public class DefaultScopeDefine {
     private static final Map<Integer, Boolean> SERVICE_INSTANCE_CATALOG = new HashMap<>();
     private static final Map<Integer, Boolean> ENDPOINT_CATALOG = new HashMap<>();
 
+    @Setter
+    private static boolean ACTIVE_EXTRA_MODEL_COLUMNS = false;
+
+    public static void activeExtraModelColumns() {
+        ACTIVE_EXTRA_MODEL_COLUMNS = true;
+    }
+
     /**
      * Annotation scan listener
      */
@@ -99,7 +107,7 @@ public class DefaultScopeDefine {
     /**
      * Add a new scope based on the scan result
      *
-     * @param declaration includes the definition.
+     * @param declaration   includes the definition.
      * @param originalClass represents the class having the {@link ScopeDeclaration} annotation
      */
     private static final void addNewScope(ScopeDeclaration declaration, Class originalClass) {
@@ -133,11 +141,13 @@ public class DefaultScopeDefine {
                 ScopeDefaultColumn.DefinedByField definedByField = field.getAnnotation(
                     ScopeDefaultColumn.DefinedByField.class);
                 if (definedByField != null) {
-                    scopeDefaultColumns.add(
-                        new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(),
-                                               definedByField
-                                                   .isID()
-                        ));
+                    if (!definedByField.requireDynamicActive() || ACTIVE_EXTRA_MODEL_COLUMNS) {
+                        scopeDefaultColumns.add(
+                            new ScopeDefaultColumn(field.getName(), definedByField.columnName(), field.getType(),
+                                                   definedByField
+                                                       .isID()
+                            ));
+                    }
                 }
             }
         }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java

@@ -42,6 +42,7 @@ public class Endpoint extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
@@ -49,6 +50,7 @@ public class Endpoint extends Source {
     private int serviceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java

@@ -43,6 +43,7 @@ public class EndpointRelation extends Source {
     private int endpointId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name", requireDynamicActive = true)
     private String endpoint;
     @Getter
     @Setter
@@ -50,10 +51,10 @@ public class EndpointRelation extends Source {
     private int serviceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "source_service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "service_instance_id")
     private int serviceInstanceId;
     @Getter
     @Setter
@@ -64,6 +65,7 @@ public class EndpointRelation extends Source {
     private int childEndpointId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name", requireDynamicActive = true)
     private String childEndpoint;
     @Getter
     @Setter
@@ -71,10 +73,10 @@ public class EndpointRelation extends Source {
     private int childServiceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true)
     private String childServiceName;
     @Getter
     @Setter
-    @ScopeDefaultColumn.DefinedByField(columnName = "child_service_instance_id")
     private int childServiceInstanceId;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EnvoyInstanceMetric.java

@@ -54,6 +54,7 @@ public class EnvoyInstanceMetric extends Source {
     private int serviceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ScopeDefaultColumn.java

@@ -51,6 +51,13 @@ public class ScopeDefaultColumn {
         String columnName();
 
         boolean isID() default false;
+
+        /**
+         * Dynamic active means this column is only activated through core setting explicitly.
+         *
+         * @return
+         */
+        boolean requireDynamicActive() default false;
     }
 
     @Target({ElementType.TYPE})

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java

@@ -42,6 +42,7 @@ public class Service extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java

@@ -46,9 +46,11 @@ public class ServiceInstance extends Source {
     private int serviceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRCPU.java

@@ -45,9 +45,11 @@ public class ServiceInstanceCLRCPU extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRGC.java

@@ -45,9 +45,11 @@ public class ServiceInstanceCLRGC extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceCLRThread.java

@@ -45,9 +45,11 @@ public class ServiceInstanceCLRThread extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMCPU.java

@@ -42,9 +42,11 @@ public class ServiceInstanceJVMCPU extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMGC.java

@@ -42,9 +42,11 @@ public class ServiceInstanceJVMGC extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemory.java

@@ -42,9 +42,11 @@ public class ServiceInstanceJVMMemory extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceJVMMemoryPool.java

@@ -42,9 +42,11 @@ public class ServiceInstanceJVMMemoryPool extends Source {
     private int id;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
     private String serviceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java

@@ -48,9 +48,11 @@ public class ServiceInstanceRelation extends Source {
     private int sourceServiceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "source_service_name", requireDynamicActive = true)
     private String sourceServiceName;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "source_service_instance_name", requireDynamicActive = true)
     private String sourceServiceInstanceName;
     @Getter
     @Setter
@@ -62,9 +64,11 @@ public class ServiceInstanceRelation extends Source {
     private int destServiceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "dest_service_name", requireDynamicActive = true)
     private String destServiceName;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "dest_service_instance_name", requireDynamicActive = true)
     private String destServiceInstanceName;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java

@@ -43,6 +43,7 @@ public class ServiceRelation extends Source {
     private int sourceServiceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "source_name", requireDynamicActive = true)
     private String sourceServiceName;
     @Getter
     @Setter
@@ -52,6 +53,7 @@ public class ServiceRelation extends Source {
     private int destServiceId;
     @Getter
     @Setter
+    @ScopeDefaultColumn.DefinedByField(columnName = "dest_name", requireDynamicActive = true)
     private String destServiceName;
     @Getter
     @Setter