Add Thread and ClassLoader Metrics for the self-observability and otel-oc-rules (#7367)

* Add Thread and ClassLoader Metrics for the self-observability

* update CHANGES.md

* update OC receiver MAL settings

CHANGES.md

@@ -118,7 +118,8 @@ Release Notes.
 * Fix the persistent session timeout mechanism bug.
 * Fix possible version_conflict_engine_exception in bulk execution.
 * Fix PrometheusMetricConverter may throw an `IllegalArgumentException` when convert metrics to SampleFamily
-* Filtering NaN value samples when build SampleFamily  
+* Filtering NaN value samples when build SampleFamily
+* Add Thread and ClassLoader Metrics for the self-observability and otel-oc-rules
 
 #### UI
 

oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml

@@ -79,3 +79,23 @@ metricsRules:
     exp: persistence_timer_bulk_execute_latency_count.sum(['service', 'instance']).increase('PT5M')
   - name: instance_persistence_prepare_count
     exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'instance']).increase('PT5M')
+  - name: jvm_thread_live_count
+    exp: jvm_threads_current.sum(['service', 'instance'])
+  - name: jvm_thread_daemon_count
+    exp: jvm_threads_daemon.sum(['service', 'instance'])
+  - name: jvm_thread_peak_count
+    exp: jvm_threads_peak.sum(['service', 'instance'])
+  - name: jvm_thread_runnable_count
+    exp: jvm_threads_state.tagMatch('state', 'RUNNABLE').sum(['service', 'instance'])
+  - name: jvm_thread_blocked_count
+    exp: jvm_threads_state.tagMatch('state', 'BLOCKED').sum(['service', 'instance'])
+  - name: jvm_thread_waiting_count
+    exp: jvm_threads_state.tagMatch('state', 'WAITING').sum(['service', 'instance'])
+  - name: jvm_thread_timed_waiting_count
+    exp: jvm_threads_state.tagMatch('state', 'TIMED_WAITING').sum(['service', 'instance'])
+  - name: jvm_class_loaded_count
+    exp: jvm_classes_loaded.sum(['service', 'instance'])
+  - name: jvm_class_total_unloaded_count
+    exp: jvm_classes_unloaded_total.sum(['service', 'instance'])
+  - name: jvm_class_total_loaded_count
+    exp: jvm_classes_loaded_total.sum(['service', 'instance'])

oap-server/server-bootstrap/src/main/resources/otel-oc-rules/oap.yaml

@@ -69,3 +69,23 @@ metricsRules:
     exp: persistence_timer_bulk_execute_latency_count.sum(['service', 'host_name']).increase('PT1M')
   - name: instance_persistence_prepare_count
     exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'host_name']).increase('PT1M')
+  - name: jvm_thread_live_count
+    exp: jvm_threads_current.sum(['service', 'host_name'])
+  - name: jvm_thread_daemon_count
+    exp: jvm_threads_daemon.sum(['service', 'host_name'])
+  - name: jvm_thread_peak_count
+    exp: jvm_threads_peak.sum(['service', 'host_name'])
+  - name: jvm_thread_runnable_count
+    exp: jvm_threads_state.tagMatch('state', 'RUNNABLE').sum(['service', 'host_name'])
+  - name: jvm_thread_blocked_count
+    exp: jvm_threads_state.tagMatch('state', 'BLOCKED').sum(['service', 'host_name'])
+  - name: jvm_thread_waiting_count
+    exp: jvm_threads_state.tagMatch('state', 'WAITING').sum(['service', 'host_name'])
+  - name: jvm_thread_timed_waiting_count
+    exp: jvm_threads_state.tagMatch('state', 'TIMED_WAITING').sum(['service', 'host_name'])
+  - name: jvm_class_loaded_count
+    exp: jvm_classes_loaded.sum(['service', 'host_name'])
+  - name: jvm_class_total_unloaded_count
+    exp: jvm_classes_unloaded_total.sum(['service', 'host_name'])
+  - name: jvm_class_total_loaded_count
+    exp: jvm_classes_loaded_total.sum(['service', 'host_name'])

oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/self-observability.yml

@@ -184,6 +184,39 @@ templates:
                   "metricLabels": "50,70,90,99",
                   "labelsIndex": "50,70,90,99",
                   "unit": "Millisecond"
+                },
+                {
+                  "width": 3,
+                  "title": "Thread Count",
+                  "height": "200",
+                  "entityType": "ServiceInstance",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_oap_jvm_thread_live_count,meter_oap_jvm_thread_daemon_count,meter_oap_jvm_thread_peak_count",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartLine"
+                },
+                {
+                  "width": 3,
+                  "title": "Thread State Count",
+                  "height": "200",
+                  "entityType": "ServiceInstance",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_oap_jvm_thread_runnable_count,meter_oap_jvm_thread_blocked_count,meter_oap_jvm_thread_waiting_count,meter_oap_jvm_thread_timed_waiting_count",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartBar"
+                },
+                {
+                  "width": 3,
+                  "title": "Class Count",
+                  "height": "200",
+                  "entityType": "ServiceInstance",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_oap_jvm_class_loaded_count,meter_oap_jvm_class_total_unloaded_count,meter_oap_jvm_class_total_loaded_count",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartArea"
                 }
               ]
             }

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java

@@ -137,13 +137,21 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
     public static String METER_INSTANCE_METRICS_FIRST_AGGREGATION = "meter_oap_instance_metrics_first_aggregation";
     public static String METER_INSTANCE_PERSISTENCE_PREPARE_COUNT = "meter_oap_instance_persistence_prepare_count";
     public static String METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT = "meter_oap_instance_persistence_execute_count";
+    public static String METER_JVM_THREAD_LIVE_COUNT = "meter_oap_jvm_thread_live_count";
+    public static String METER_JVM_THREAD_RUNNABLE_COUNT = "meter_oap_jvm_thread_runnable_count";
+    public static String METER_JVM_CLASS_LOADED_COUNT = "meter_oap_jvm_class_loaded_count";
+    public static String METER_JVM_CLASS_TOTAL_LOADED_COUNT = "meter_oap_jvm_class_total_loaded_count";
 
     public static String[] ALL_SO11Y_LINER_METRICS = {
         METER_INSTANCE_CPU_PERCENTAGE,
         METER_INSTANCE_JVM_MEMORY_BYTES_USED,
         METER_INSTANCE_METRICS_FIRST_AGGREGATION,
         METER_INSTANCE_PERSISTENCE_PREPARE_COUNT,
-        METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT 
+        METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT,
+        METER_JVM_THREAD_LIVE_COUNT,
+        METER_JVM_THREAD_RUNNABLE_COUNT,
+        METER_JVM_CLASS_LOADED_COUNT,
+        METER_JVM_CLASS_TOTAL_LOADED_COUNT
     };
 
     public static String[] ALL_ENVOY_LINER_METRICS = {