Fix no data bug of oap self observability instance_metrics_second_aggregation metrics. (#5028)

docs/en/setup/backend/grafana.json

@@ -766,7 +766,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(metrics_aggregation {dimensionality=\"min\"} [1m])) by (level)",
+          "expr": "sum(rate(metrics_aggregation {dimensionality=\"minute\"} [1m])) by (level)",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 1,

oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml

@@ -216,7 +216,7 @@ metricsRules:
         range: PT1M
         labelFilter:
           - key: dimensionality
-            options: ["min"]
+            options: ["minute"]
           - key: level
             options: ["1"]
         relabel:
@@ -233,7 +233,7 @@ metricsRules:
         range: PT1M
         labelFilter:
           - key: dimensionality
-            options: ["min"]
+            options: ["minute"]
           - key: level
             options: ["2"]
         relabel:

oap-server/server-bootstrap/src/main/resources/oc-rules/oap.yaml

@@ -207,7 +207,7 @@ metricsRules:
         range: PT1M
         labelFilter:
           - key: dimensionality
-            options: ["min"]
+            options: ["minute"]
           - key: level
             options: ["1"]
         relabel:
@@ -224,7 +224,7 @@ metricsRules:
         range: PT1M
         labelFilter:
           - key: dimensionality
-            options: ["min"]
+            options: ["minute"]
           - key: level
             options: ["2"]
         relabel:

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsAggregateWorker.java

@@ -69,7 +69,7 @@ public class MetricsAggregateWorker extends AbstractWorker<Metrics> {
                                                           .getService(MetricsCreator.class);
         aggregationCounter = metricsCreator.createCounter(
             "metrics_aggregation", "The number of rows in aggregation",
-            new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(modelName, "1", "min")
+            new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(modelName, "1", "minute")
         );
     }
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java

@@ -41,6 +41,10 @@ import org.apache.skywalking.oap.server.core.storage.model.Model;
 import org.apache.skywalking.oap.server.core.worker.AbstractWorker;
 import org.apache.skywalking.oap.server.library.client.request.PrepareRequest;
 import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder;
+import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
+import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
+import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
 
 /**
  * MetricsPersistentWorker is an extension of {@link PersistenceWorker} and focuses on the Metrics data persistent.
@@ -56,6 +60,7 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
     private final Optional<MetricsTransWorker> transWorker;
     private final boolean enableDatabaseSession;
     private final boolean supportUpdate;
+    private CounterMetrics aggregationCounter;
 
     MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO,
                             AbstractWorker<Metrics> nextAlarmWorker, AbstractWorker<ExportEvent> nextExportWorker,
@@ -84,6 +89,14 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
 
         this.dataCarrier = new DataCarrier<>("MetricsPersistentWorker." + model.getName(), name, 1, 2000);
         this.dataCarrier.consume(ConsumerPoolFactory.INSTANCE.get(name), new PersistentConsumer());
+
+        MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME)
+                .provider()
+                .getService(MetricsCreator.class);
+        aggregationCounter = metricsCreator.createCounter(
+                "metrics_aggregation", "The number of rows in aggregation",
+                new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(model.getName(), "2", model.getDownsampling().getName())
+        );
     }
 
     /**
@@ -102,6 +115,7 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
      */
     @Override
     public void in(Metrics metrics) {
+        aggregationCounter.inc();
         dataCarrier.produce(metrics);
     }
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsStreamProcessor.java

@@ -146,7 +146,7 @@ public class MetricsStreamProcessor implements StreamProcessor<Metrics> {
             }
 
             transWorker = new MetricsTransWorker(
-                moduleDefineHolder, stream.getName(), hourPersistentWorker, dayPersistentWorker);
+                moduleDefineHolder, hourPersistentWorker, dayPersistentWorker);
         }
 
         Model model = modelSetter.add(

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsTransWorker.java

@@ -22,10 +22,6 @@ import java.util.Objects;
 import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
 import org.apache.skywalking.oap.server.core.worker.AbstractWorker;
 import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder;
-import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
-import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
-import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
-import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
 
 /**
  * MetricsTransWorker is transferring the metrics for downsampling. All streaming process metrics are in the minute
@@ -36,28 +32,12 @@ public class MetricsTransWorker extends AbstractWorker<Metrics> {
     private final MetricsPersistentWorker hourPersistenceWorker;
     private final MetricsPersistentWorker dayPersistenceWorker;
 
-    private final CounterMetrics aggregationHourCounter;
-    private final CounterMetrics aggregationDayCounter;
-
     public MetricsTransWorker(ModuleDefineHolder moduleDefineHolder,
-                              String modelName,
                               MetricsPersistentWorker hourPersistenceWorker,
                               MetricsPersistentWorker dayPersistenceWorker) {
         super(moduleDefineHolder);
         this.hourPersistenceWorker = hourPersistenceWorker;
         this.dayPersistenceWorker = dayPersistenceWorker;
-
-        MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME)
-                                                          .provider()
-                                                          .getService(MetricsCreator.class);
-        aggregationHourCounter = metricsCreator.createCounter(
-            "metrics_aggregation", "The number of rows in aggregation", new MetricsTag.Keys("metricName", "level",
-                                                                                            "dimensionality"
-            ), new MetricsTag.Values(modelName, "2", "hour"));
-        aggregationDayCounter = metricsCreator.createCounter(
-            "metrics_aggregation", "The number of rows in aggregation", new MetricsTag.Keys("metricName", "level",
-                                                                                            "dimensionality"
-            ), new MetricsTag.Values(modelName, "2", "day"));
     }
 
     /**
@@ -68,11 +48,9 @@ public class MetricsTransWorker extends AbstractWorker<Metrics> {
     @Override
     public void in(Metrics metrics) {
         if (Objects.nonNull(hourPersistenceWorker)) {
-            aggregationHourCounter.inc();
             hourPersistenceWorker.in(metrics.toHour());
         }
         if (Objects.nonNull(dayPersistenceWorker)) {
-            aggregationDayCounter.inc();
             dayPersistenceWorker.in(metrics.toDay());
         }
     }