diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index c8f88b8b5e..80abde735c 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -6,6 +6,7 @@ * Add self observability metrics for GraphQL query, `graphql_query_latency`. * Reduce the count of process index and adding time range when query process index. * Bump up Apache commons-io to 2.17.0. +* Polish eBPF so11y metrics and add error count for query metrics. #### UI diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java index 8ba0d730dc..bb51696637 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.query.graphql; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; @@ -51,6 +52,16 @@ public class GraphQLQueryHandler { "The processing latency of graphql query", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); + @Getter(lazy = true) + private final CounterMetrics errorCount = + moduleManager.find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class) + .createCounter( + "graphql_query_error_count", + "The error count of graphql query", + MetricsTag.EMPTY_KEY, + MetricsTag.EMPTY_VALUE); public GraphQLQueryHandler( final ModuleManager moduleManager, @@ -81,6 +92,9 @@ public class GraphQLQueryHandler { final HttpRequest req) throws Exception { try (final var ignored = getQueryHistogram().createTimer()) { return graphqlService.serve(ctx, req); + } catch (Exception e) { + getErrorCount().inc(); + throw e; } } } diff --git a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java index 66f277719a..cd1421678a 100644 --- a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java @@ -91,6 +91,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess protected final NamingControl namingControl; private final CounterMetrics inCounter; + private final CounterMetrics errorStreamCounter; private final HistogramMetrics processHistogram; private final CounterMetrics dropCounter; private final ConcurrentHashMap dropReasons = new ConcurrentHashMap<>(); @@ -103,13 +104,16 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess .provider() .getService(MetricsCreator.class); this.inCounter = metricsCreator.createCounter( - "k8s_als_in_count", "The count of envoy ALS message received", MetricsTag.EMPTY_KEY, + "k8s_als_in_count", "The count of eBPF log entries received", MetricsTag.EMPTY_KEY, + MetricsTag.EMPTY_VALUE); + this.errorStreamCounter = metricsCreator.createCounter( + "k8s_als_error_streams", "The error count of eBPF log streams that OAP failed to process", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); this.processHistogram = metricsCreator.createHistogramMetric( - "k8s_als_in_latency", "The process latency of envoy ALS message received", MetricsTag.EMPTY_KEY, + "k8s_als_in_latency", "The processing latency of eBPF log streams", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); this.dropCounter = metricsCreator.createCounter( - "k8s_als_drop_count", "The count of envoy ALS message dropped", MetricsTag.EMPTY_KEY, + "k8s_als_drop_count", "The count of eBPF log entries dropped", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE); // schedule to print the drop reasons(debug log) @@ -125,8 +129,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess @Override public void onNext(EBPFAccessLogMessage logMessage) { - final HistogramMetrics.Timer timer = processHistogram.createTimer(); - try { + try (final var ignored = processHistogram.createTimer()) { if (isFirst || logMessage.hasNode()) { isFirst = false; node = new NodeInfo(logMessage.getNode()); @@ -158,8 +161,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess } } catch (Exception e) { log.error("Access log service handler process error.", e); - } finally { - timer.finish(); + errorStreamCounter.inc(); } } diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml index 485a02d15f..72e6b92284 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml @@ -102,6 +102,10 @@ metricsRules: exp: k8s_als_drop_count.sum(['service', 'host_name']).increase('PT1M') - name: instance_k8s_als_latency_percentile exp: k8s_als_in_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99]) + - name: instance_k8s_als_streams + exp: k8s_als_in_latency_count.sum(['service', 'host_name']).increase('PT1M') + - name: instance_k8s_als_error_streams + exp: k8s_als_error_streams.sum(['service', 'host_name']).increase('PT1M') - name: otel_metrics_received exp: otel_metrics_latency_count.sum(['service', 'host_name']).increase('PT1M') - name: otel_logs_received @@ -116,3 +120,7 @@ metricsRules: exp: otel_spans_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99]) - name: graphql_query_latency_percentile exp: graphql_query_latency.sum(['le', 'service', 'host_name']).increase('PT5M').histogram().histogram_percentile([50,70,90,99]) + - name: instance_graphql_query_count + exp: graphql_query_latency_count.sum(['service', 'host_name']).increase('PT1M') + - name: instance_graphql_query_error_count + exp: graphql_query_error_count.sum(['service', 'host_name']).increase('PT1M') diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json index 34edf346c6..3a9403c260 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json @@ -342,6 +342,33 @@ "unit": "ns" } ] + }, + { + "x": 6, + "y": 39, + "w": 6, + "h": 11, + "i": "17", + "type": "Widget", + "widget": { + "title": "GraphQL Query Count" + }, + "graph": { + "type": "Bar", + "showBackground": true + }, + "expressions": [ + "meter_oap_instance_graphql_query_count", + "meter_oap_instance_graphql_query_error_count" + ], + "metricConfig": [ + { + "label": "Total Count" + }, + { + "label": "Error Count" + } + ] } ] }, @@ -474,7 +501,7 @@ ] }, { - "x": 6, + "x": 12, "y": 13, "w": 6, "h": 13, @@ -496,8 +523,8 @@ ] }, { - "x": 12, - "y": 13, + "x": 6, + "y": 26, "w": 6, "h": 13, "i": "19", @@ -523,8 +550,8 @@ ] }, { - "x": 12, - "y": 26, + "x": 0, + "y": 39, "w": 6, "h": 13, "i": "20", @@ -545,7 +572,7 @@ } }, { - "x": 0, + "x": 12, "y": 26, "w": 6, "h": 13, @@ -572,8 +599,8 @@ ] }, { - "x": 18, - "y": 13, + "x": 0, + "y": 26, "w": 6, "h": 13, "i": "22", @@ -594,7 +621,7 @@ } }, { - "x": 6, + "x": 18, "y": 26, "w": 6, "h": 13, @@ -617,7 +644,7 @@ }, { "x": 18, - "y": 26, + "y": 13, "w": 6, "h": 13, "i": "24", @@ -636,6 +663,25 @@ "widget": { "title": "OpenTelemetry Logs Latency (ms / min)" } + }, + { + "x": 6, + "y": 13, + "w": 6, + "h": 13, + "i": "25", + "type": "Widget", + "widget": { + "title": "K8S ALS Log Streams (Per Minute)" + }, + "graph": { + "type": "Bar", + "showBackground": true + }, + "expressions": [ + "meter_oap_instance_k8s_als_streams", + "meter_oap_instance_k8s_als_error_streams" + ] } ] }