kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Polish eBPF so11y metrics and add error count for query metrics (#12692)

This commit is contained in:
kezhenxu94 2024-10-11 22:49:08 +08:00 committed by GitHub
parent f9d6d950ed
commit 29618d7f2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 88 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/changes/changes.md
View File

@ -6,6 +6,7 @@
* Add self observability metrics for GraphQL query, `graphql_query_latency`.
* Reduce the count of process index and adding time range when query process index.
* Bump up Apache commons-io to 2.17.0.
* Polish eBPF so11y metrics and add error count for query metrics.
#### UI

14
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryHandler.java
View File

@ -20,6 +20,7 @@ package org.apache.skywalking.oap.query.graphql;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
@ -51,6 +52,16 @@ public class GraphQLQueryHandler {
"The processing latency of graphql query",
MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
@Getter(lazy = true)
private final CounterMetrics errorCount =
moduleManager.find(TelemetryModule.NAME)
.provider()
.getService(MetricsCreator.class)
.createCounter(
"graphql_query_error_count",
"The error count of graphql query",
MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
public GraphQLQueryHandler(
final ModuleManager moduleManager,
@ -81,6 +92,9 @@ public class GraphQLQueryHandler {
final HttpRequest req) throws Exception {
try (final var ignored = getQueryHistogram().createTimer()) {
return graphqlService.serve(ctx, req);
} catch (Exception e) {
getErrorCount().inc();
throw e;
}
}
}

16
oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java
View File

@ -91,6 +91,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
protected final NamingControl namingControl;
private final CounterMetrics inCounter;
private final CounterMetrics errorStreamCounter;
private final HistogramMetrics processHistogram;
private final CounterMetrics dropCounter;
private final ConcurrentHashMap<String, DropDataReason> dropReasons = new ConcurrentHashMap<>();
@ -103,13 +104,16 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
.provider()
.getService(MetricsCreator.class);
this.inCounter = metricsCreator.createCounter(
"k8s_als_in_count", "The count of envoy ALS message received", MetricsTag.EMPTY_KEY,
"k8s_als_in_count", "The count of eBPF log entries received", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
this.errorStreamCounter = metricsCreator.createCounter(
"k8s_als_error_streams", "The error count of eBPF log streams that OAP failed to process", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
this.processHistogram = metricsCreator.createHistogramMetric(
"k8s_als_in_latency", "The process latency of envoy ALS message received", MetricsTag.EMPTY_KEY,
"k8s_als_in_latency", "The processing latency of eBPF log streams", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
this.dropCounter = metricsCreator.createCounter(
"k8s_als_drop_count", "The count of envoy ALS message dropped", MetricsTag.EMPTY_KEY,
"k8s_als_drop_count", "The count of eBPF log entries dropped", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE);
// schedule to print the drop reasons(debug log)
@ -125,8 +129,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
@Override
public void onNext(EBPFAccessLogMessage logMessage) {
final HistogramMetrics.Timer timer = processHistogram.createTimer();
try {
try (final var ignored = processHistogram.createTimer()) {
if (isFirst || logMessage.hasNode()) {
isFirst = false;
node = new NodeInfo(logMessage.getNode());
@ -158,8 +161,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
}
} catch (Exception e) {
log.error("Access log service handler process error.", e);
} finally {
timer.finish();
errorStreamCounter.inc();
}
}

8
oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
View File

@ -102,6 +102,10 @@ metricsRules:
exp: k8s_als_drop_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_k8s_als_latency_percentile
exp: k8s_als_in_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
- name: instance_k8s_als_streams
exp: k8s_als_in_latency_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_k8s_als_error_streams
exp: k8s_als_error_streams.sum(['service', 'host_name']).increase('PT1M')
- name: otel_metrics_received
exp: otel_metrics_latency_count.sum(['service', 'host_name']).increase('PT1M')
- name: otel_logs_received
@ -116,3 +120,7 @@ metricsRules:
exp: otel_spans_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,70,90,99])
- name: graphql_query_latency_percentile
exp: graphql_query_latency.sum(['le', 'service', 'host_name']).increase('PT5M').histogram().histogram_percentile([50,70,90,99])
- name: instance_graphql_query_count
exp: graphql_query_latency_count.sum(['service', 'host_name']).increase('PT1M')
- name: instance_graphql_query_error_count
exp: graphql_query_error_count.sum(['service', 'host_name']).increase('PT1M')

66
oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
View File

@ -342,6 +342,33 @@
"unit": "ns"
}
]
},
{
"x": 6,
"y": 39,
"w": 6,
"h": 11,
"i": "17",
"type": "Widget",
"widget": {
"title": "GraphQL Query Count"
},
"graph": {
"type": "Bar",
"showBackground": true
},
"expressions": [
"meter_oap_instance_graphql_query_count",
"meter_oap_instance_graphql_query_error_count"
],
"metricConfig": [
{
"label": "Total Count"
},
{
"label": "Error Count"
}
]
}
]
},
@ -474,7 +501,7 @@
]
},
{
"x": 6,
"x": 12,
"y": 13,
"w": 6,
"h": 13,
@ -496,8 +523,8 @@
]
},
{
"x": 12,
"y": 13,
"x": 6,
"y": 26,
"w": 6,
"h": 13,
"i": "19",
@ -523,8 +550,8 @@
]
},
{
"x": 12,
"y": 26,
"x": 0,
"y": 39,
"w": 6,
"h": 13,
"i": "20",
@ -545,7 +572,7 @@
}
},
{
"x": 0,
"x": 12,
"y": 26,
"w": 6,
"h": 13,
@ -572,8 +599,8 @@
]
},
{
"x": 18,
"y": 13,
"x": 0,
"y": 26,
"w": 6,
"h": 13,
"i": "22",
@ -594,7 +621,7 @@
}
},
{
"x": 6,
"x": 18,
"y": 26,
"w": 6,
"h": 13,
@ -617,7 +644,7 @@
},
{
"x": 18,
"y": 26,
"y": 13,
"w": 6,
"h": 13,
"i": "24",
@ -636,6 +663,25 @@
"widget": {
"title": "OpenTelemetry Logs Latency (ms / min)"
}
},
{
"x": 6,
"y": 13,
"w": 6,
"h": 13,
"i": "25",
"type": "Widget",
"widget": {
"title": "K8S ALS Log Streams (Per Minute)"
},
"graph": {
"type": "Bar",
"showBackground": true
},
"expressions": [
"meter_oap_instance_k8s_als_streams",
"meter_oap_instance_k8s_als_error_streams"
]
}
]
}