kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support sort queries on metrics generated by eBPF receiver (#12539)

This commit is contained in:
mrproliu 2024-08-19 09:34:58 +08:00 committed by GitHub
parent 4b0a4f2668
commit 3320bf91ee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 50 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/changes/changes.md
View File

@ -51,6 +51,7 @@
* Fix format the endpoint name with empty string.
* Support async query for the composite GraphQL query.
* Get endpoint list order by timestamp desc.
* Support sort queries on metrics generated by eBPF receiver.
#### UI

17
docs/en/concepts-and-designs/scope-definitions.md
View File

@ -529,9 +529,9 @@ following **package**/**protocol** level metric contents.
| destServiceInstanceName | The dest pod name in kubernetes. | | string |
| destLayer | The dest layer service in kubernetes. | | string |
#### Endpoint and Endpoint Relation
#### Endpoint
For `K8SEndpoint` and `K8SEndpointRelation`, they only have the following **protocol** level metric contents.
For `K8SEndpoint`, they only have the following **protocol** level metric contents.
| Name | Remarks | Group Key | Type |
|-------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|-----------|----------------------|
@ -555,19 +555,6 @@ For `K8SEndpoint` and `K8SEndpointRelation`, they only have the following **prot
| endpointName | The endpoint name detect in kubernetes service. | | string |
| duration | The duration of the service endpoint response latency. | | long |
##### SCOPE `K8SEndpointRelation`
| Name | Remarks | Group Key | Type |
|--------------------|--------------------------------------------------------------------|-----------|--------|
| sourceServiceName | The source service name in kubernetes. | | string |
| sourceServiceName | The layer in kubernetes source service. | | string |
| sourceEndpointName | The endpoint name detect in kubernetes source service. | | string |
| detectPoint | Where the relation is detected. The value may be client or server. | | enum |
| componentId | The ID of component used in this call. | | string |
| destServiceName | The dest service name in kubernetes. | | string |
| destServiceName | The layer in kubernetes dest service. | | string |
| destEndpointName | The endpoint name detect in kubernetes dest service. | | string |
### SCOPES with `Cilium` Prefix
All metrics starting with `Cilium` are derived from Cilium monitoring by Cilium Hubble.

1
oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
View File

@ -57,7 +57,6 @@ SRC_K8S_SERVICE_INSTANCE: 'K8SServiceInstance';
SRC_K8S_ENDPOINT: 'K8SEndpoint';
SRC_K8S_SERVICE_RELATION: 'K8SServiceRelation';
SRC_K8S_SERVICE_INSTANCE_RELATION: 'K8SServiceInstanceRelation';
SRC_K8S_ENDPOINT_RELATION: 'K8SEndpointRelation';
SRC_CILIUM_SERVICE: 'CiliumService';
SRC_CILIUM_SERVICE_INSTANCE: 'CiliumServiceInstance';
SRC_CILIUM_ENDPOINT: 'CiliumEndpoint';

2
oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
View File

@ -58,7 +58,7 @@ source
SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF |
SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC |
SRC_EVENT | SRC_MQ_ACCESS | SRC_MQ_ENDPOINT_ACCESS |
SRC_K8S_SERVICE | SRC_K8S_SERVICE_INSTANCE | SRC_K8S_ENDPOINT | SRC_K8S_SERVICE_RELATION | SRC_K8S_SERVICE_INSTANCE_RELATION | SRC_K8S_ENDPOINT_RELATION |
SRC_K8S_SERVICE | SRC_K8S_SERVICE_INSTANCE | SRC_K8S_ENDPOINT | SRC_K8S_SERVICE_RELATION | SRC_K8S_SERVICE_INSTANCE_RELATION |
SRC_CILIUM_SERVICE | SRC_CILIUM_SERVICE_INSTANCE | SRC_CILIUM_ENDPOINT | SRC_CILIUM_SERVICE_RELATION | SRC_CILIUM_SERVICE_INSTANCE_RELATION | SRC_CILIUM_ENDPOINT_RELATION
;

46
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/relation/endpoint/K8SEndpointCallRelationDispatcher.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis.manual.relation.endpoint;
import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
import org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.source.K8SEndpointRelation;
public class K8SEndpointCallRelationDispatcher implements SourceDispatcher<K8SEndpointRelation> {
@Override
public void dispatch(K8SEndpointRelation source) {
switch (source.getDetectPoint()) {
case SERVER:
serverSide(source);
break;
default:
}
}
private void serverSide(K8SEndpointRelation source) {
EndpointRelationServerSideMetrics metrics = new EndpointRelationServerSideMetrics();
metrics.setTimeBucket(source.getTimeBucket());
metrics.setSourceEndpoint(source.getSourceEndpointId());
metrics.setDestEndpoint(source.getDestEndpointId());
metrics.setComponentId(source.getComponentId());
metrics.setEntityId(source.getEntityId());
MetricsStreamProcessor.getInstance().in(metrics);
}
}

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
View File

@ -135,7 +135,6 @@ public class DefaultScopeDefine {
public static final int K8S_SERVICE_RELATION = 74;
public static final int K8S_SERVICE_INSTANCE_RELATION = 75;
public static final int K8S_ENDPOINT = 76;
public static final int K8S_ENDPOINT_REALATION = 77;
public static final int CILIUM_SERVICE = 78;
public static final int CILIUM_SERVICE_INSTANCE = 79;

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpoint.java
View File

@ -31,9 +31,12 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.K8
public class K8SEndpoint extends K8SMetrics.ProtocolMetrics {
private volatile String entityId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_id", groupByCondInTopN = true)
private String serviceId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
private String serviceName;
private Layer layer;
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String endpointName;
private long duration;

70
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SEndpointRelation.java
View File

@ -1,70 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.source;
import lombok.Data;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.Layer;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_RELATION_CATALOG_NAME;
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.K8S_ENDPOINT_REALATION;
@Data
@ScopeDeclaration(id = K8S_ENDPOINT_REALATION, name = "K8SEndpointRelation", catalog = ENDPOINT_RELATION_CATALOG_NAME)
@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class)
public class K8SEndpointRelation extends K8SMetrics.ProtocolMetrics {
private volatile String entityId;
private String sourceServiceId;
private String sourceServiceName;
private Layer sourceLayer;
private String sourceEndpointId;
private String sourceEndpointName;
private DetectPoint detectPoint;
private int componentId;
private String destServiceId;
private String destServiceName;
private Layer destLayer;
private String destEndpointId;
private String destEndpointName;
@Override
public int scope() {
return K8S_ENDPOINT_REALATION;
}
@Override
public String getEntityId() {
return entityId;
}
@Override
public void prepare() {
sourceServiceId = IDManager.ServiceID.buildId(sourceServiceName, sourceLayer.isNormal());
sourceEndpointId = IDManager.EndpointID.buildId(sourceServiceId, sourceEndpointName);
destServiceId = IDManager.ServiceID.buildId(destServiceName, destLayer.isNormal());
destEndpointId = IDManager.EndpointID.buildId(destServiceId, destEndpointName);
entityId = IDManager.EndpointID.buildRelationId(new IDManager.EndpointID.EndpointRelationDefine(
sourceServiceId, sourceEndpointName, destServiceId, destEndpointName
));
}
}

1
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SService.java
View File

@ -32,6 +32,7 @@ public class K8SService extends K8SMetrics {
private volatile String entityId;
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
public Layer layer;

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SServiceInstance.java
View File

@ -31,8 +31,11 @@ import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SE
public class K8SServiceInstance extends K8SMetrics {
private volatile String entityId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
private String serviceId;
@ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
private String serviceName;
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String serviceInstanceName;
public Layer layer;
private DetectPoint detectPoint;

4
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SServiceInstanceRelation.java
View File

@ -33,16 +33,20 @@ public class K8SServiceInstanceRelation extends K8SMetrics {
private volatile String entityId;
private String sourceServiceId;
@ScopeDefaultColumn.DefinedByField(columnName = "source_service_name", requireDynamicActive = true)
private String sourceServiceName;
private String sourceServiceInstanceId;
@ScopeDefaultColumn.DefinedByField(columnName = "source_service_instance_name", requireDynamicActive = true)
private String sourceServiceInstanceName;
private Layer sourceLayer;
private DetectPoint detectPoint;
private String destServiceId;
@ScopeDefaultColumn.DefinedByField(columnName = "dest_service_name", requireDynamicActive = true)
private String destServiceName;
private String destServiceInstanceId;
@ScopeDefaultColumn.DefinedByField(columnName = "dest_service_instance_name", requireDynamicActive = true)
private String destServiceInstanceName;
private Layer destLayer;

2
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/K8SServiceRelation.java
View File

@ -34,6 +34,7 @@ public class K8SServiceRelation extends K8SMetrics {
private volatile String entityId;
private String sourceServiceId;
@ScopeDefaultColumn.DefinedByField(columnName = "source_name", requireDynamicActive = true)
private String sourceServiceName;
private Layer sourceLayer;
@ -42,6 +43,7 @@ public class K8SServiceRelation extends K8SMetrics {
private AccessLogConnectionTLSMode tlsMode;
private String destServiceId;
@ScopeDefaultColumn.DefinedByField(columnName = "dest_name", requireDynamicActive = true)
private String destServiceName;
private Layer destLayer;

23
oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/AccessLogServiceHandler.java
View File

@ -52,7 +52,6 @@ import org.apache.skywalking.oap.server.core.analysis.Layer;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.source.K8SEndpoint;
import org.apache.skywalking.oap.server.core.source.K8SEndpointRelation;
import org.apache.skywalking.oap.server.core.source.K8SMetrics;
import org.apache.skywalking.oap.server.core.source.K8SService;
import org.apache.skywalking.oap.server.core.source.K8SServiceInstance;
@ -332,7 +331,7 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
// endpoint, endpoint relation
final String endpointName = buildProtocolEndpointName(connection, protocolLog);
Stream.of(connection.toEndpoint(endpointName, success, duration), connection.toEndpointRelation(endpointName, success))
Stream.of(connection.toEndpoint(endpointName, success, duration))
.filter(Objects::nonNull)
.forEach(metric -> {
metric.setType(protocol.getType());
@ -648,26 +647,6 @@ public class AccessLogServiceHandler extends EBPFAccessLogServiceGrpc.EBPFAccess
return endpoint;
}
public K8SEndpointRelation toEndpointRelation(String endpointName, boolean success) {
final Tuple2<KubernetesProcessAddress, KubernetesProcessAddress> tuple = convertSourceAndDestAddress();
final String sourceServiceName = buildServiceNameByAddress(nodeInfo, tuple._1);
final String destServiceName = buildServiceNameByAddress(nodeInfo, tuple._2);
final K8SEndpointRelation endpointRelation = new K8SEndpointRelation();
endpointRelation.setSourceServiceName(sourceServiceName);
endpointRelation.setSourceEndpointName(namingControl.formatEndpointName(sourceServiceName, endpointName));
endpointRelation.setSourceLayer(Layer.K8S_SERVICE);
endpointRelation.setDetectPoint(parseToSourceRole());
endpointRelation.setComponentId(buildComponentId());
endpointRelation.setDestServiceName(destServiceName);
endpointRelation.setDestEndpointName(namingControl.formatEndpointName(destServiceName, endpointName));
endpointRelation.setDestLayer(Layer.K8S_SERVICE);
endpointRelation.setSuccess(success);
return endpointRelation;
}
private int buildComponentId() {
boolean isTLS = tlsMode == AccessLogConnectionTLSMode.TLS;
switch (protocolType) {

7
test/e2e-v2/cases/profiling/ebpf/access_log/accesslog-cases.yaml
View File

@ -32,9 +32,6 @@ cases:
# service endpoints
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql endpoint list --service-name=details.default
expected: expected/service-endpoint-reviews.yml
# endpoints topology
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dependency endpoint --service-name=details.default --endpoint-name=/details/0
expected: expected/dependency-endpoint-reviews.yml
# service level metrics
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_connect_cpm --service-name=productpage.default
@ -91,12 +88,16 @@ cases:
instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name reviews.default | yq '.[0].name' -)
swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_http_call_cpm --service-name=reviews.default --instance-name=$instance_name
expected: expected/metrics-has-value.yml
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression="top_n(kubernetes_service_instance_write_l4_duration,10,des)" --service-name=details.default
expected: expected/metrics-sorted-has-value.yml
# service endpoint level metrics
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_endpoint_call_cpm --service-name=details.default --endpoint-name=/details/0
expected: expected/metrics-has-value.yml
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_endpoint_http_call_cpm --service-name=details.default --endpoint-name=/details/0
expected: expected/metrics-has-value.yml
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression="top_n(kubernetes_service_endpoint_call_duration,10,des)" --service-name=details.default
expected: expected/metrics-sorted-has-value.yml
# service relation metrics
- query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_connect_cpm --service-name=productpage.default --dest-service-name=details.default

40
test/e2e-v2/cases/profiling/ebpf/access_log/expected/dependency-endpoint-reviews.yml
View File

@ -1,40 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nodes:
{{- contains .nodes }}
- id: {{ b64enc "productpage.default" }}.1_{{ b64enc "/details/0" }}
name: /details/0
serviceid: {{ b64enc "productpage.default" }}.1
servicename: productpage.default
type: ""
isreal: true
- id: {{ b64enc "details.default" }}.1_{{ b64enc "/details/0" }}
name: /details/0
serviceid: {{ b64enc "details.default" }}.1
servicename: details.default
type: ""
isreal: true
{{- end }}
calls:
{{- contains .calls }}
- source: {{ b64enc "productpage.default" }}.1_{{ b64enc "/details/0" }}
sourcecomponents: []
target: {{ b64enc "details.default" }}.1_{{ b64enc "/details/0" }}
targetcomponents: []
id: {{ b64enc "productpage.default" }}.1-{{ b64enc "/details/0" }}-{{ b64enc "details.default" }}.1-{{ b64enc "/details/0" }}
detectpoints:
- SERVER
{{- end }}

28
test/e2e-v2/cases/profiling/ebpf/access_log/expected/metrics-sorted-has-value.yml Normal file
View File

@ -0,0 +1,28 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type: SORTED_LIST
results:
{{- contains .results }}
- metric:
labels: []
values:
{{- contains .values }}
- id: {{ notEmpty .id }}
value: {{ .value }}
traceid: null
{{- end}}
{{- end}}
error: null