kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add on demand logs support to service mesh (#9171)

This commit is contained in:
kezhenxu94 2022-06-06 13:16:55 +08:00 committed by GitHub
parent f436e8b6c1
commit 8d34c17871
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 132 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apm-protocol/apm-network/src/main/proto

@ -1 +1 @@
Subproject commit e89af18cc7be7dc04ec4a7c4ff75f0ae19fa929b
Subproject commit 8f92ea931a814e0c85e38a741b76c93bf28d296c

3
oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Logs.java
View File

@ -22,11 +22,14 @@ import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
@Setter
@Getter
@Accessors(chain = true)
public class Logs {
private final List<Log> logs;
private String errorReason;
public Logs() {
this.logs = new ArrayList<>();

37
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java
View File

@ -36,6 +36,7 @@ import org.apache.skywalking.oap.query.graphql.type.InternalLog;
import org.apache.skywalking.oap.query.graphql.type.LogAdapter;
import org.apache.skywalking.oap.query.graphql.type.OndemandContainergQueryCondition;
import org.apache.skywalking.oap.query.graphql.type.OndemandLogQueryCondition;
import org.apache.skywalking.oap.query.graphql.type.PodContainers;
import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic.PropertyUtil;
import org.apache.skywalking.oap.server.core.query.input.Duration;
import org.apache.skywalking.oap.server.core.query.type.Attribute;
@ -65,7 +66,7 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
private final MetadataQueryV2 metadataQuery;
public List<String> listContainers(final OndemandContainergQueryCondition condition)
public PodContainers listContainers(final OndemandContainergQueryCondition condition)
throws IOException {
final ServiceInstance instance =
metadataQuery.getInstance(condition.getServiceInstanceId());
@ -104,17 +105,18 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
return attributesMap;
}
public List<String> listContainers(
public PodContainers listContainers(
final String namespace,
final String podName) throws IOException {
try {
if (Strings.isNullOrEmpty(namespace) || Strings.isNullOrEmpty(podName)) {
return Collections.emptyList();
return new PodContainers()
.setErrorReason("namespace and podName can't be null or empty");
}
final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null);
final V1PodSpec spec = pod.getSpec();
if (isNull(spec)) {
return Collections.emptyList();
return new PodContainers().setErrorReason("No pod spec can be found");
}
final List<String> containers = spec.getContainers().stream()
@ -127,13 +129,16 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
containers.addAll(init);
}
return containers;
return new PodContainers().setContainers(containers);
} catch (ApiException e) {
log.error("Failed to list containers from Kubernetes, {}", e.getResponseBody(), e);
Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
String message = responseBody.getOrDefault("message", e.getCode()).toString();
throw new RuntimeException(message);
if (!Strings.isNullOrEmpty(e.getResponseBody())) {
Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
String message = responseBody.getOrDefault("message", e.getCode()).toString();
return new PodContainers().setErrorReason(message);
}
return new PodContainers().setErrorReason(e.getMessage() + ": " + e.getCode());
}
}
@ -141,15 +146,18 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
final String namespace,
final String podName,
final OndemandLogQueryCondition condition) throws IOException {
if (Strings.isNullOrEmpty(namespace) || Strings.isNullOrEmpty(podName)) {
return new Logs().setErrorReason("namespace and podName can't be null or empty");
}
try {
final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null);
final V1ObjectMeta podMetadata = pod.getMetadata();
if (isNull(podMetadata)) {
return new Logs();
return new Logs().setErrorReason("No pod metadata can be found");
}
final V1PodSpec spec = pod.getSpec();
if (isNull(spec)) {
return new Logs();
return new Logs().setErrorReason("No pod spec can be found");
}
final Duration duration = new Duration();
@ -188,9 +196,12 @@ public class OndemandLogQuery implements GraphQLQueryResolver {
} catch (ApiException e) {
log.error("Failed to fetch logs from Kubernetes, {}", e.getResponseBody(), e);
Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
String message = responseBody.getOrDefault("message", e.getCode()).toString();
throw new RuntimeException(message);
if (!Strings.isNullOrEmpty(e.getResponseBody())) {
Map<String, Object> responseBody = gson.fromJson(e.getResponseBody(), responseType);
String message = responseBody.getOrDefault("message", e.getCode()).toString();
return new Logs().setErrorReason(message);
}
return new Logs().setErrorReason(e.getMessage() + ": " + e.getCode());
}
}

34
oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/PodContainers.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.skywalking.oap.query.graphql.type;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
@Getter
@Setter
@Accessors(chain = true)
public class PodContainers {
private String errorReason;
private List<String> containers = new ArrayList<>();
}

2
oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol

@ -1 +1 @@
Subproject commit faf3a768087825258f64dd782a4dc405dc271cf8
Subproject commit 6f23bf5b2b510f93320fe1b086075e062b0e8ea2

21
oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/LogEntry2MetricsAdapter.java
View File

@ -30,6 +30,7 @@ import java.util.List;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.apm.network.common.v3.DetectPoint;
import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
import org.apache.skywalking.apm.network.servicemesh.v3.Protocol;
import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
@ -129,6 +130,26 @@ public class LogEntry2MetricsAdapter {
.map(ServiceMetaInfo::getServiceInstanceName)
.ifPresent(builder::setDestServiceInstance);
Optional
.ofNullable(sourceService)
.map(ServiceMetaInfo::getTags)
.ifPresent(tags -> {
tags.forEach(p -> {
builder.addSourceInstanceProperties(
KeyStringValuePair.newBuilder().setKey(p.getKey()).setValue(p.getValue()));
});
});
Optional
.ofNullable(targetService)
.map(ServiceMetaInfo::getTags)
.ifPresent(tags -> {
tags.forEach(p -> {
builder.addDestInstanceProperties(
KeyStringValuePair.newBuilder().setKey(p.getKey()).setValue(p.getValue()));
});
});
return builder;
}

14
oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/K8SServiceRegistry.java
View File

@ -34,7 +34,7 @@ import io.kubernetes.client.openapi.models.V1Service;
import io.kubernetes.client.openapi.models.V1ServiceList;
import io.kubernetes.client.util.Config;
import java.io.IOException;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@ -262,14 +262,18 @@ public class K8SServiceRegistry {
));
}
protected List<ServiceMetaInfo.KeyValue> transformLabelsToTags(final Map<String, String> labels) {
protected List<ServiceMetaInfo.KeyValue> transformLabelsToTags(final V1ObjectMeta podMeta) {
final Map<String, String> labels = podMeta.getLabels();
final List<ServiceMetaInfo.KeyValue> tags = new ArrayList<>();
tags.add(new ServiceMetaInfo.KeyValue("pod", podMeta.getName()));
tags.add(new ServiceMetaInfo.KeyValue("namespace", podMeta.getNamespace()));
if (isNull(labels)) {
return Collections.emptyList();
return tags;
}
return labels.entrySet()
.stream()
.map(each -> new ServiceMetaInfo.KeyValue(each.getKey(), each.getValue()))
.collect(Collectors.toList());
.collect(Collectors.toCollection(() -> tags));
}
public ServiceMetaInfo findService(final String ip) {
@ -315,7 +319,7 @@ public class K8SServiceRegistry {
}
serviceMetaInfo.setServiceInstanceName(
String.format("%s.%s", podMetadata.getName(), podMetadata.getNamespace()));
serviceMetaInfo.setTags(transformLabelsToTags(podMetadata.getLabels()));
serviceMetaInfo.setTags(transformLabelsToTags(podMetadata));
return serviceMetaInfo;
});

12
oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelper.java
View File

@ -37,6 +37,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.util.ResourceUtils;
import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo.KeyValue;
import org.yaml.snakeyaml.Yaml;
import com.google.common.base.Splitter;
@ -174,6 +175,17 @@ public enum FieldsHelper {
fieldSetterMapping.get(entry.getKey()).accept(serviceMetaInfo, value);
}
}
final Map<String, Value> fieldsMap = metadata.getFieldsMap();
final List<KeyValue> tags = new ArrayList<>();
if (fieldsMap.containsKey("NAME")) {
tags.add(new KeyValue("pod", fieldsMap.get("NAME").getStringValue()));
}
if (fieldsMap.containsKey("NAMESPACE")) {
tags.add(new KeyValue("namespace", fieldsMap.get("NAMESPACE").getStringValue()));
}
if (!tags.isEmpty()) {
serviceMetaInfo.setTags(tags);
}
}
@RequiredArgsConstructor

20
oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java
View File

@ -23,6 +23,7 @@ import org.apache.skywalking.apm.network.servicemesh.v3.Protocol;
import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.Layer;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.config.NamingControl;
@ -32,6 +33,7 @@ import org.apache.skywalking.oap.server.core.source.RequestType;
import org.apache.skywalking.oap.server.core.source.Service;
import org.apache.skywalking.oap.server.core.source.ServiceInstance;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceRelation;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceUpdate;
import org.apache.skywalking.oap.server.core.source.ServiceRelation;
import org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@ -41,6 +43,7 @@ import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;
import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
import com.google.gson.JsonObject;
/**
* TelemetryDataDispatcher processes the {@link ServiceMeshMetric} format telemetry data, transfers it to source
@ -110,6 +113,7 @@ public class TelemetryDataDispatcher {
if (org.apache.skywalking.apm.network.common.v3.DetectPoint.server.equals(metrics.getDetectPoint())) {
toService(metrics, minuteTimeBucket);
toServiceInstance(metrics, minuteTimeBucket);
toServiceInstanceTraffic(metrics, minuteTimeBucket);
toEndpoint(metrics, minuteTimeBucket);
}
@ -187,6 +191,22 @@ public class TelemetryDataDispatcher {
SOURCE_RECEIVER.receive(serviceInstance);
}
private static void toServiceInstanceTraffic(ServiceMeshMetric.Builder metrics, long minuteTimeBucket) {
ServiceInstanceUpdate instanceTraffic = new ServiceInstanceUpdate();
instanceTraffic.setTimeBucket(minuteTimeBucket);
instanceTraffic.setName(metrics.getDestServiceInstance());
instanceTraffic.setServiceId(IDManager.ServiceID.buildId(metrics.getDestServiceName(), true));
if (metrics.getDestInstancePropertiesList() != null) {
final JsonObject properties = new JsonObject();
metrics
.getDestInstancePropertiesList()
.stream()
.forEach(it -> properties.addProperty(it.getKey(), it.getValue()));
instanceTraffic.setProperties(properties);
}
SOURCE_RECEIVER.receive(instanceTraffic);
}
private static void toServiceInstanceRelation(ServiceMeshMetric.Builder metrics, long minuteTimeBucket) {
ServiceInstanceRelation serviceRelation = new ServiceInstanceRelation();
serviceRelation.setTimeBucket(minuteTimeBucket);

8
test/e2e-v2/cases/istio/als/expected/service-instance.yml
View File

@ -16,7 +16,13 @@
{{- contains . }}
- id: {{ notEmpty .id }}
name: {{ notEmpty .name }}
attributes: []
attributes:
{{- contains .attributes }}
- name: pod
value: {{ notEmpty .value }}
- name: namespace
value: {{ notEmpty .value }}
{{- end }}
language: UNKNOWN
instanceuuid: {{ notEmpty .instanceuuid }}
{{- end }}