diff --git a/docs/en/setup/backend/on-demand-pod-log.md b/docs/en/setup/backend/on-demand-pod-log.md index 7fd6e5ef6b..c8c32ff9f6 100644 --- a/docs/en/setup/backend/on-demand-pod-log.md +++ b/docs/en/setup/backend/on-demand-pod-log.md @@ -11,8 +11,13 @@ feature is disabled by default, please read the configuration documentation to e As the name indicates, this feature only works for Kubernetes Pods. -SkyWalking OAP lists the Kubernetes namespaces, services, Pods and containers in the UI for users to select, -users can select the same and UI should fetch the logs in a given interval and display the logs in UI. +SkyWalking OAP collects and saves the service instance's namespace and Pod name in ther serivce instance's +properties, named `namespace` and `pod`, users can select the same and UI should fetch the logs by service +instance in a given interval and display the logs in UI, OAP receives the query and checks the instance's +properties and use the `namespace` and `pod` to locate the Pod and query the logs. + +If you want to register a service instance that has on demand logs available, you should add `namespace` +and `pod` in the service instance properties, so that you can query the real time logs from that Pod. That said, in order to make this feature work properly, you should in advance configure the cluster role for OAP to list/get namespaces, services, pods and pods/log. diff --git a/docs/menu.yml b/docs/menu.yml index 720c9b080e..2a004e38ed 100644 --- a/docs/menu.yml +++ b/docs/menu.yml @@ -134,7 +134,7 @@ catalog: - name: "Log Collecting And Analysis" path: "/en/setup/backend/log-analyzer" - name: "On Demand Pod Logs" - path: "/en/setup/backend/on-demand-pod-logg" + path: "/en/setup/backend/on-demand-pod-log" - name: "Extension" catalog: - name: "Metrics Exporter" diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java index a36c29168d..ef38bc995f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/instance/InstanceTraffic.java @@ -175,6 +175,17 @@ public class InstanceTraffic extends Metrics { } public static class PropertyUtil { + /** + * `namespace` and `pod` are key properties that help "on demand Pod logs" + * to locate the corresponding Pod in Kubernetes, when language agent is + * registering a new service instance that is supposed to work in terms of + * "on demand Pod logs", the agent should also fill in these 2 properties. + * + * @since 9.1.0 + */ + public static final String NAMESPACE = "namespace"; + public static final String POD = "pod"; + public static final String LANGUAGE = "language"; public static final String IPV4 = "ipv4"; public static final String IPV4S = "ipv4s"; diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java index 919fdbb038..f4f694a96e 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/GraphQLQueryProvider.java @@ -78,6 +78,7 @@ public class GraphQLQueryProvider extends ModuleProvider { @Override public void prepare() throws ServiceNotProvidedException { + final MetadataQueryV2 metadataQueryV2 = new MetadataQueryV2(getManager()); schemaBuilder.file("query-protocol/common.graphqls") .resolvers(new Query(), new Mutation(), new HealthQuery(getManager())) .file("query-protocol/metadata.graphqls") @@ -118,14 +119,14 @@ public class GraphQLQueryProvider extends ModuleProvider { .file("query-protocol/event.graphqls") .resolvers(new EventQuery(getManager())) .file("query-protocol/metadata-v2.graphqls") - .resolvers(new MetadataQueryV2(getManager())) + .resolvers(metadataQueryV2) .file("query-protocol/ebpf-profiling.graphqls") .resolvers(new EBPFProcessProfilingQuery(getManager()), new EBPFProcessProfilingMutation(getManager())); if (config.isEnableOnDemandPodLog()) { schemaBuilder .file("query-protocol/ondemand-pod-log.graphqls") - .resolvers(new OndemandLogQuery()); + .resolvers(new OndemandLogQuery(metadataQueryV2)); } schemaBuilder.scalars(ExtendedScalars.GraphQLLong); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java index 458572ba16..f453a81355 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/OndemandLogQuery.java @@ -23,9 +23,9 @@ import static java.util.Objects.isNull; import static java.util.Objects.nonNull; import java.io.IOException; import java.lang.reflect.Type; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import com.google.common.base.Predicate; import com.google.common.base.Splitter; @@ -34,64 +34,87 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.skywalking.oap.query.graphql.type.InternalLog; import org.apache.skywalking.oap.query.graphql.type.LogAdapter; +import org.apache.skywalking.oap.query.graphql.type.OndemandContainergQueryCondition; import org.apache.skywalking.oap.query.graphql.type.OndemandLogQueryCondition; -import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic.PropertyUtil; import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Attribute; import org.apache.skywalking.oap.server.core.query.type.Log; import org.apache.skywalking.oap.server.core.query.type.Logs; +import org.apache.skywalking.oap.server.core.query.type.ServiceInstance; import org.apache.skywalking.oap.server.library.util.StringUtil; import graphql.kickstart.tools.GraphQLQueryResolver; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; -import io.kubernetes.client.openapi.models.V1Namespace; -import io.kubernetes.client.openapi.models.V1NamespaceList; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1Pod; import io.kubernetes.client.openapi.models.V1PodSpec; import io.kubernetes.client.util.Config; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @Slf4j +@RequiredArgsConstructor public class OndemandLogQuery implements GraphQLQueryResolver { private final Gson gson = new Gson(); private final Type responseType = new TypeToken>() { }.getType(); private CoreV1Api kApi; - public List listNamespaces() throws IOException { - try { - final V1NamespaceList nsList = - kApi().listNamespace(null, null, null, null, null, null, null, null, null, null); - return nsList - .getItems() - .stream() - .map(V1Namespace::getMetadata) - .filter(Objects::nonNull) - .map(V1ObjectMeta::getName) - .collect(Collectors.toList()); - } catch (ApiException e) { - log.error("Failed to list namespaces from Kubernetes, {}", e.getResponseBody(), e); + private final MetadataQueryV2 metadataQuery; - Map responseBody = gson.fromJson(e.getResponseBody(), responseType); - String message = responseBody.getOrDefault("message", e.getCode()).toString(); - throw new RuntimeException(message); - } + public List listContainers(final OndemandContainergQueryCondition condition) + throws IOException { + final ServiceInstance instance = + metadataQuery.getInstance(condition.getServiceInstanceId()); + final Map attributesMap = convertInstancePropertiesToMap(instance); + final String ns = attributesMap.get(PropertyUtil.NAMESPACE); + final String pod = attributesMap.get(PropertyUtil.POD); + return listContainers(ns, pod); } - public List listContainers(final OndemandLogQueryCondition condition) + public Logs ondemandPodLogs(final OndemandLogQueryCondition condition) throws IOException { - final String ns = condition.getNamespace(); - final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = - IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId()); - final String instanceName = instanceIDDefinition.getName(); + final ServiceInstance instance = + metadataQuery.getInstance(condition.getServiceInstanceId()); + final Map attributesMap = convertInstancePropertiesToMap(instance); + final String ns = attributesMap.get(PropertyUtil.NAMESPACE); + final String pod = attributesMap.get(PropertyUtil.POD); + return ondemandPodLogs(ns, pod, condition); + } + protected Map convertInstancePropertiesToMap(final ServiceInstance instance) { + if (instance == null) { + return Collections.emptyMap(); + } + final List attributes = instance.getAttributes(); + if (attributes == null || attributes.isEmpty()) { + return Collections.emptyMap(); + } + final Map attributesMap = + attributes + .stream() + .collect(Collectors.toMap(Attribute::getName, Attribute::getValue)); + if (!attributesMap.containsKey(PropertyUtil.NAMESPACE) + || !attributesMap.containsKey(PropertyUtil.POD)) { + return Collections.emptyMap(); + } + return attributesMap; + } + + public List listContainers( + final String namespace, + final String podName) throws IOException { try { - final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null); + if (Strings.isNullOrEmpty(namespace) || Strings.isNullOrEmpty(podName)) { + return Collections.emptyList(); + } + final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null); final V1PodSpec spec = pod.getSpec(); if (isNull(spec)) { - throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName)); + return Collections.emptyList(); } final List containers = spec.getContainers().stream() @@ -114,23 +137,19 @@ public class OndemandLogQuery implements GraphQLQueryResolver { } } - public Logs ondemandPodLogs(OndemandLogQueryCondition condition) - throws IOException { - final String ns = condition.getNamespace(); - final IDManager.ServiceInstanceID.InstanceIDDefinition instanceIDDefinition = - IDManager.ServiceInstanceID.analysisId(condition.getServiceInstanceId()); - final String instanceName = instanceIDDefinition.getName(); - + public Logs ondemandPodLogs( + final String namespace, + final String podName, + final OndemandLogQueryCondition condition) throws IOException { try { - final V1Pod pod = kApi().readNamespacedPod(instanceName, ns, null); + final V1Pod pod = kApi().readNamespacedPod(podName, namespace, null); final V1ObjectMeta podMetadata = pod.getMetadata(); if (isNull(podMetadata)) { - throw new RuntimeException( - String.format("No such instance: %s:%s", ns, instanceName)); + return new Logs(); } final V1PodSpec spec = pod.getSpec(); if (isNull(spec)) { - throw new RuntimeException(String.format("No spec: %s:%s", ns, instanceName)); + return new Logs(); } final Duration duration = new Duration(); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java new file mode 100644 index 0000000000..754412e0e6 --- /dev/null +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandContainergQueryCondition.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.skywalking.oap.query.graphql.type; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class OndemandContainergQueryCondition { + private String serviceInstanceId; +} diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java index d186e1d496..43397b2748 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/type/OndemandLogQueryCondition.java @@ -28,9 +28,7 @@ import lombok.Setter; @Getter @Setter public class OndemandLogQueryCondition { - private String namespace; private String container; - private String serviceId; private String serviceInstanceId; private Duration duration; private List keywordsOfContent = Collections.emptyList(); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index edf5871a3a..faf3a76808 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit edf5871a3a2e8f7300eb06175240b56c21072fe9 +Subproject commit faf3a768087825258f64dd782a4dc405dc271cf8 diff --git a/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml new file mode 100644 index 0000000000..8d723fbd4b --- /dev/null +++ b/oap-server/server-starter/src/main/resources/otel-oc-rules/k8s-instance.yaml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filter: "{ tags -> tags.job_name in [ 'kubernetes-cadvisor', 'kube-state-metrics' ] }" # The OpenTelemetry job name +expSuffix: |- + service(['cluster' , 'service'], '::', Layer.K8S_SERVICE) + .instance(['cluster', 'service'], '::', ['pod'], '', Layer.K8S_SERVICE, { tags -> ['pod': tags.pod, 'namespace': tags.namespace] }) +metricPrefix: k8s_service_instance +metricsRules: + - name: pod_instance_status + exp: kube_pod_status_phase.retagByK8sMeta('service' , K8sRetagType.Pod2Service , 'pod' , 'namespace').tagNotEqual('service' , '').sum(['cluster', 'namespace', 'service' , 'pod'])