From 2c26b9819893398fd2bdc8b5cee9d731c33e6adb Mon Sep 17 00:00:00 2001 From: innerpeacez Date: Wed, 5 Apr 2023 00:41:01 +0800 Subject: [PATCH] Support monitoring the total number metrics of k8s StatefulSet and DaemonSet. (#10639) --- docs/en/changes/changes.md | 1 + .../setup/backend/backend-k8s-monitoring.md | 38 +++++----- .../resources/otel-rules/k8s/k8s-cluster.yaml | 6 ++ .../k8s/k8s-cluster.json | 70 ++++++++++++++++--- 4 files changed, 86 insertions(+), 29 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 62826806f8..568cc38e69 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -26,6 +26,7 @@ * Fix `NPE` when query the not exist series indexes in ElasticSearch storage. * Support collecting memory buff/cache metrics in VM monitoring. * PromQL: Remove empty values from the query result, fix `/api/v1/metadata` param `limit` could cause out of bound. +* Support monitoring the total number metrics of k8s StatefulSet and DaemonSet. #### UI * Revert: cpm5d function. This feature is cancelled from backend. diff --git a/docs/en/setup/backend/backend-k8s-monitoring.md b/docs/en/setup/backend/backend-k8s-monitoring.md index 6bd53cd8e9..29bac54f92 100644 --- a/docs/en/setup/backend/backend-k8s-monitoring.md +++ b/docs/en/setup/backend/backend-k8s-monitoring.md @@ -18,24 +18,26 @@ For a quick start, we have provided a complete example of configuration and reco K8s cluster monitoring provides monitoring of the status and resources of the whole cluster and each node. K8s cluster as a `Service` in OAP, K8s node as an `Instance` in OAP, and land on the `Layer: K8S`. ### Kubernetes Cluster Supported Metrics -| Monitoring Panel | Unit | Metric Name | Description | Data Source | -|-----|------|-----|-----|-----| -| Node Total | | k8s_cluster_node_total | The number of nodes | K8s kube-state-metrics| -| Namespace Total | | k8s_cluster_namespace_total | The number of namespaces | K8s kube-state-metrics| -| Deployment Total | | k8s_cluster_deployment_total | The number of deployments | K8s kube-state-metrics| -| Service Total | | k8s_cluster_service_total | The number of services | K8s kube-state-metrics| -| Pod Total | | k8s_cluster_pod_total | The number of pods | K8s kube-state-metrics| -| Container Total | | k8s_cluster_container_total | The number of containers | K8s kube-state-metrics| -| CPU Resources | m | k8s_cluster_cpu_cores
k8s_cluster_cpu_cores_requests
k8s_cluster_cpu_cores_limits
k8s_cluster_cpu_cores_allocatable | The capacity and the Requests / Limits / Allocatable of the CPU | K8s kube-state-metrics| -| Memory Resources | Gi | k8s_cluster_memory_total
k8s_cluster_memory_requests
k8s_cluster_memory_limits
k8s_cluster_memory_allocatable | The capacity and the Requests / Limits / Allocatable of the memory | K8s kube-state-metrics| -| Storage Resources | Gi | k8s_cluster_storage_total
k8s_cluster_storage_allocatable | The capacity and allocatable of the storage | K8s kube-state-metrics| -| Node Status | | k8s_cluster_node_status | The current status of the nodes | K8s kube-state-metrics| -| Deployment Status | | k8s_cluster_deployment_status | The current status of the deployment | K8s kube-state-metrics| -| Deployment Spec Replicas | | k8s_cluster_deployment_spec_replicas | The number of desired pods for a deployment | K8s kube-state-metrics| -| Service Status | | k8s_cluster_service_pod_status | The services current status, depending on the related pods' status | K8s kube-state-metrics| -| Pod Status Not Running | | k8s_cluster_pod_status_not_running | The pods which are not running in the current phase | K8s kube-state-metrics| -| Pod Status Waiting | | k8s_cluster_pod_status_waiting | The pods and containers which are currently in the waiting status, with reasons shown | K8s kube-state-metrics| -| Pod Status Terminated | | k8s_cluster_container_status_terminated | The pods and containers which are currently in the terminated status, with reasons shown | K8s kube-state-metrics| +| Monitoring Panel | Unit | Metric Name | Description | Data Source | +|--------------------------|------|------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|------------------------| +| Node Total | | k8s_cluster_node_total | The number of nodes | K8s kube-state-metrics | +| Namespace Total | | k8s_cluster_namespace_total | The number of namespaces | K8s kube-state-metrics | +| Deployment Total | | k8s_cluster_deployment_total | The number of deployments | K8s kube-state-metrics | +| StatefulSet Total | | k8s_cluster_statefulset_total | The number of statefulsets | K8s kube-state-metrics | +| DaemonSet Total | | k8s_cluster_daemonset_total | The number of daemonsets | K8s kube-state-metrics | +| Service Total | | k8s_cluster_service_total | The number of services | K8s kube-state-metrics | +| Pod Total | | k8s_cluster_pod_total | The number of pods | K8s kube-state-metrics | +| Container Total | | k8s_cluster_container_total | The number of containers | K8s kube-state-metrics | +| CPU Resources | m | k8s_cluster_cpu_cores
k8s_cluster_cpu_cores_requests
k8s_cluster_cpu_cores_limits
k8s_cluster_cpu_cores_allocatable | The capacity and the Requests / Limits / Allocatable of the CPU | K8s kube-state-metrics | +| Memory Resources | Gi | k8s_cluster_memory_total
k8s_cluster_memory_requests
k8s_cluster_memory_limits
k8s_cluster_memory_allocatable | The capacity and the Requests / Limits / Allocatable of the memory | K8s kube-state-metrics | +| Storage Resources | Gi | k8s_cluster_storage_total
k8s_cluster_storage_allocatable | The capacity and allocatable of the storage | K8s kube-state-metrics | +| Node Status | | k8s_cluster_node_status | The current status of the nodes | K8s kube-state-metrics | +| Deployment Status | | k8s_cluster_deployment_status | The current status of the deployment | K8s kube-state-metrics | +| Deployment Spec Replicas | | k8s_cluster_deployment_spec_replicas | The number of desired pods for a deployment | K8s kube-state-metrics | +| Service Status | | k8s_cluster_service_pod_status | The services current status, depending on the related pods' status | K8s kube-state-metrics | +| Pod Status Not Running | | k8s_cluster_pod_status_not_running | The pods which are not running in the current phase | K8s kube-state-metrics | +| Pod Status Waiting | | k8s_cluster_pod_status_waiting | The pods and containers which are currently in the waiting status, with reasons shown | K8s kube-state-metrics | +| Pod Status Terminated | | k8s_cluster_container_status_terminated | The pods and containers which are currently in the terminated status, with reasons shown | K8s kube-state-metrics | ### Kubernetes Cluster Node Supported Metrics | Monitoring Panel | Unit | Metric Name | Description | Data Source | diff --git a/oap-server/server-starter/src/main/resources/otel-rules/k8s/k8s-cluster.yaml b/oap-server/server-starter/src/main/resources/otel-rules/k8s/k8s-cluster.yaml index b141d5c391..33bd30f9f6 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/k8s/k8s-cluster.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/k8s/k8s-cluster.yaml @@ -70,6 +70,12 @@ metricsRules: - name: deployment_spec_replicas exp: kube_deployment_spec_replicas.sum(['cluster' , 'deployment' , 'namespace']) + - name: statefulset_total + exp: kube_statefulset_labels.sum(['cluster']) + + - name: daemonset_total + exp: kube_daemonset_labels.sum(['cluster']) + - name: service_total exp: kube_service_info.sum(['cluster']) - name: service_pod_status diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/k8s/k8s-cluster.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/k8s/k8s-cluster.json index 485f72139b..d886412b91 100644 --- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/k8s/k8s-cluster.json +++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/k8s/k8s-cluster.json @@ -295,9 +295,9 @@ "moved": false }, { - "x": 20, + "x": 21, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "10", "type": "Widget", @@ -319,9 +319,9 @@ "moved": false }, { - "x": 16, + "x": 18, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "11", "type": "Widget", @@ -343,9 +343,9 @@ "moved": false }, { - "x": 12, + "x": 15, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "12", "type": "Widget", @@ -369,7 +369,7 @@ { "x": 0, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "13", "type": "Widget", @@ -391,9 +391,9 @@ "moved": false }, { - "x": 4, + "x": 3, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "14", "type": "Widget", @@ -415,9 +415,9 @@ "moved": false }, { - "x": 8, + "x": 6, "y": 0, - "w": 4, + "w": 3, "h": 6, "i": "15", "type": "Widget", @@ -437,6 +437,54 @@ "readMetricsValue" ], "moved": false + }, + { + "x": 9, + "y": 0, + "w": 3, + "h": 6, + "i": "17", + "type": "Widget", + "widget": { + "title": "StatefulSet Total" + }, + "graph": { + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUint": true + }, + "metrics": [ + "k8s_cluster_statefulset_total" + ], + "metricTypes": [ + "readMetricsValue" + ], + "moved": false + }, + { + "x": 12, + "y": 0, + "w": 3, + "h": 6, + "i": "18", + "type": "Widget", + "widget": { + "title": "DaemonSet Total" + }, + "graph": { + "type": "Card", + "fontSize": 14, + "textAlign": "center", + "showUint": true + }, + "metrics": [ + "k8s_cluster_daemonset_total" + ], + "metricTypes": [ + "readMetricsValue" + ], + "moved": false } ] },