Enable otlp OC handler by default and polish doc (#9499)

This commit is contained in:
kezhenxu94 2022-08-22 16:09:59 +08:00 committed by GitHub
parent 828669ce2a
commit c8a78149a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 80 additions and 60 deletions

View File

@ -57,6 +57,8 @@
* Bump up Kubernetes Java client to fix cve.
* Adapt OpenTelemetry native metrics protocol.
* [Breaking Change] rename configuration folder from `otel-oc-rules` to `otel-rules`.
* [Breaking Change] rename configuration field from `enabledOcRules` to `enabledOtelRules` and
environment variable name from `SW_OTEL_RECEIVER_ENABLED_OC_RULES` to `SW_OTEL_RECEIVER_ENABLED_OTEL_RULES`.
#### UI

View File

@ -40,15 +40,15 @@ prometheus-fetcher:
selector: ${SW_PROMETHEUS_FETCHER:default}
default:
enabledRules: ${SW_PROMETHEUS_FETCHER_ENABLED_RULES:"self"}
```
```
3. Make sure `config/fetcher-prom-rules/self.yaml` exists.
3. Make sure `config/fetcher-prom-rules/self.yaml` exists.
Once you deploy an OAP server cluster, the target host should be replaced with a dedicated IP or hostname. For instance,
if there are three OAP servers in your cluster, their hosts are `service1`, `service2`, and `service3`, respectively. You should
update each `self.yaml` to switch the target host.
service1:
service1:
```yaml
fetcherInterval: PT15S
fetcherTimeout: PT10S
@ -62,7 +62,7 @@ staticConfig:
...
```
service2:
service2:
```yaml
fetcherInterval: PT15S
fetcherTimeout: PT10S
@ -76,7 +76,7 @@ staticConfig:
...
```
service3:
service3:
```yaml
fetcherInterval: PT15S
fetcherTimeout: PT10S
@ -90,7 +90,7 @@ staticConfig:
...
```
### Service discovery on Kubernetes
If you deploy an OAP server cluster on Kubernetes, the oap-server instance (pod) would not have a static IP or hostname. We can leverage [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/getting-started/#kubernetes) to discover the oap-server instance, and scrape & transfer the metrics to OAP [OpenTelemetry receiver](opentelemetry-receiver.md).
If you deploy an OAP server cluster on Kubernetes, the oap-server instance (pod) would not have a static IP or hostname. We can leverage [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/getting-started/#kubernetes) to discover the oap-server instance, and scrape & transfer the metrics to OAP [OpenTelemetry receiver](opentelemetry-receiver.md).
On how to install SkyWalking on k8s, you can refer to [Apache SkyWalking Kubernetes](https://github.com/apache/skywalking-kubernetes).
@ -103,9 +103,9 @@ Set this up following these steps:
```
- Set environment variables.
```
SW_TELEMETRY=prometheus
SW_OTEL_RECEIVER=default
SW_OTEL_RECEIVER_ENABLED_OC_RULES=oap
SW_TELEMETRY=prometheus
SW_OTEL_RECEIVER=default
SW_OTEL_RECEIVER_ENABLED_OTEL_RULES=oap
```
Here is an example to install by Apache SkyWalking Kubernetes:
@ -123,7 +123,7 @@ Set this up following these steps:
--set oap.ports.prometheus-port=1234 \ # <<< Expose self observability metrics port
--set oap.env.SW_TELEMETRY=prometheus \
--set oap.env.SW_OTEL_RECEIVER=default \ # <<< Enable Otel receiver
--set oap.env.SW_OTEL_RECEIVER_ENABLED_OC_RULES=oap # <<< Add oap analyzer for Otel metrics
--set oap.env.SW_OTEL_RECEIVER_ENABLED_OTEL_RULES=oap # <<< Add oap analyzer for Otel metrics
```
2. Set up OpenTelemetry Collector and config a scrape job:
``` yaml
@ -134,14 +134,14 @@ Set this up following these steps:
relabel_configs:
- source_labels: [__meta_kubernetes_pod_container_name, __meta_kubernetes_pod_container_port_name]
action: keep
regex: oap;prometheus-port
regex: oap;prometheus-port
- source_labels: []
target_label: service
replacement: oap-server
- source_labels: [__meta_kubernetes_pod_name]
target_label: host_name
regex: (.+)
replacement: $$1
replacement: $$1
```
For the full example for OpenTelemetry Collector configuration and recommended version, you can refer to [showcase](https://github.com/apache/skywalking-showcase/tree/main/deploy/platform/kubernetes/feature-so11y).
@ -185,5 +185,5 @@ telemetry:
```
### Grafana Visualization
Provide the Grafana dashboard settings.
Provide the Grafana dashboard settings.
Check [SkyWalking OAP Cluster Monitor Dashboard](grafana-cluster.json) config and [SkyWalking OAP Instance Monitor Dashboard](grafana-instance.json) config.

View File

@ -192,8 +192,7 @@ The Configuration Vocabulary lists all available configurations provided by `app
| - | - | k8sServiceNameRule | `k8sServiceNameRule` allows you to customize the service name in ALS via Kubernetes metadata. The available variables are `pod` and `service`. E.g. you can use `${service.metadata.name}-${pod.metadata.labels.version}` to append the version number to the service name. Note that when using environment variables to pass this configuration, use single quotes(`''`) to avoid being evaluated by the shell. | - | |
| receiver-otel | default | A receiver for analyzing metrics data from OpenTelemetry. | - | - | |
| - | - | enabledHandlers | Enabled handlers for otel. | SW_OTEL_RECEIVER_ENABLED_HANDLERS | - |
| - | - | enabledOcRules | Enabled metric rules for OC handler. | SW_OTEL_RECEIVER_ENABLED_OC_RULES | - |
| - | - | enabledOtlpRules | Enabled metric rules for OC handler. | SW_OTEL_RECEIVER_ENABLED_OTLP_RULES | - |
| - | - | enabledOtelRules | Enabled metric rules for OC handler. | SW_OTEL_RECEIVER_ENABLED_OTEL_RULES | - |
| receiver-zipkin | default | A receiver for Zipkin traces. | - | - | |
| - | - | sampleRate | The sample rate precision is 1/10000, should be between 0 and 10000 | SW_ZIPKIN_SAMPLE_RATE | 10000 |
| - | - | searchableTracesTags | Defines a set of span tag keys which are searchable. Multiple values are separated by commas. | SW_ZIPKIN_SEARCHABLE_TAG_KEYS | http.method |

View File

@ -20,21 +20,24 @@ receiver-otel:
// Change selector value to default, for activating the otel receiver.
selector: ${SW_OTEL_RECEIVER:default}
default:
enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"oc"}
enabledOcRules: ${SW_OTEL_RECEIVER_ENABLED_OC_RULES:"istio-controlplane"}
enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"oc,otlp"}
enabledOtelRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_RULES:"istio-controlplane"}
```
The receiver adds labels with `key = node_identifier_host_name` and `key = node_identifier_pid` to the collected data samples,
and values from `Node.identifier.host_name` and `Node.identifier.pid` defined in OpenCensus Agent Proto,
The receiver adds label with key `node_identifier_host_name` to the collected data samples,
and its value is from `Node.identifier.host_name` defined in OpenCensus Agent Proto,
or `net.host.name` (or `host.name` for some OTLP versions) resource attributes defined in OpenTelemetry proto,
for identification of the metric data.
| Rule Name | Description | Configuration File | Data Source |
|----|----|-----|----|
|istio-controlplane| Metrics of Istio Control Plane | otel-rules/istio-controlplane.yaml | Istio Control Plane -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|oap| Metrics of SkyWalking OAP server itself | otel-rules/oap.yaml | SkyWalking OAP Server(SelfObservability) -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|vm| Metrics of VMs | otel-rules/vm.yaml | Prometheus node-exporter(VMs) -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|k8s-cluster| Metrics of K8s cluster | otel-rules/k8s-cluster.yaml | K8s kube-state-metrics -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|k8s-node| Metrics of K8s cluster | otel-rules/k8s-node.yaml | cAdvisor & K8s kube-state-metrics -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|k8s-service| Metrics of K8s cluster | otel-rules/k8s-service.yaml | cAdvisor & K8s kube-state-metrics -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|mysql| Metrics of MySQL| otel-rules/mysql.yaml | prometheus/mysqld_exporter -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|postgresql| Metrics of PostgreSQL| otel-rules/postgresql.yaml | postgres_exporter -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
|istio-controlplane| Metrics of Istio Control Plane | otel-rules/istio-controlplane.yaml | Istio Control Plane -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|oap| Metrics of SkyWalking OAP server itself | otel-rules/oap.yaml | SkyWalking OAP Server(SelfObservability) -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|vm| Metrics of VMs | otel-rules/vm.yaml | Prometheus node-exporter(VMs) -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|k8s-cluster| Metrics of K8s cluster | otel-rules/k8s-cluster.yaml | K8s kube-state-metrics -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|k8s-node| Metrics of K8s cluster | otel-rules/k8s-node.yaml | cAdvisor & K8s kube-state-metrics -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|k8s-service| Metrics of K8s cluster | otel-rules/k8s-service.yaml | cAdvisor & K8s kube-state-metrics -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|mysql| Metrics of MYSQL| otel-rules/mysql.yaml | prometheus/mysqld_exporter -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
|postgresql| Metrics of PostgreSQL| otel-rules/postgresql.yaml | postgres_exporter -> OpenTelemetry Collector -- OC/OTLP exporter --> SkyWalking OAP Server |
**Note**: You can also use OpenTelemetry exporter to transport the metrics to SkyWalking OAP directly. See [OpenTelemetry Exporter](./backend-meter.md#opentelemetry-exporter).

View File

@ -54,7 +54,7 @@ public interface Handler {
String type();
void active(List<String> enabledRules, MeterSystem service,
GRPCHandlerRegister grpcHandlerRegister) throws ModuleStartException;
void active(OtelMetricReceiverConfig config,
MeterSystem meterSystem,
GRPCHandlerRegister grpcHandlerRegister) throws ModuleStartException;
}

View File

@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.skywalking.oap.server.library.util.StringUtil;
@ -34,8 +35,8 @@ public class OtelMetricReceiverConfig extends ModuleConfig {
private String enabledHandlers;
private String enabledOcRules;
private String enabledOtlpRules;
@Getter
private String enabledOtelRules;
public List<String> getEnabledHandlers() {
return split(enabledHandlers);
@ -53,7 +54,7 @@ public class OtelMetricReceiverConfig extends ModuleConfig {
}
f.setAccessible(true);
try {
return split(f.get(this));
return split(f.get(this));
} catch (IllegalAccessException e) {
log.warn("failed to access disabled rule list of {}", handler, e);
return Collections.emptyList();

View File

@ -68,7 +68,7 @@ public class OtelMetricReceiverProvider extends ModuleProvider {
.filter(h -> config.getEnabledHandlers().contains(h.type()))
.collect(toList());
for (Handler h : handlers) {
h.active(config.getEnabledRulesFrom(h.type()), meterSystem, grpcHandlerRegister);
h.active(config, meterSystem, grpcHandlerRegister);
}
}

View File

@ -18,6 +18,7 @@
package org.apache.skywalking.oap.server.receiver.otel.oc;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.protobuf.Timestamp;
import io.grpc.stub.StreamObserver;
@ -49,12 +50,12 @@ import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Gauge;
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Histogram;
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Summary;
import org.apache.skywalking.oap.server.receiver.otel.Handler;
import org.apache.skywalking.oap.server.receiver.otel.OtelMetricReceiverConfig;
import static java.util.stream.Collectors.toList;
public class OCMetricHandler extends MetricsServiceGrpc.MetricsServiceImplBase implements Handler {
private static final String HOST_NAME_LABEL = "node_identifier_host_name";
private List<PrometheusMetricConverter> metrics;
private List<PrometheusMetricConverter> converters;
@Override public StreamObserver<ExportMetricsServiceRequest> export(
StreamObserver<ExportMetricsServiceResponse> responseObserver) {
@ -72,9 +73,6 @@ public class OCMetricHandler extends MetricsServiceGrpc.MetricsServiceImplBase i
if (StringUtil.isNotBlank(node.getIdentifier().getHostName())) {
nodeLabels.put(HOST_NAME_LABEL, node.getIdentifier().getHostName());
}
if (node.getIdentifier().getPid() > 0) {
nodeLabels.put("node_identifier_pid", String.valueOf(node.getIdentifier().getPid()));
}
}
final String name = node.getServiceInfo().getName();
if (!Strings.isNullOrEmpty(name)) {
@ -88,7 +86,7 @@ public class OCMetricHandler extends MetricsServiceGrpc.MetricsServiceImplBase i
nodeLabels.put(HOST_NAME_LABEL, resource.getLabelsMap().get("net.host.name"));
}
}
metrics.forEach(m -> m.toMeter(request.getMetricsList().stream()
converters.forEach(m -> m.toMeter(request.getMetricsList().stream()
.flatMap(metric -> metric.getTimeseriesList().stream().map(timeSeries ->
Tuple.of(metric.getMetricDescriptor(),
buildLabelsFromNodeInfo(
@ -175,16 +173,19 @@ public class OCMetricHandler extends MetricsServiceGrpc.MetricsServiceImplBase i
@Override
public void active(
List<String> enabledRules,
OtelMetricReceiverConfig config,
MeterSystem meterSystem,
GRPCHandlerRegister grpcHandlerRegister)
throws ModuleStartException {
final List<String> enabledRules =
Splitter.on(",")
.omitEmptyStrings()
.splitToList(config.getEnabledOtelRules());
final List<Rule> rules = Rules.loadRules("otel-rules", enabledRules);
if (rules.isEmpty()) {
return;
}
this.metrics = rules.stream().map(r -> new PrometheusMetricConverter(r, meterSystem))
this.converters = rules.stream().map(r -> new PrometheusMetricConverter(r, meterSystem))
.collect(toList());
grpcHandlerRegister.addHandler(this);
}

View File

@ -39,6 +39,8 @@ import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Histogra
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Metric;
import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Summary;
import org.apache.skywalking.oap.server.receiver.otel.Handler;
import org.apache.skywalking.oap.server.receiver.otel.OtelMetricReceiverConfig;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import io.grpc.stub.StreamObserver;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
@ -48,7 +50,9 @@ import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.metrics.v1.Sum;
import io.opentelemetry.proto.metrics.v1.SummaryDataPoint.ValueAtQuantile;
import io.vavr.Function1;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class OpenTelemetryMetricHandler
extends MetricsServiceGrpc.MetricsServiceImplBase
implements Handler {
@ -56,8 +60,10 @@ public class OpenTelemetryMetricHandler
private static final Map<String, String> LABEL_MAPPINGS =
ImmutableMap
.<String, String>builder()
.put("net.host.name", "node_identifier_host_name")
.put("host.name", "node_identifier_host_name")
.put("job", "job_name")
.put("service.name", "job_name")
.build();
private List<PrometheusMetricConverter> converters;
@ -68,10 +74,13 @@ public class OpenTelemetryMetricHandler
@Override
public void active(
final List<String> enabledRules,
final OtelMetricReceiverConfig config,
final MeterSystem service,
final GRPCHandlerRegister grpcHandlerRegister) throws ModuleStartException {
final List<String> enabledRules =
Splitter.on(",")
.omitEmptyStrings()
.splitToList(config.getEnabledOtelRules());
final List<Rule> rules = Rules.loadRules("otel-rules", enabledRules);
if (rules.isEmpty()) {
@ -92,6 +101,10 @@ public class OpenTelemetryMetricHandler
final StreamObserver<ExportMetricsServiceResponse> responseObserver) {
requests.getResourceMetricsList().forEach(request -> {
if (log.isDebugEnabled()) {
log.debug("Resource attributes: {}", request.getResource().getAttributesList());
}
final Map<String, String> nodeLabels =
request
.getResource()
@ -101,7 +114,8 @@ public class OpenTelemetryMetricHandler
it -> LABEL_MAPPINGS
.getOrDefault(it.getKey(), it.getKey())
.replaceAll("\\.", "_"),
it -> it.getValue().getStringValue()));
it -> it.getValue().getStringValue(),
(v1, v2) -> v1));
converters
.forEach(convert -> convert.toMeter(

View File

@ -339,8 +339,8 @@ receiver-otel:
selector: ${SW_OTEL_RECEIVER:default}
default:
enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"oc"}
enabledOcRules: ${SW_OTEL_RECEIVER_ENABLED_OC_RULES:"istio-controlplane,k8s-node,oap,vm,mysql,postgresql"}
enabledOtlpRules: ${SW_OTEL_RECEIVER_ENABLED_OTLP_RULES:"istio-controlplane,k8s-node,oap,vm,mysql,postgresql"}
enabledHandlers: ${SW_OTEL_RECEIVER_ENABLED_HANDLERS:"oc,otlp"}
enabledOtelRules: ${SW_OTEL_RECEIVER_ENABLED_OTEL_RULES:"istio-controlplane,k8s-node,oap,vm,mysql,postgresql"}
receiver-zipkin:
selector: ${SW_RECEIVER_ZIPKIN:-}

View File

@ -22,11 +22,11 @@ services:
service: oap
environment:
SW_OTEL_RECEIVER: default
SW_OTEL_RECEIVER_ENABLED_OC_RULES: mysql
SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: mysql
ports:
- 12800
mysql:
image: mysql:8.0.30
image: mysql:8.0.30
networks:
- e2e
ports:
@ -36,7 +36,7 @@ services:
- MYSQL_DATABASE=test2
mysql-load:
image: mysql:8.0.30
image: mysql:8.0.30
depends_on:
- otel-collector
networks:
@ -47,7 +47,7 @@ services:
command:
- -c
- "mysql -h mysql -uroot -ppassword < /docker/mock.sql"
mysql-service:
image: prom/mysqld-exporter:v0.14.0
ports:
@ -58,7 +58,7 @@ services:
- e2e
depends_on:
- mysql
otel-collector:
image: otel/opentelemetry-collector:0.50.0
@ -72,6 +72,6 @@ services:
depends_on:
oap:
condition: service_healthy
networks:
e2e:
e2e:

View File

@ -15,19 +15,19 @@
receivers:
prometheus:
config:
config:
scrape_configs:
- job_name: 'mysql-monitoring'
scrape_interval: 5s
static_configs:
- targets: ['mysql-service:9104']
labels:
labels:
host_name: showcase
processors:
batch:
exporters:
opencensus:
otlp:
endpoint: oap:11800
tls:
insecure: true
@ -39,4 +39,4 @@ service:
processors:
- batch
exporters:
- opencensus
- otlp

View File

@ -22,7 +22,7 @@ services:
service: oap
environment:
SW_OTEL_RECEIVER: default
SW_OTEL_RECEIVER_ENABLED_OC_RULES: vm
SW_OTEL_RECEIVER_ENABLED_OTEL_RULES: vm
ports:
- 12800