builder()
.putAll(t.getLabels())
.put("quantile", b.getKey().toString())
@@ -141,6 +141,11 @@ public class PrometheusMetricConverter {
if (ss.length < 1) {
return Optional.empty();
}
- return Optional.of(Tuple.of(metric.getName(), SampleFamilyBuilder.newBuilder(ss).build()));
+ return Optional.of(Tuple.of(escapedName(metric.getName()), SampleFamilyBuilder.newBuilder(ss).build()));
+ }
+
+ // Returns the escaped name of the given one, with "." replaced by "_"
+ protected String escapedName(final String name) {
+ return name.replaceAll("\\.", "_");
}
}
diff --git a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/AggregationTest.java b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/AggregationTest.java
index 72ddef55f..78934c1b1 100644
--- a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/AggregationTest.java
+++ b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/AggregationTest.java
@@ -81,6 +81,93 @@ public class AggregationTest {
).build()),
false,
},
+
+ {
+ "min",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t3")).value(100).build(),
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t2")).value(3).build()
+ ).build()),
+ "http_success_request.min()",
+ Result.success(SampleFamilyBuilder.newBuilder(Sample.builder().labels(ImmutableMap.of()).value(3).build()).build()),
+ false,
+ },
+ {
+ "min-by",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "svc", "catalog")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "svc", "product")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "instance", "10.0.0.1")).value(100).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "instance", "10.0.0.1")).value(3).build()
+ ).build()),
+ "http_success_request.min(by = ['region', 'idc'])",
+ Result.success(SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1", "region", "")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us")).value(50).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn")).value(3).build()
+ ).build()),
+ false,
+ },
+
+ {
+ "max",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t3")).value(100).build(),
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t2")).value(3).build()
+ ).build()),
+ "http_success_request.max()",
+ Result.success(SampleFamilyBuilder.newBuilder(Sample.builder().labels(ImmutableMap.of()).value(100).build()).build()),
+ false,
+ },
+ {
+ "max-by",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "svc", "catalog")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "svc", "product")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "instance", "10.0.0.1")).value(100).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "instance", "10.0.0.1")).value(3).build()
+ ).build()),
+ "http_success_request.max(by = ['region', 'idc'])",
+ Result.success(SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1", "region", "")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us")).value(100).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn")).value(50).build()
+ ).build()),
+ false,
+ },
+
+ {
+ "avg",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t3")).value(100).build(),
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t2")).value(3).build()
+ ).build()),
+ "http_success_request.avg()",
+ Result.success(SampleFamilyBuilder.newBuilder(Sample.builder().labels(ImmutableMap.of()).value(51).build()).build()),
+ false,
+ },
+ {
+ "avg-by",
+ of("http_success_request", SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1")).value(50).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "svc", "catalog")).value(51).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "svc", "product")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us", "instance", "10.0.0.1")).value(100).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn", "instance", "10.0.0.1")).value(3).build()
+ ).build()),
+ "http_success_request.avg(by = ['region', 'idc'])",
+ Result.success(SampleFamilyBuilder.newBuilder(
+ Sample.builder().labels(of("idc", "t1", "region", "")).value(50).build(),
+ Sample.builder().labels(of("idc", "t1", "region", "us")).value(75).build(),
+ Sample.builder().labels(of("idc", "t3", "region", "cn")).value(27).build()
+ ).build()),
+ false,
+ },
});
}
@@ -102,4 +189,4 @@ public class AggregationTest {
}
assertThat(r, is(want));
}
-}
\ No newline at end of file
+}
diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml
index 78db4ed4a..8c636417d 100644
--- a/oap-server/server-bootstrap/pom.xml
+++ b/oap-server/server-bootstrap/pom.xml
@@ -259,6 +259,7 @@
endpoint-name-grouping.yml
oal/
fetcher-prom-rules/
+ envoy-metrics-rules/
meter-analyzer-config/
otel-oc-rules/
ui-initialized-templates/
@@ -267,4 +268,4 @@
-
\ No newline at end of file
+
diff --git a/oap-server/server-bootstrap/src/main/resources/envoy-metrics-rules/envoy.yaml b/oap-server/server-bootstrap/src/main/resources/envoy-metrics-rules/envoy.yaml
new file mode 100644
index 000000000..a8c5a8b7a
--- /dev/null
+++ b/oap-server/server-bootstrap/src/main/resources/envoy-metrics-rules/envoy.yaml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This will parse a textual representation of a duration. The formats
+# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS}
+# with days considered to be exactly 24 hours.
+#
+# Examples:
+#
+# "PT20.345S" -- parses as "20.345 seconds"
+# "PT15M" -- parses as "15 minutes" (where a minute is 60 seconds)
+# "PT10H" -- parses as "10 hours" (where an hour is 3600 seconds)
+# "P2D" -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+# "P2DT3H4M" -- parses as "2 days, 3 hours and 4 minutes"
+# "P-6H3M" -- parses as "-6 hours and +3 minutes"
+# "-P6H3M" -- parses as "-6 hours and -3 minutes"
+# "-P-6H+3M" -- parses as "+6 hours and -3 minutes"
+#
+
+expSuffix: tag({tags -> tags.cluster = 'istio-dp::' + tags.cluster}).instance(['cluster'], ['instance'])
+metricPrefix: envoy
+metricsRules:
+ - name: heap_memory_used
+ exp: server_memory_heap_size
+ - name: heap_memory_max_used
+ exp: server_memory_heap_size.max(['cluster', 'instance'])
+ - name: memory_allocated
+ exp: server_memory_allocated
+ - name: memory_allocated_max
+ exp: server_memory_allocated.max(['cluster', 'instance'])
+ - name: memory_physical_size
+ exp: server_memory_physical_size
+ - name: memory_physical_size_max
+ exp: server_memory_physical_size.max(['cluster', 'instance'])
+
+ - name: total_connections_used
+ exp: server_total_connections.max(['cluster', 'instance'])
+ - name: parent_connections_used
+ exp: server_parent_connections.max(['cluster', 'instance'])
+
+ - name: worker_threads
+ exp: server_concurrency
+ - name: worker_threads_max
+ exp: server_concurrency.max(['cluster', 'instance'])
+
+ - name: bug_failures
+ exp: server_envoy_bug_failures
diff --git a/oap-server/server-bootstrap/src/main/resources/oal/envoy.oal b/oap-server/server-bootstrap/src/main/resources/oal/envoy.oal
deleted file mode 100644
index 90d7b4d03..000000000
--- a/oap-server/server-bootstrap/src/main/resources/oal/envoy.oal
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-// Envoy instance metrics
-envoy_heap_memory_max_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.memory_heap_size").maxDouble();
-envoy_total_connections_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.total_connections").maxDouble();
-envoy_parent_connections_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.parent_connections").maxDouble();
\ No newline at end of file
diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio-dp.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio-dp.yml
new file mode 100644
index 000000000..9441b9560
--- /dev/null
+++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio-dp.yml
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+templates:
+ - name: "Istio Data Plane"
+ type: "DASHBOARD"
+ configuration: |-
+ [
+ {
+ "name": "Istio Data Plane",
+ "type": "service",
+ "serviceGroup": "istio-dp",
+ "children": [
+ {
+ "name": "Data Plane",
+ "children": [
+ {
+ "width": "3",
+ "title": "Heap Memory Used",
+ "height": 350,
+ "entityType": "ServiceInstance",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "envoy_heap_memory_max_used,envoy_heap_memory_used,envoy_memory_allocated_max,envoy_memory_allocated,envoy_memory_physical_size,envoy_memory_physical_size_max",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine",
+ "unit": "MB",
+ "aggregation": "/",
+ "aggregationNum": "1048576"
+ },
+ {
+ "width": "3",
+ "title": "Connections Used",
+ "height": 350,
+ "entityType": "ServiceInstance",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "envoy_total_connections_used,envoy_parent_connections_used",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine"
+ },
+ {
+ "width": "3",
+ "title": "Concurrency",
+ "height": 350,
+ "entityType": "ServiceInstance",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "envoy_worker_threads,envoy_worker_threads_max",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine"
+ },
+ {
+ "width": "3",
+ "title": "Envoy Bug Failure",
+ "height": 350,
+ "entityType": "ServiceInstance",
+ "independentSelector": false,
+ "metricType": "REGULAR_VALUE",
+ "metricName": "envoy_bug_failures",
+ "queryMetricType": "readMetricsValues",
+ "chartType": "ChartLine"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ activated: true
+ disabled: false
diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml
index bea98db2e..1e0f6dfa0 100644
--- a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml
+++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml
@@ -172,4 +172,4 @@ templates:
# False means providing a basic template, user needs to add it manually.
activated: true
# True means wouldn't show up on the dashboard. Only keeps the definition in the storage.
- disabled: false
\ No newline at end of file
+ disabled: false
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
index 93eed36d2..c1caf8b8e 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/pom.xml
@@ -34,6 +34,11 @@
receiver-proto
${project.version}
+
+ org.apache.skywalking
+ meter-analyzer
+ ${project.version}
+
org.apache.skywalking
skywalking-mesh-receiver-plugin
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
index 5819d8093..fb029fb85 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverConfig.java
@@ -24,7 +24,10 @@ import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Getter;
+import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rule;
+import org.apache.skywalking.oap.meter.analyzer.prometheus.rule.Rules;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+import org.apache.skywalking.oap.server.library.module.ModuleStartException;
public class EnvoyMetricReceiverConfig extends ModuleConfig {
@Getter
@@ -39,4 +42,8 @@ public class EnvoyMetricReceiverConfig extends ModuleConfig {
}
return Arrays.stream(alsHTTPAnalysis.trim().split(",")).map(String::trim).collect(Collectors.toList());
}
+
+ public List rules() throws ModuleStartException {
+ return Rules.loadRules("envoy-metrics-rules", Collections.singletonList("envoy"));
+ }
}
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
index bbbce0bf5..1e3d8c06c 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyMetricReceiverProvider.java
@@ -20,19 +20,21 @@ package org.apache.skywalking.oap.server.receiver.envoy;
import org.apache.skywalking.aop.server.receiver.mesh.MeshReceiverModule;
import org.apache.skywalking.oap.server.core.CoreModule;
-import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService;
import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister;
import org.apache.skywalking.oap.server.library.module.ModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
+import org.apache.skywalking.oap.server.receiver.envoy.als.mx.FieldsHelper;
import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
public class EnvoyMetricReceiverProvider extends ModuleProvider {
private final EnvoyMetricReceiverConfig config;
+ protected String fieldMappingFile = "metadata-service-mapping.yaml";
+
public EnvoyMetricReceiverProvider() {
config = new EnvoyMetricReceiverConfig();
}
@@ -54,7 +56,11 @@ public class EnvoyMetricReceiverProvider extends ModuleProvider {
@Override
public void prepare() throws ServiceNotProvidedException, ModuleStartException {
-
+ try {
+ FieldsHelper.SINGLETON.init(fieldMappingFile);
+ } catch (final Exception e) {
+ throw new ModuleStartException("Failed to load metadata-service-mapping.yaml", e);
+ }
}
@Override
@@ -63,12 +69,7 @@ public class EnvoyMetricReceiverProvider extends ModuleProvider {
.provider()
.getService(GRPCHandlerRegister.class);
if (config.isAcceptMetricsService()) {
- getManager().find(CoreModule.NAME)
- .provider()
- .getService(OALEngineLoaderService.class)
- .load(EnvoyOALDefine.INSTANCE);
-
- final MetricServiceGRPCHandler handler = new MetricServiceGRPCHandler(getManager());
+ final MetricServiceGRPCHandler handler = new MetricServiceGRPCHandler(getManager(), config);
service.addHandler(handler);
service.addHandler(new MetricServiceGRPCHandlerV3(handler));
}
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyOALDefine.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyOALDefine.java
deleted file mode 100644
index da0ba4caa..000000000
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/EnvoyOALDefine.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.skywalking.oap.server.receiver.envoy;
-
-import org.apache.skywalking.oap.server.core.oal.rt.OALDefine;
-
-/**
- * Envoy OAl script includes the metrics related to Envoy only.
- */
-public class EnvoyOALDefine extends OALDefine {
- public static final EnvoyOALDefine INSTANCE = new EnvoyOALDefine();
-
- private EnvoyOALDefine() {
- super(
- "oal/envoy.oal",
- "org.apache.skywalking.oap.server.core.source"
- );
- }
-}
\ No newline at end of file
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/MetricServiceGRPCHandler.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/MetricServiceGRPCHandler.java
index bb274cf92..2538a34f0 100644
--- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/MetricServiceGRPCHandler.java
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/MetricServiceGRPCHandler.java
@@ -18,23 +18,31 @@
package org.apache.skywalking.oap.server.receiver.envoy;
-import io.envoyproxy.envoy.config.core.v3.Node;
-import io.envoyproxy.envoy.service.metrics.v3.MetricsServiceGrpc;
+import io.envoyproxy.envoy.service.metrics.v2.MetricsServiceGrpc;
import io.envoyproxy.envoy.service.metrics.v3.StreamMetricsMessage;
import io.envoyproxy.envoy.service.metrics.v3.StreamMetricsResponse;
import io.grpc.stub.StreamObserver;
import io.prometheus.client.Metrics;
import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.util.StringUtil;
+import org.apache.skywalking.oap.meter.analyzer.prometheus.PrometheusMetricConverter;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
-import org.apache.skywalking.oap.server.core.source.EnvoyInstanceMetric;
+import org.apache.skywalking.oap.server.core.analysis.meter.MeterSystem;
import org.apache.skywalking.oap.server.core.analysis.NodeType;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceUpdate;
import org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.library.module.ModuleStartException;
+import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Metric;
+import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
+import org.apache.skywalking.oap.server.receiver.envoy.als.mx.ServiceMetaInfoAdapter;
+import org.apache.skywalking.oap.server.receiver.envoy.metrics.adapters.ProtoMetricFamily2MetricsAdapter;
import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics;
import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics;
@@ -44,10 +52,11 @@ import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;
@Slf4j
public class MetricServiceGRPCHandler extends MetricsServiceGrpc.MetricsServiceImplBase {
private final SourceReceiver sourceReceiver;
- private CounterMetrics counter;
- private HistogramMetrics histogram;
+ private final CounterMetrics counter;
+ private final HistogramMetrics histogram;
+ private final List converters;
- public MetricServiceGRPCHandler(ModuleManager moduleManager) {
+ public MetricServiceGRPCHandler(final ModuleManager moduleManager, final EnvoyMetricReceiverConfig config) throws ModuleStartException {
sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class);
MetricsCreator metricsCreator = moduleManager.find(TelemetryModule.NAME)
.provider()
@@ -60,16 +69,23 @@ public class MetricServiceGRPCHandler extends MetricsServiceGrpc.MetricsServiceI
"envoy_metric_in_latency", "The process latency of service metrics receiver", MetricsTag.EMPTY_KEY,
MetricsTag.EMPTY_VALUE
);
+
+ final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
+
+ converters = config.rules()
+ .stream()
+ .map(rule -> new PrometheusMetricConverter(rule, meterSystem))
+ .collect(Collectors.toList());
}
@Override
public StreamObserver streamMetrics(StreamObserver responseObserver) {
return new StreamObserver() {
private volatile boolean isFirst = true;
- private String serviceName = null;
- private String serviceInstanceName = null;
+ private ServiceMetaInfo service;
@Override
+ @SneakyThrows
public void onNext(StreamMetricsMessage message) {
if (log.isDebugEnabled()) {
log.debug("Received msg {}", message);
@@ -77,94 +93,41 @@ public class MetricServiceGRPCHandler extends MetricsServiceGrpc.MetricsServiceI
if (isFirst) {
isFirst = false;
- StreamMetricsMessage.Identifier identifier = message.getIdentifier();
- Node node = identifier.getNode();
- if (node != null) {
- String nodeId = node.getId();
- if (!StringUtil.isEmpty(nodeId)) {
- serviceInstanceName = nodeId;
- }
- String cluster = node.getCluster();
- if (!StringUtil.isEmpty(cluster)) {
- serviceName = cluster;
- if (serviceInstanceName == null) {
- serviceInstanceName = serviceName;
- }
- }
- }
-
- if (serviceName == null) {
- serviceName = serviceInstanceName;
- }
+ service = new ServiceMetaInfoAdapter(message.getIdentifier().getNode().getMetadata());
}
if (log.isDebugEnabled()) {
- log.debug(
- "Envoy metrics reported from service[{}], service instance[{}]", serviceName,
- serviceInstanceName
- );
+ log.debug("Envoy metrics reported from service[{}]", service);
}
- if (StringUtil.isNotEmpty(serviceName) && StringUtil.isNotEmpty(serviceInstanceName)) {
+ if (service != null && StringUtil.isNotEmpty(service.getServiceName()) && StringUtil.isNotEmpty(service.getServiceInstanceName())) {
List list = message.getEnvoyMetricsList();
boolean needHeartbeatUpdate = true;
- for (int i = 0; i < list.size(); i++) {
+
+ for (final Metrics.MetricFamily metricFamily : list) {
counter.inc();
- final String serviceId = IDManager.ServiceID.buildId(serviceName, NodeType.Normal);
- final String serviceInstanceId = IDManager.ServiceInstanceID.buildId(
- serviceId, serviceInstanceName);
+ final String serviceId = IDManager.ServiceID.buildId(service.getServiceName(), NodeType.Normal);
- HistogramMetrics.Timer timer = histogram.createTimer();
- try {
- Metrics.MetricFamily metricFamily = list.get(i);
- double value = 0;
- long timestamp = 0;
- switch (metricFamily.getType()) {
- case GAUGE:
- for (Metrics.Metric metrics : metricFamily.getMetricList()) {
- timestamp = metrics.getTimestampMs();
- value = metrics.getGauge().getValue();
+ try (final HistogramMetrics.Timer ignored = histogram.createTimer()) {
+ final ProtoMetricFamily2MetricsAdapter adapter = new ProtoMetricFamily2MetricsAdapter(metricFamily);
+ final Stream metrics = adapter.adapt().peek(it -> {
+ it.getLabels().putIfAbsent("cluster", service.getServiceName());
+ it.getLabels().putIfAbsent("instance", service.getServiceInstanceName());
+ });
+ converters.forEach(converter -> converter.toMeter(metrics));
- if (timestamp > 1000000000000000000L) {
- /**
- * Several versions of envoy in istio.deps send timestamp in nanoseconds,
- * instead of milliseconds(protocol says).
- *
- * Sadly, but have to fix it forcedly.
- *
- * An example of timestamp is '1552303033488741055', clearly it is not in milliseconds.
- *
- * This should be removed in the future.
- */
- timestamp /= 1_000_000;
- }
+ if (needHeartbeatUpdate && list.get(0).getMetricCount() > 0) {
+ final long timestamp = adapter.adaptTimestamp(list.get(0).getMetric(0));
- EnvoyInstanceMetric metricSource = new EnvoyInstanceMetric();
- metricSource.setServiceId(serviceId);
- metricSource.setServiceName(serviceName);
- metricSource.setId(serviceInstanceId);
- metricSource.setName(serviceInstanceName);
- metricSource.setMetricName(metricFamily.getName());
- metricSource.setValue(value);
- metricSource.setTimeBucket(TimeBucket.getMinuteTimeBucket(timestamp));
- sourceReceiver.receive(metricSource);
- }
- break;
- default:
- continue;
- }
- if (needHeartbeatUpdate) {
// Send heartbeat
ServiceInstanceUpdate serviceInstanceUpdate = new ServiceInstanceUpdate();
- serviceInstanceUpdate.setName(serviceInstanceName);
+ serviceInstanceUpdate.setName(service.getServiceInstanceName());
serviceInstanceUpdate.setServiceId(serviceId);
serviceInstanceUpdate.setTimeBucket(TimeBucket.getMinuteTimeBucket(timestamp));
sourceReceiver.receive(serviceInstanceUpdate);
needHeartbeatUpdate = false;
}
- } finally {
- timer.finish();
}
}
}
diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/metrics/adapters/ProtoMetricFamily2MetricsAdapter.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/metrics/adapters/ProtoMetricFamily2MetricsAdapter.java
new file mode 100644
index 000000000..ffcef3927
--- /dev/null
+++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/metrics/adapters/ProtoMetricFamily2MetricsAdapter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.receiver.envoy.metrics.adapters;
+
+import io.prometheus.client.Metrics;
+import java.util.Map;
+import java.util.stream.Stream;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Gauge;
+import org.apache.skywalking.oap.server.library.util.prometheus.metrics.Metric;
+
+import static java.util.stream.Collectors.toMap;
+
+@RequiredArgsConstructor
+public class ProtoMetricFamily2MetricsAdapter {
+ protected final Metrics.MetricFamily metricFamily;
+
+ public Stream adapt() {
+ switch (metricFamily.getType()) {
+ case GAUGE:
+ return metricFamily.getMetricList()
+ .stream()
+ .map(it -> Gauge.builder()
+ .name(adaptMetricsName(it))
+ .value(adaptValue(it))
+ .timestamp(adaptTimestamp(it))
+ .labels(adaptLabels(it))
+ .build());
+ default:
+ return Stream.of();
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public String adaptMetricsName(final Metrics.Metric metric) {
+ return metricFamily.getName();
+ }
+
+ public double adaptValue(final Metrics.Metric it) {
+ return it.getGauge().getValue();
+ }
+
+ public Map adaptLabels(final Metrics.Metric metric) {
+ return metric.getLabelList()
+ .stream()
+ .collect(toMap(Metrics.LabelPair::getName, Metrics.LabelPair::getValue));
+ }
+
+ public long adaptTimestamp(final Metrics.Metric metric) {
+ long timestamp = metric.getTimestampMs();
+
+ if (timestamp > 1000000000000000000L) {
+ /*
+ * Several versions of envoy in istio.deps send timestamp in nanoseconds,
+ * instead of milliseconds(protocol says).
+ *
+ * Sadly, but have to fix it forcefully.
+ *
+ * An example of timestamp is '1552303033488741055', clearly it is not in milliseconds.
+ *
+ * This should be removed in the future.
+ */
+ timestamp /= 1_000_000;
+ }
+
+ return timestamp;
+ }
+}
diff --git a/test/e2e/e2e-common/src/main/java/org/apache/skywalking/e2e/utils/Yamls.java b/test/e2e/e2e-common/src/main/java/org/apache/skywalking/e2e/utils/Yamls.java
index 054f753ca..c9b695e38 100644
--- a/test/e2e/e2e-common/src/main/java/org/apache/skywalking/e2e/utils/Yamls.java
+++ b/test/e2e/e2e-common/src/main/java/org/apache/skywalking/e2e/utils/Yamls.java
@@ -38,6 +38,10 @@ public final class Yamls {
T as(final Class klass);
}
+ public static boolean exists(final String file) {
+ return new ClassPathResource(Envs.resolve(file)).exists();
+ }
+
public static AsTypeBuilder load(final String file) throws IOException {
final InputStream inputStream = new ClassPathResource(Envs.resolve(file)).getInputStream();
diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
index 020e3ee39..d86e3eb62 100644
--- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
+++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
@@ -120,6 +120,18 @@ public class MetricsQuery extends AbstractQuery {
METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT
};
+ public static String[] ALL_ENVOY_LINER_METRICS = {
+ "envoy_heap_memory_used",
+ "envoy_heap_memory_max_used",
+ "envoy_memory_allocated",
+ "envoy_memory_allocated_max",
+ "envoy_memory_physical_size",
+ "envoy_memory_physical_size_max",
+ "envoy_total_connections_used",
+ "envoy_worker_threads",
+ "envoy_worker_threads_max"
+ };
+
public static String METER_INSTANCE_PERSISTENCE_EXECUTE_PERCENTILE = "meter_oap_instance_persistence_execute_percentile";
public static String[] ALL_SO11Y_LABELED_METRICS = {
diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/mesh/MetricsServiceE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/mesh/MetricsServiceE2E.java
new file mode 100644
index 000000000..0d87e9be3
--- /dev/null
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/mesh/MetricsServiceE2E.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.mesh;
+
+import com.google.common.base.Strings;
+import java.net.URL;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
+import org.apache.skywalking.e2e.base.TrafficController;
+import org.apache.skywalking.e2e.common.HostAndPort;
+import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
+import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
+import org.apache.skywalking.e2e.metrics.ReadMetrics;
+import org.apache.skywalking.e2e.metrics.ReadMetricsQuery;
+import org.apache.skywalking.e2e.retryable.RetryableTest;
+import org.apache.skywalking.e2e.service.Service;
+import org.apache.skywalking.e2e.service.ServicesMatcher;
+import org.apache.skywalking.e2e.service.ServicesQuery;
+import org.apache.skywalking.e2e.service.instance.Instance;
+import org.apache.skywalking.e2e.service.instance.Instances;
+import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
+import org.apache.skywalking.e2e.service.instance.InstancesQuery;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
+
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENVOY_LINER_METRICS;
+import static org.apache.skywalking.e2e.utils.Times.now;
+import static org.apache.skywalking.e2e.utils.Yamls.exists;
+import static org.apache.skywalking.e2e.utils.Yamls.load;
+
+@Slf4j
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class MetricsServiceE2E extends SkyWalkingTestAdapter {
+ private final String swWebappHost = Optional.ofNullable(Strings.emptyToNull(System.getenv("WEBAPP_HOST"))).orElse("127.0.0.1");
+
+ private final String swWebappPort = Optional.ofNullable(Strings.emptyToNull(System.getenv("WEBAPP_PORT"))).orElse("12800");
+
+ protected HostAndPort swWebappHostPort = HostAndPort.builder()
+ .host(swWebappHost)
+ .port(Integer.parseInt(swWebappPort))
+ .build();
+
+ @BeforeAll
+ public void setUp() throws Exception {
+ LOGGER.info("set up");
+
+ queryClient(swWebappHostPort);
+
+ String gatewayHost = Strings.isNullOrEmpty(System.getenv("GATEWAY_HOST")) ? "127.0.0.1" : System.getenv("GATEWAY_HOST");
+ String gatewayPort = Strings.isNullOrEmpty(System.getenv("GATEWAY_PORT")) ? "80" : System.getenv("GATEWAY_PORT");
+
+ HostAndPort serviceHostPort = HostAndPort.builder()
+ .host(gatewayHost)
+ .port(Integer.parseInt(gatewayPort))
+ .build();
+
+ final URL url = new URL("http", serviceHostPort.host(), serviceHostPort.port(), "/productpage");
+
+ trafficController =
+ TrafficController.builder()
+ .logResult(false)
+ .sender(() -> restTemplate.getForEntity(url.toURI(), String.class))
+ .build()
+ .start();
+
+ LOGGER.info("set up done");
+ }
+
+ @RetryableTest
+ void test() throws Exception {
+ List services = graphql.services(new ServicesQuery().start(startTime).end(now()));
+
+ services = services.stream().filter(s -> s.getLabel().startsWith("istio-dp::")).collect(Collectors.toList());
+ LOGGER.info("services: {}", services);
+ load("expected/metricsservice/services.yml").as(ServicesMatcher.class).verify(services);
+ for (final Service service : services) {
+ if (service.getLabel().contains("egressgateway")) {
+ continue;
+ }
+
+ final Instances instances = graphql.instances(
+ new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now())
+ );
+
+ LOGGER.info("instances: {}", instances);
+
+ String instancesFile = "expected/metricsservice/instances-" + service.getLabel() + ".yml";
+ instancesFile = instancesFile.replaceAll("::", "-");
+ if (!exists(instancesFile)) {
+ instancesFile = "expected/metricsservice/instances.yml";
+ }
+ load(instancesFile).as(InstancesMatcher.class).verify(instances);
+ for (Instance instance : instances.getInstances()) {
+ for (String metricsName : ALL_ENVOY_LINER_METRICS) {
+ LOGGER.info("verifying service instance: {}", instance);
+ final ReadMetrics instanceMetrics = graphql.readMetrics(
+ new ReadMetricsQuery().stepByMinute().metricsName(metricsName)
+ .serviceName(service.getLabel()).instanceName(instance.getLabel())
+ );
+
+ LOGGER.info("{}: {}", metricsName, instanceMetrics);
+ final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+ final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+ greaterThanZero.setValue("gt 0");
+ instanceRespTimeMatcher.setValue(greaterThanZero);
+ instanceRespTimeMatcher.verify(instanceMetrics.getValues());
+ }
+ }
+ }
+ }
+}
diff --git a/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances-istio-dp-reviews.yml b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances-istio-dp-reviews.yml
new file mode 100644
index 000000000..1abf02cff
--- /dev/null
+++ b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances-istio-dp-reviews.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+instances:
+ - key: not null
+ label: not null
+ - key: not null
+ label: not null
+ - key: not null
+ label: not null
diff --git a/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances.yml b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances.yml
new file mode 100644
index 000000000..7de572a77
--- /dev/null
+++ b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/instances.yml
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+instances:
+ - key: not null
+ label: not null
diff --git a/test/e2e/e2e-test/src/test/resources/expected/metricsservice/services.yml b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/services.yml
new file mode 100644
index 000000000..145398536
--- /dev/null
+++ b/test/e2e/e2e-test/src/test/resources/expected/metricsservice/services.yml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+ - key: not null
+ label: istio-dp::ratings
+ - key: not null
+ label: istio-dp::reviews
+ - key: not null
+ label: istio-dp::productpage
+ - key: not null
+ label: istio-dp::details
+ - key: not null
+ label: istio-dp::istio-ingressgateway
+ - key: not null
+ label: istio-dp::istio-egressgateway