From e143ae62eb8a2300b124f345b235bb1e3eb3c139 Mon Sep 17 00:00:00 2001 From: Gao Hongtao Date: Mon, 22 Jun 2020 09:10:51 +0800 Subject: [PATCH] Reduce the length of metric name (#4936) --- .../resources/fetcher-prom-rules/self.yaml | 10 +-- .../resources/ui-initialized-templates.yml | 6 +- .../skywalking/e2e/SimpleQueryClient.java | 58 +++++++++++++++++ .../skywalking/e2e/metrics/MetricsQuery.java | 21 +++++++ .../e2e/metrics/ReadLabeledMetricsData.java | 28 +++++++++ .../skywalking/e2e/metrics/ReadMetrics.java | 27 ++++++++ .../e2e/metrics/ReadMetricsData.java | 26 ++++++++ .../e2e/metrics/ReadMetricsQuery.java | 43 +++++++++++++ .../e2e/service/ServicesMatcher.java | 4 ++ .../main/resources/read-labeled-metrics.gql | 40 ++++++++++++ .../src/main/resources/read-metrics.gql | 34 ++++++++++ .../docker/simple/auth/docker-compose.yml | 2 + .../docker/simple/jdk/docker-compose.yml | 3 + .../docker/simple/ssl/docker-compose.yml | 2 + .../docker/storage/docker-compose.es6.yml | 2 + .../docker/storage/docker-compose.es7.yml | 2 + .../storage/docker-compose.influxdb.yml | 2 + .../docker/storage/docker-compose.mysql.yml | 2 + .../skywalking/e2e/simple/SimpleE2E.java | 63 ++++++++++++++++++- .../skywalking/e2e/storage/StorageE2E.java | 61 +++++++++++++++++- .../expected/simple/so11y-instances.yml | 18 ++++++ .../expected/simple/so11y-services.yml | 18 ++++++ 22 files changed, 461 insertions(+), 11 deletions(-) create mode 100644 test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadLabeledMetricsData.java create mode 100644 test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetrics.java create mode 100644 test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsData.java create mode 100644 test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsQuery.java create mode 100644 test/e2e/e2e-data/src/main/resources/read-labeled-metrics.gql create mode 100644 test/e2e/e2e-data/src/main/resources/read-metrics.gql create mode 100644 test/e2e/e2e-test/src/test/resources/expected/simple/so11y-instances.yml create mode 100644 test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml diff --git a/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml b/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml index 2a703c081..f0f0368dd 100644 --- a/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml +++ b/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml @@ -241,7 +241,7 @@ metricsRules: - service instance: - instance - - name: instance_persistence_timer_execute_latency_percentile + - name: instance_persistence_execute_percentile scope: SERVICE_INSTANCE operation: avgHistogramPercentile percentiles: [50, 70, 90, 99] @@ -254,7 +254,7 @@ metricsRules: - service instance: - instance - - name: instance_persistence_timer_prepare_latency_percentile + - name: instance_persistence_prepare_percentile scope: SERVICE_INSTANCE operation: avgHistogramPercentile percentiles: [50, 70, 90, 99] @@ -267,7 +267,7 @@ metricsRules: - service instance: - instance - - name: instance_persistence_timer_bulk_error_count + - name: instance_persistence_error_count scope: SERVICE_INSTANCE operation: avg sources: @@ -279,7 +279,7 @@ metricsRules: - service instance: - instance - - name: instance_persistence_timer_execute_count + - name: instance_persistence_execute_count scope: SERVICE_INSTANCE operation: avg sources: @@ -291,7 +291,7 @@ metricsRules: - service instance: - instance - - name: instance_persistence_timer_prepare_count + - name: instance_persistence_prepare_count scope: SERVICE_INSTANCE operation: avg sources: diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml index 20dd8974b..0a2dcfd83 100644 --- a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml +++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates.yml @@ -1025,7 +1025,7 @@ templates: "entityType": "ServiceInstance", "independentSelector": false, "metricType": "REGULAR_VALUE", - "metricName": "meter_instance_persistence_timer_prepare_count,meter_instance_persistence_timer_execute_count,meter_instance_persistence_timer_bulk_error_count", + "metricName": "meter_instance_persistence_prepare_count,meter_instance_persistence_execute_count,meter_instance_persistence_error_count", "queryMetricType": "readMetricsValues", "chartType": "ChartBar", "unit": "Per 5 Minutes" @@ -1040,7 +1040,7 @@ templates: "unit": "Millisecond", "queryMetricType": "readLabeledMetricsValues", "chartType": "ChartLine", - "metricName": "meter_instance_persistence_timer_prepare_latency_percentile", + "metricName": "meter_instance_persistence_prepare_percentile", "metricLabels": "50,70,90,99", "labelsIndex": "50,70,90,99" }, @@ -1051,7 +1051,7 @@ templates: "entityType": "ServiceInstance", "independentSelector": false, "metricType": "LABELED_VALUE", - "metricName": "meter_instance_persistence_timer_execute_latency_percentile", + "metricName": "meter_instance_persistence_execute_percentile", "queryMetricType": "readLabeledMetricsValues", "chartType": "ChartLine", "metricLabels": "50,70,90,99", diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java index 32b2771ff..5287dc316 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java @@ -29,6 +29,10 @@ import org.apache.skywalking.e2e.metrics.Metrics; import org.apache.skywalking.e2e.metrics.MetricsData; import org.apache.skywalking.e2e.metrics.MetricsQuery; import org.apache.skywalking.e2e.metrics.MultiMetricsData; +import org.apache.skywalking.e2e.metrics.ReadLabeledMetricsData; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; +import org.apache.skywalking.e2e.metrics.ReadMetrics; +import org.apache.skywalking.e2e.metrics.ReadMetricsData; import org.apache.skywalking.e2e.service.Service; import org.apache.skywalking.e2e.service.ServicesData; import org.apache.skywalking.e2e.service.ServicesQuery; @@ -52,7 +56,10 @@ import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; +import lombok.extern.slf4j.Slf4j; + @SuppressWarnings("UnstableApiUsage") +@Slf4j public class SimpleQueryClient { protected final RestTemplate restTemplate = new RestTemplate(); @@ -253,4 +260,55 @@ public class SimpleQueryClient { return Objects.requireNonNull(responseEntity.getBody()).getData().getMetrics(); } + public ReadMetrics readMetrics(final ReadMetricsQuery query) throws Exception { + final URL queryFileUrl = Resources.getResource("read-metrics.gql"); + final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) + .stream() + .filter(it -> !it.startsWith("#")) + .collect(Collectors.joining()) + .replace("{step}", query.step()) + .replace("{start}", query.start()) + .replace("{end}", query.end()) + .replace("{metricsName}", query.metricsName()) + .replace("{serviceName}", query.serviceName()) + .replace("{instanceName}", query.instanceName()); + LOGGER.info("Query: {}", queryString); + final ResponseEntity> responseEntity = restTemplate.exchange( + new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)), + new ParameterizedTypeReference>() { + } + ); + + if (responseEntity.getStatusCode() != HttpStatus.OK) { + throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode()); + } + + return Objects.requireNonNull(responseEntity.getBody()).getData().getReadMetricsValues(); + } + + public List readLabeledMetrics(final ReadMetricsQuery query) throws Exception { + final URL queryFileUrl = Resources.getResource("read-labeled-metrics.gql"); + final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) + .stream() + .filter(it -> !it.startsWith("#")) + .collect(Collectors.joining()) + .replace("{step}", query.step()) + .replace("{start}", query.start()) + .replace("{end}", query.end()) + .replace("{metricsName}", query.metricsName()) + .replace("{serviceName}", query.serviceName()) + .replace("{instanceName}", query.instanceName()); + LOGGER.info("Query: {}", queryString); + final ResponseEntity> responseEntity = restTemplate.exchange( + new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)), + new ParameterizedTypeReference>() { + } + ); + + if (responseEntity.getStatusCode() != HttpStatus.OK) { + throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode()); + } + + return Objects.requireNonNull(responseEntity.getBody()).getData().getReadLabeledMetricsValues(); + } } diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java index 9f719a164..97f8e275b 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java @@ -96,6 +96,27 @@ public class MetricsQuery extends AbstractQuery { SERVICE_INSTANCE_RELATION_SERVER_CPM }; + public static String METER_INSTANCE_CPU_PERCENTAGE = "meter_instance_cpu_percentage"; + public static String METER_INSTANCE_JVM_MEMORY_BYTES_USED = "meter_instance_jvm_memory_bytes_used"; + public static String METER_INSTANCE_TRACE_COUNT = "meter_instance_trace_count"; + public static String METER_INSTANCE_METRICS_FIRST_AGGREGATION = "meter_instance_metrics_first_aggregation"; + public static String METER_INSTANCE_PERSISTENCE_PREPARE_COUNT = "meter_instance_persistence_prepare_count"; + public static String METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT = "meter_instance_persistence_execute_count"; + + public static String[] ALL_SO11Y_LINER_METRICS = { + METER_INSTANCE_CPU_PERCENTAGE, + METER_INSTANCE_JVM_MEMORY_BYTES_USED, + METER_INSTANCE_TRACE_COUNT, + METER_INSTANCE_METRICS_FIRST_AGGREGATION, + METER_INSTANCE_PERSISTENCE_PREPARE_COUNT, + METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT + }; + + public static String METER_INSTANCE_PERSISTENCE_EXECUTE_PERCENTILE = "meter_instance_persistence_execute_percentile"; + + public static String[] ALL_SO11Y_LABELED_METRICS = { + METER_INSTANCE_PERSISTENCE_EXECUTE_PERCENTILE + }; private String id; private String metricsName; diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadLabeledMetricsData.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadLabeledMetricsData.java new file mode 100644 index 000000000..98f4665c0 --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadLabeledMetricsData.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.metrics; + +import java.util.List; + +import lombok.Data; + +@Data +public class ReadLabeledMetricsData { + private List readLabeledMetricsValues; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetrics.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetrics.java new file mode 100644 index 000000000..dd8507902 --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetrics.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.metrics; + +import lombok.Data; + +@Data +public class ReadMetrics { + private String label; + private Metrics values; +} \ No newline at end of file diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsData.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsData.java new file mode 100644 index 000000000..140db40cb --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsData.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.metrics; + +import lombok.Data; + +@Data +public class ReadMetricsData { + private ReadMetrics readMetricsValues; +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsQuery.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsQuery.java new file mode 100644 index 000000000..f1b40773a --- /dev/null +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsQuery.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.metrics; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.Accessors; +import org.apache.skywalking.e2e.AbstractQuery; + +@Data +@Accessors(fluent = true) +@EqualsAndHashCode(callSuper = true) +public class ReadMetricsQuery extends AbstractQuery { + + public static String METER_INSTANCE_CPU_PERCENTAGE = "meter_instance_cpu_percentage"; + + public static String[] ALL_SO11Y_LINER_METRICS = { + METER_INSTANCE_CPU_PERCENTAGE + }; + + private String metricsName; + + private String serviceName; + + private String instanceName; + +} diff --git a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/service/ServicesMatcher.java b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/service/ServicesMatcher.java index cde40d541..38cc9d96e 100644 --- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/service/ServicesMatcher.java +++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/service/ServicesMatcher.java @@ -21,11 +21,13 @@ package org.apache.skywalking.e2e.service; import java.util.LinkedList; import java.util.List; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; @Data +@Slf4j public class ServicesMatcher { private List services; @@ -34,6 +36,8 @@ public class ServicesMatcher { } public void verify(final List services) { + + LOGGER.info("services:{} matchers:{}", services, this.getServices()); assertThat(services).hasSameSizeAs(this.getServices()); for (int i = 0; i < getServices().size(); i++) { diff --git a/test/e2e/e2e-data/src/main/resources/read-labeled-metrics.gql b/test/e2e/e2e-data/src/main/resources/read-labeled-metrics.gql new file mode 100644 index 000000000..4bdc36f68 --- /dev/null +++ b/test/e2e/e2e-data/src/main/resources/read-labeled-metrics.gql @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "query":"query queryData($condition: MetricsCondition!, $labels: [String!]!, $duration: Duration!) {\n readLabeledMetricsValues: readLabeledMetricsValues(\n condition: $condition,\n labels: $labels,\n duration: $duration) {\n label\n values {\n values {value}\n }\n }}", + "variables":{ + "duration":{ + "start":"{start}", + "end":"{end}", + "step":"{step}" + }, + "condition":{ + "name":"{metricsName}", + "entity":{ + "scope":"ServiceInstance", + "serviceName":"{serviceName}", + "serviceInstanceName":"{instanceName}", + "normal":true + } + }, + "labels":[ + "50", + "70", + "90", + "99" + ] + } +} \ No newline at end of file diff --git a/test/e2e/e2e-data/src/main/resources/read-metrics.gql b/test/e2e/e2e-data/src/main/resources/read-metrics.gql new file mode 100644 index 000000000..5889e8058 --- /dev/null +++ b/test/e2e/e2e-data/src/main/resources/read-metrics.gql @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "query":"query queryData($condition: MetricsCondition!, $duration: Duration!) {\n readMetricsValues: readMetricsValues(condition: $condition, duration: $duration) {\n label\n values {\n values {value}\n }\n }}", + "variables":{ + "duration":{ + "start":"{start}", + "end":"{end}", + "step":"{step}" + }, + "condition":{ + "name":"{metricsName}", + "entity":{ + "scope":"ServiceInstance", + "serviceName":"{serviceName}", + "serviceInstanceName":"{instanceName}", + "normal":true + } + } + } +} \ No newline at end of file diff --git a/test/e2e/e2e-test/docker/simple/auth/docker-compose.yml b/test/e2e/e2e-test/docker/simple/auth/docker-compose.yml index 444e98232..c816f305a 100644 --- a/test/e2e/e2e-test/docker/simple/auth/docker-compose.yml +++ b/test/e2e/e2e-test/docker/simple/auth/docker-compose.yml @@ -22,6 +22,8 @@ services: service: oap environment: SW_AUTHENTICATION: test-token + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus ui: extends: diff --git a/test/e2e/e2e-test/docker/simple/jdk/docker-compose.yml b/test/e2e/e2e-test/docker/simple/jdk/docker-compose.yml index 562ed6195..9985b8a03 100644 --- a/test/e2e/e2e-test/docker/simple/jdk/docker-compose.yml +++ b/test/e2e/e2e-test/docker/simple/jdk/docker-compose.yml @@ -17,6 +17,9 @@ version: '2.1' services: oap: + environment: + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus extends: file: ../../base-compose.yml service: oap diff --git a/test/e2e/e2e-test/docker/simple/ssl/docker-compose.yml b/test/e2e/e2e-test/docker/simple/ssl/docker-compose.yml index 97bd39737..ae608d1ab 100644 --- a/test/e2e/e2e-test/docker/simple/ssl/docker-compose.yml +++ b/test/e2e/e2e-test/docker/simple/ssl/docker-compose.yml @@ -27,6 +27,8 @@ services: SW_CORE_GRPC_SSL_KEY_PATH: /skywalking/certs/server-key.pem SW_CORE_GRPC_SSL_CERT_CHAIN_PATH: /skywalking/certs/server.crt SW_CORE_GRPC_SSL_TRUSTED_CA_PATH: /skywalking/certs/ca.crt + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus ui: extends: diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.es6.yml b/test/e2e/e2e-test/docker/storage/docker-compose.es6.yml index b80079ac3..e937609b7 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.es6.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.es6.yml @@ -37,6 +37,8 @@ services: service: oap environment: SW_STORAGE: elasticsearch + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus depends_on: es: condition: service_healthy diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.es7.yml b/test/e2e/e2e-test/docker/storage/docker-compose.es7.yml index 858a5280b..a9c974462 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.es7.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.es7.yml @@ -37,6 +37,8 @@ services: service: oap-es7 environment: SW_STORAGE: elasticsearch7 + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus depends_on: es: condition: service_healthy diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/storage/docker-compose.influxdb.yml index 377ea791e..184fa08f6 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.influxdb.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.influxdb.yml @@ -34,6 +34,8 @@ services: service: oap environment: SW_STORAGE: influxdb + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus depends_on: influxdb: condition: service_healthy diff --git a/test/e2e/e2e-test/docker/storage/docker-compose.mysql.yml b/test/e2e/e2e-test/docker/storage/docker-compose.mysql.yml index 04917800d..096d71b3f 100644 --- a/test/e2e/e2e-test/docker/storage/docker-compose.mysql.yml +++ b/test/e2e/e2e-test/docker/storage/docker-compose.mysql.yml @@ -38,6 +38,8 @@ services: service: oap environment: SW_STORAGE: mysql + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus depends_on: mysql: condition: service_healthy diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java index d124897cc..67af38853 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java @@ -19,6 +19,8 @@ package org.apache.skywalking.e2e.simple; import java.util.List; +import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; import org.apache.skywalking.e2e.annotation.DockerCompose; @@ -29,6 +31,8 @@ import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; import org.apache.skywalking.e2e.metrics.Metrics; import org.apache.skywalking.e2e.metrics.MetricsQuery; import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.metrics.ReadMetrics; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; import org.apache.skywalking.e2e.retryable.RetryableTest; import org.apache.skywalking.e2e.service.Service; import org.apache.skywalking.e2e.service.ServicesMatcher; @@ -63,6 +67,8 @@ import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANC import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_CLIENT_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_SERVER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LINER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LABELED_METRICS; import static org.apache.skywalking.e2e.utils.Times.now; import static org.apache.skywalking.e2e.utils.Yamls.load; @@ -126,11 +132,12 @@ public class SimpleE2E extends SkyWalkingTestAdapter { public void tearDown() { trafficController.stop(); } - + @RetryableTest void services() throws Exception { - final List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/services.yml").as(ServicesMatcher.class).verify(services); @@ -185,6 +192,58 @@ public class SimpleE2E extends SkyWalkingTestAdapter { verifyServiceInstanceRelationMetrics(topology.getCalls()); } + + @RetryableTest + void so11y() throws Exception { + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + LOGGER.info("services: {}", services); + load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); + for (final Service service : services) { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/simple/so11y-instances.yml").as(InstancesMatcher.class).verify(instances); + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_SO11Y_LINER_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final ReadMetrics instanceMetrics = graphql.readMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics.getValues()); + } + for (String metricsName : ALL_SO11Y_LABELED_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final List instanceMetrics = graphql.readLabeledMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + Metrics allValues = new Metrics(); + for (ReadMetrics readMetrics : instanceMetrics) { + allValues.getValues().addAll(readMetrics.getValues().getValues()); + } + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(allValues); + } + } + } + } private Instances verifyServiceInstances(final Service service) throws Exception { final Instances instances = graphql.instances( diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java index 082e5f9a4..d9da6494c 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java @@ -21,6 +21,8 @@ package org.apache.skywalking.e2e.storage; import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.e2e.UIConfigurationManagementClient; import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; @@ -38,6 +40,8 @@ import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; import org.apache.skywalking.e2e.metrics.Metrics; import org.apache.skywalking.e2e.metrics.MetricsQuery; import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.metrics.ReadMetrics; +import org.apache.skywalking.e2e.metrics.ReadMetricsQuery; import org.apache.skywalking.e2e.retryable.RetryableTest; import org.apache.skywalking.e2e.service.Service; import org.apache.skywalking.e2e.service.ServicesMatcher; @@ -73,6 +77,8 @@ import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_INSTANC import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_CLIENT_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_RELATION_SERVER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LINER_METRICS; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LABELED_METRICS; import static org.apache.skywalking.e2e.utils.Times.now; import static org.apache.skywalking.e2e.utils.Yamls.load; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -112,8 +118,9 @@ public class StorageE2E extends SkyWalkingTestAdapter { @RetryableTest void services() throws Exception { - final List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/storage/services.yml").as(ServicesMatcher.class).verify(services); @@ -219,6 +226,58 @@ public class StorageE2E extends SkyWalkingTestAdapter { verifyTemplates("expected/storage/dashboardConfiguration-disable.yml"); } + @RetryableTest + void so11y() throws Exception { + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + + services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + LOGGER.info("services: {}", services); + load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); + for (final Service service : services) { + final Instances instances = graphql.instances( + new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) + ); + + LOGGER.info("instances: {}", instances); + + load("expected/simple/so11y-instances.yml").as(InstancesMatcher.class).verify(instances); + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_SO11Y_LINER_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final ReadMetrics instanceMetrics = graphql.readMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics.getValues()); + } + for (String metricsName : ALL_SO11Y_LABELED_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final List instanceMetrics = graphql.readLabeledMetrics( + new ReadMetricsQuery().stepByMinute().metricsName(metricsName) + .serviceName(service.getLabel()).instanceName(instance.getLabel()) + ); + + LOGGER.info("{}: {}", metricsName, instanceMetrics); + Metrics allValues = new Metrics(); + for (ReadMetrics readMetrics : instanceMetrics) { + allValues.getValues().addAll(readMetrics.getValues().getValues()); + } + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(allValues); + } + } + } + } + private Instances verifyServiceInstances(final Service service) throws Exception { final Instances instances = graphql.instances( new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now()) diff --git a/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-instances.yml b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-instances.yml new file mode 100644 index 000000000..7de572a77 --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-instances.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +instances: + - key: not null + label: not null diff --git a/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml new file mode 100644 index 000000000..c19deb25c --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + - key: not null + label: "oap-server"