kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add E2E test for VM monitoring that the metrics from Promethues node-exporter via OTEL. (#6404)

This commit is contained in:
wankai123 2021-02-20 16:11:44 +08:00 committed by GitHub
parent 6d92bc66dd
commit 6c6a8a7109
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 275 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.md
View File

@ -6,6 +6,7 @@ Release Notes.
------------------
#### Project
* Update frontend-maven-plugin to 1.11.0, for Download node x64 binary on Apple Silicon.
* Add E2E test for VM monitoring that metrics from Promethues node-exporter.
#### Java Agent
* Remove invalid mysql configuration in agent.config.

3
test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java
View File

@ -329,7 +329,8 @@ public class SimpleQueryClient {
.replace("{end}", query.end())
.replace("{metricsName}", query.metricsName())
.replace("{serviceName}", query.serviceName())
.replace("{instanceName}", query.instanceName());
.replace("{instanceName}", query.instanceName())
.replace("{scope}", query.scope());
LOGGER.info("Query: {}", queryString);
final ResponseEntity<GQLResponse<ReadMetricsData>> responseEntity = restTemplate.exchange(
new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),

18
test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
View File

@ -160,4 +160,22 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
.put("meter_agent_vfs_fs_size", Arrays.asList("/-total"))
.build();
public static String[] SIMPLE_PROM_VM_METERS = {
"meter_vm_memory_used",
"meter_vm_memory_total",
"meter_vm_memory_available",
"meter_vm_disk_written",
"meter_vm_network_transmit",
"meter_vm_tcp_curr_estab",
"meter_vm_tcp_alloc",
"meter_vm_sockets_used",
"meter_vm_udp_inuse",
"meter_vm_filefd_allocated"
};
public static Map<String, List<String>> SIMPLE_PROM_VM_LABELED_METERS = ImmutableMap.<String, List<String>>builder()
.put("meter_vm_cpu_average_used", Arrays.asList("idle"))
.put("meter_vm_filesystem_percentage", Arrays.asList("/etc/hosts"))
.build();
}

2
test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/ReadMetricsQuery.java
View File

@ -40,4 +40,6 @@ public class ReadMetricsQuery extends AbstractQuery<ReadMetricsQuery> {
private String instanceName;
private String scope = "ServiceInstance";
}

2
test/e2e/e2e-data/src/main/resources/read-metrics.gql
View File

@ -24,7 +24,7 @@
"condition":{
"name":"{metricsName}",
"entity":{
"scope":"ServiceInstance",
"scope":"{scope}",
"serviceName":"{serviceName}",
"serviceInstanceName":"{instanceName}",
"normal":true

21
test/e2e/e2e-test/docker/promOtelVM/Dockerfile.nodeExporter Normal file
View File

@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM ubuntu:latest
ADD https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_exporter-1.0.1.linux-amd64.tar.gz ./
RUN tar xvfz node_exporter-1.0.1.linux-amd64.tar.gz
WORKDIR /node_exporter-1.0.1.linux-amd64
CMD ["./node_exporter"]

56
test/e2e/e2e-test/docker/promOtelVM/docker-compose.yml Normal file
View File

@ -0,0 +1,56 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
version: '2.1'
services:
oap:
extends:
file: ../base-compose.yml
service: oap
environment:
SW_OTEL_RECEIVER: default
SW_OTEL_RECEIVER_ENABLED_OC_RULES: vm
node-exporter:
build:
context: .
dockerfile: Dockerfile.nodeExporter
networks:
- e2e
expose:
- 9100
otel-collector:
image: otel/opentelemetry-collector:0.19.0
networks:
- e2e
command: [ "--config=/etc/otel-collector-config.yaml" ]
volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
expose:
- 55678
depends_on:
oap:
condition: service_healthy
ui:
extends:
file: ../base-compose.yml
service: ui
depends_on:
oap:
condition: service_healthy
networks:
e2e:

41
test/e2e/e2e-test/docker/promOtelVM/otel-collector-config.yaml Normal file
View File

@ -0,0 +1,41 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
receivers:
prometheus:
config:
scrape_configs:
- job_name: 'otel-collector'
scrape_interval: 10s
static_configs:
- targets: [ 'node-exporter:9100' ]
processors:
batch:
exporters:
opencensus:
endpoint: "oap:11800" # The OAP Server address
insecure: true
# Exports data to the console
logging:
logLevel: debug
service:
pipelines:
metrics:
receivers: [prometheus]
processors: [batch]
exporters: [opencensus,logging]

115
test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/promOtelVM/PromOtelVME2E.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.e2e.promOtelVM;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.e2e.UIConfigurationManagementClient;
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
import org.apache.skywalking.e2e.annotation.DockerCompose;
import org.apache.skywalking.e2e.base.SkyWalkingE2E;
import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
import org.apache.skywalking.e2e.common.HostAndPort;
import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
import org.apache.skywalking.e2e.metrics.Metrics;
import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
import org.apache.skywalking.e2e.metrics.ReadLabeledMetricsQuery;
import org.apache.skywalking.e2e.metrics.ReadMetrics;
import org.apache.skywalking.e2e.metrics.ReadMetricsQuery;
import org.apache.skywalking.e2e.retryable.RetryableTest;
import org.apache.skywalking.e2e.service.Service;
import org.apache.skywalking.e2e.service.ServicesMatcher;
import org.apache.skywalking.e2e.service.ServicesQuery;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.DockerComposeContainer;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.SIMPLE_PROM_VM_LABELED_METERS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.SIMPLE_PROM_VM_METERS;
import static org.apache.skywalking.e2e.utils.Times.now;
import static org.apache.skywalking.e2e.utils.Yamls.load;
@Slf4j
@SkyWalkingE2E
public class PromOtelVME2E extends SkyWalkingTestAdapter {
@DockerCompose({"docker/promOtelVM/docker-compose.yml"})
private DockerComposeContainer<?> compose;
@ContainerHostAndPort(name = "ui", port = 8080)
private HostAndPort swWebappHostPort;
private UIConfigurationManagementClient graphql;
@BeforeAll
public void setUp() throws Exception {
graphql = new UIConfigurationManagementClient(swWebappHostPort.host(), swWebappHostPort.port());
}
@RetryableTest
void testMetrics() throws Exception {
List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList());
LOGGER.info("services: {}", services);
load("expected/promOtelVM/services.yml").as(ServicesMatcher.class).verify(services);
Service service = services.get(0);
for (String metricsName : SIMPLE_PROM_VM_METERS) {
LOGGER.info("verifying prom vm metrics: {}", metricsName);
ReadMetrics metrics = graphql.readMetrics(
new ReadMetricsQuery().stepByMinute()
.metricsName(metricsName)
.serviceName(service.getLabel())
.scope("Service")
.instanceName("")
);
LOGGER.info("prom vm metrics: {}", metrics);
final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
greaterThanZero.setValue("gt 0");
instanceRespTimeMatcher.setValue(greaterThanZero);
instanceRespTimeMatcher.verify(metrics.getValues());
}
for (Map.Entry<String, List<String>> entry : SIMPLE_PROM_VM_LABELED_METERS.entrySet()) {
String metricsName = entry.getKey();
List<String> labels = entry.getValue();
LOGGER.info("verifying prom vm labeledMetrics: {}", metricsName);
List<ReadMetrics> labeledMetrics = graphql.readLabeledMetrics(
new ReadLabeledMetricsQuery().stepByMinute().metricsName(metricsName)
.serviceName(service.getLabel()).scope("Service").instanceName("")
.labels(labels)
);
LOGGER.info("prom vm labeledMetrics: {}", labeledMetrics);
Metrics allValues = new Metrics();
for (ReadMetrics readMetrics : labeledMetrics) {
allValues.getValues().addAll(readMetrics.getValues().getValues());
}
final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
greaterThanZero.setValue("gt 0");
instanceRespTimeMatcher.setValue(greaterThanZero);
instanceRespTimeMatcher.verify(allValues);
}
}
}

18
test/e2e/e2e-test/src/test/resources/expected/promOtelVM/services.yml Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
- key: not null
label: "vm::node-exporter"