test: extract so11y test and add logs (#5955)

* Increase metrics time range * Remove unused codes * Update SO11yE2E.java Co-authored-by: Hongtao Gao <hanahmily@gmail.com>
2020-12-06 22:15:37 +08:00 · 2020-12-06 22:15:37 +08:00 · b5367aa6b1
parent c24dbd6ec1
commit b5367aa6b1
8 changed files with 239 additions and 9 deletions
--- a/.github/workflows/e2e.so11y.yaml
+++ b/.github/workflows/e2e.so11y.yaml
@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: E2E
+
+on:
+  pull_request:
+    paths:
+      - '**'
+      - '!**.md'
+  push:
+    branches:
+      - master
+
+env:
+  SKIP_TEST: true
+  SW_AGENT_JDK_VERSION: 8
+  CODECOV_TOKEN: d2065307-8f01-4637-9715-2781ef096db7
+
+jobs:
+  SelfObservability:
+    name: SelfObservability
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Compile and Build
+        run: make docker
+      - name: Copy dist package
+        run: cp -R dist test/e2e/
+      - name: ${{ matrix.case }}
+        run: ./mvnw --batch-mode -f test/e2e/pom.xml -am -DfailIfNoTests=false verify -Dit.test=org.apache.skywalking.e2e.SO11yE2E
+      - name: Report Coverage
+        run: bash -x tools/coverage/report.sh
+      - uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: logs
+          path: logs
--- a/docker/oap/log4j2.xml
+++ b/docker/oap/log4j2.xml
@ -28,7 +28,9 @@
        <logger name="org.apache.zookeeper" level="INFO"/>
        <logger name="org.elasticsearch.common.network.IfConfig" level="INFO"/>
        <logger name="io.grpc.netty" level="INFO"/>
+        <logger name="org.apache.skywalking.oap.meter.analyzer" level="DEBUG"/>
        <logger name="org.apache.skywalking.oap.server.receiver.istio.telemetry" level="DEBUG"/>
+        <logger name="org.apache.skywalking.oap.server.fetcher.prometheus" level="DEBUG"/>
        <logger name="org.apache.skywalking.oap.server.receiver.envoy.als" level="DEBUG"/>
        <Root level="INFO">
            <AppenderRef ref="Console"/>
--- a/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml
+++ b/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml
@ -66,16 +66,16 @@ metricsRules:
  - name: instance_mesh_analysis_error_count
    exp: mesh_analysis_error_count.sum(['service', 'instance']).increase('PT1M')
  - name: instance_metrics_first_aggregation
-    exp: metrics_aggregation.tagEqual('dimensionality', 'minute', 'level', '1').sum(['service', 'instance']).increase('PT1M')
+    exp: metrics_aggregation.tagEqual('dimensionality', 'minute', 'level', '1').sum(['service', 'instance']).increase('PT5M')
  - name: instance_metrics_second_aggregation
-    exp: metrics_aggregation.tagEqual('dimensionality', 'minute', 'level', '2').sum(['service', 'instance']).increase('PT1M')
+    exp: metrics_aggregation.tagEqual('dimensionality', 'minute', 'level', '2').sum(['service', 'instance']).increase('PT5M')
  - name: instance_persistence_execute_percentile
    exp: persistence_timer_bulk_execute_latency.sum(['le', 'service', 'instance']).increase('PT5M').histogram().histogram_percentile([50,70,90,99])
  - name: instance_persistence_prepare_percentile
    exp: persistence_timer_bulk_prepare_latency.sum(['le', 'service', 'instance']).increase('PT5M').histogram().histogram_percentile([50,70,90,99])
  - name: instance_persistence_error_count
-    exp: persistence_timer_bulk_error_count.sum(['service', 'instance']).increase('PT1M')
+    exp: persistence_timer_bulk_error_count.sum(['service', 'instance']).increase('PT5M')
  - name: instance_persistence_execute_count
-    exp: persistence_timer_bulk_execute_latency_count.sum(['service', 'instance']).increase('PT1M')
+    exp: persistence_timer_bulk_execute_latency_count.sum(['service', 'instance']).increase('PT5M')
  - name: instance_persistence_prepare_count
-    exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'instance']).increase('PT1M')
+    exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'instance']).increase('PT5M')
--- a/oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/provider/PrometheusFetcherProvider.java
+++ b/oap-server/server-fetcher-plugin/prometheus-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/fetcher/prometheus/provider/PrometheusFetcherProvider.java
@ -134,6 +134,9 @@ public class PrometheusFetcherProvider extends ModuleProvider {
                                    result.addAll(mf.getMetrics());
                                }
                            }
+                            if (log.isDebugEnabled()) {
+                                log.debug("Fetch metrics from prometheus: {}", result);
+                            }
                            return result;
                        }))
                        .flatMap(tryIt -> MetricConvert.log(tryIt, "Load metric"))
--- a/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
+++ b/test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java
@ -115,7 +115,6 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
    public static String[] ALL_SO11Y_LINER_METRICS = {
        METER_INSTANCE_CPU_PERCENTAGE,
        METER_INSTANCE_JVM_MEMORY_BYTES_USED,
-        METER_INSTANCE_TRACE_COUNT,
        METER_INSTANCE_METRICS_FIRST_AGGREGATION,
        METER_INSTANCE_PERSISTENCE_PREPARE_COUNT,
        METER_INSTANCE_PERSISTENCE_EXECUTE_COUNT 
--- a/test/e2e/e2e-test/docker/simple/so11y/docker-compose.yml
+++ b/test/e2e/e2e-test/docker/simple/so11y/docker-compose.yml
@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  oap:
+    environment:
+      SW_PROMETHEUS_FETCHER: "default"
+      SW_TELEMETRY: prometheus
+    extends:
+      file: ../../base-compose.yml
+      service: oap
+
+  ui:
+    extends:
+      file: ../../base-compose.yml
+      service: ui
+    depends_on:
+      oap:
+        condition: service_healthy
+
+networks:
+  e2e:
--- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/SO11yE2E.java
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/SO11yE2E.java
@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
+import org.apache.skywalking.e2e.annotation.DockerCompose;
+import org.apache.skywalking.e2e.base.SkyWalkingE2E;
+import org.apache.skywalking.e2e.base.SkyWalkingTestAdapter;
+import org.apache.skywalking.e2e.common.HostAndPort;
+import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
+import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
+import org.apache.skywalking.e2e.metrics.ReadMetrics;
+import org.apache.skywalking.e2e.metrics.ReadMetricsQuery;
+import org.apache.skywalking.e2e.retryable.RetryableTest;
+import org.apache.skywalking.e2e.service.Service;
+import org.apache.skywalking.e2e.service.ServicesMatcher;
+import org.apache.skywalking.e2e.service.ServicesQuery;
+import org.apache.skywalking.e2e.service.instance.Instance;
+import org.apache.skywalking.e2e.service.instance.Instances;
+import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
+import org.apache.skywalking.e2e.service.instance.InstancesQuery;
+import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.DockerComposeContainer;
+
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SO11Y_LINER_METRICS;
+import static org.apache.skywalking.e2e.utils.Times.now;
+import static org.apache.skywalking.e2e.utils.Yamls.load;
+
+/**
+ * A test case for simple functionality verification
+ *
+ * The components are typically:
+ *
+ * - an single agent, (provider), generating traffic data
+ *
+ * - an OAP node, (oap)
+ *
+ * - a webapp, (ui) for querying
+ *
+ * The verifications are:
+ *
+ * - services
+ *
+ * - services metrics
+ *
+ * - services relations
+ *
+ * - endpoints
+ *
+ * - endpoints metrics
+ *
+ * - instances
+ *
+ * - instance metrics
+ *
+ * - topology
+ *
+ * - traces
+ *
+ * if your case needs the same aforementioned verifications, consider simply provide a docker-compose.yml with the specific orchestration and reuse these codes.
+ */
+@Slf4j
+@SkyWalkingE2E
+public class SO11yE2E extends SkyWalkingTestAdapter {
+    @SuppressWarnings("unused")
+    @DockerCompose("docker/simple/so11y/docker-compose.yml")
+    protected DockerComposeContainer<?> justForSideEffects;
+
+    @SuppressWarnings("unused")
+    @ContainerHostAndPort(name = "ui", port = 8080)
+    protected HostAndPort swWebappHostPort;
+
+    @BeforeAll
+    void setUp() {
+        queryClient(swWebappHostPort);
+    }
+
+    @RetryableTest
+    void so11y() throws Exception {
+        List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
+
+        services = services.stream().filter(s -> s.getLabel().equals("oap::oap-server")).collect(Collectors.toList());
+        LOGGER.info("services: {}", services);
+        load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services);
+        for (final Service service : services) {
+            final Instances instances = graphql.instances(
+                new InstancesQuery().serviceId(service.getKey()).start(startTime).end(now())
+            );
+
+            LOGGER.info("instances: {}", instances);
+
+            load("expected/simple/so11y-instances.yml").as(InstancesMatcher.class).verify(instances);
+            for (Instance instance : instances.getInstances()) {
+                for (String metricsName : ALL_SO11Y_LINER_METRICS) {
+                    LOGGER.info("verifying service instance response time: {}", instance);
+                    final ReadMetrics instanceMetrics = graphql.readMetrics(
+                        new ReadMetricsQuery().stepByMinute().metricsName(metricsName)
+                                              .serviceName(service.getLabel()).instanceName(instance.getLabel())
+                    );
+
+                    LOGGER.info("{}: {}", metricsName, instanceMetrics);
+                    final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                    final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                    greaterThanZero.setValue("gt 0");
+                    instanceRespTimeMatcher.setValue(greaterThanZero);
+                    instanceRespTimeMatcher.verify(instanceMetrics.getValues());
+                }
+            }
+        }
+    }
+}
--- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java
+++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java
@ -18,6 +18,8 @@

 package org.apache.skywalking.e2e.simple;

+import java.util.List;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
 import org.apache.skywalking.e2e.annotation.DockerCompose;
@ -54,9 +56,6 @@ import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.testcontainers.containers.DockerComposeContainer;

-import java.util.List;
-import java.util.stream.Collectors;
-
 import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
 import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyPercentileMetrics;
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;