From d1dcd3f96cea210e1613c3fdfe224b8282507a73 Mon Sep 17 00:00:00 2001 From: wankai123 Date: Fri, 22 Oct 2021 10:54:49 +0800 Subject: [PATCH] Replace e2e cases to e2e-v2: Kafka: Base, Meter, Log, Profile (#7986) * Replace e2e cases to e2e-v2: Kafka: Base, Meter, Log, Profile * Set `SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG`, `SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG` default `true`. --- .github/workflows/e2e.kafka.yaml | 13 +- CHANGES.md | 2 + .../setup/backend/configuration-vocabulary.md | 4 +- .../kafka/module/KafkaFetcherConfig.java | 4 +- .../src/main/resources/application.yml | 4 +- test/e2e-v2/cases/kafka/Dockerfile.provider | 33 ++++ .../e2e-v2/cases/kafka/log/Dockerfile.fluentd | 19 ++ .../cases/kafka/log/Dockerfile.satellite | 24 +++ .../e2e-v2/cases/kafka/log/docker-compose.yml | 178 ++++++++++++++++++ test/e2e-v2/cases/kafka/log/e2e.yaml | 59 ++++++ .../kafka/log/expected/logs-filebeat.yml | 112 +++++++++++ .../cases/kafka/log/expected/logs-fluentd.yml | 82 ++++++++ test/e2e-v2/cases/kafka/log/filebeat.yml | 91 +++++++++ test/e2e-v2/cases/kafka/log/fluentd.conf | 66 +++++++ .../cases/kafka/log/satellite_config.yaml | 61 ++++++ .../cases/kafka/meter/docker-compose.yml | 125 ++++++++++++ test/e2e-v2/cases/kafka/meter/e2e.yaml | 47 +++++ .../cases/kafka/profile/docker-compose.yml | 139 ++++++++++++++ test/e2e-v2/cases/kafka/profile/e2e.yaml | 37 ++++ .../kafka/simple-so11y/docker-compose.yml | 137 ++++++++++++++ test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml | 48 +++++ test/e2e-v2/cases/log/es/e2e.yaml | 18 +- test/e2e-v2/cases/log/fluent-bit/e2e.yaml | 18 +- test/e2e-v2/cases/log/h2/e2e.yaml | 18 +- test/e2e-v2/cases/log/influxdb/e2e.yaml | 18 +- test/e2e-v2/cases/log/log-cases.yaml | 32 ++++ test/e2e-v2/cases/log/mysql/e2e.yaml | 18 +- test/e2e-v2/cases/log/postgres/e2e.yaml | 18 +- test/e2e-v2/cases/meter/e2e.yaml | 15 +- test/e2e-v2/cases/meter/meter-cases.yaml | 29 +++ test/e2e-v2/cases/simple/auth/e2e.yaml | 36 +--- .../expected/dependency-endpoint-consumer.yml | 53 ++++++ ...t.yml => dependency-endpoint-provider.yml} | 0 .../expected/dependency-services-consumer.yml | 48 +++++ ...s.yml => dependency-services-provider.yml} | 0 .../expected/metrics-has-value-percentile.yml | 47 +++++ .../expected/service-endpoint-consumer.yml | 19 ++ ...oint.yml => service-endpoint-provider.yml} | 0 .../expected/service-instance-consumer.yml | 40 ++++ ...ance.yml => service-instance-provider.yml} | 0 test/e2e-v2/cases/simple/jdk/e2e.yaml | 36 +--- test/e2e-v2/cases/simple/mtls/e2e.yaml | 36 +--- test/e2e-v2/cases/simple/simple-cases.yaml | 131 +++++++++++++ test/e2e-v2/cases/simple/ssl/e2e.yaml | 36 +--- test/e2e-v2/cases/so11y/e2e.yaml | 29 +-- test/e2e-v2/cases/so11y/so11y-cases.yaml | 44 +++++ .../e2e-service-provider/pom.xml | 6 +- .../e2e/controller/FileLogController.java | 6 +- .../e2e/controller/LogController.java | 6 +- test/e2e-v2/script/env | 2 +- 50 files changed, 1751 insertions(+), 293 deletions(-) create mode 100644 test/e2e-v2/cases/kafka/Dockerfile.provider create mode 100644 test/e2e-v2/cases/kafka/log/Dockerfile.fluentd create mode 100644 test/e2e-v2/cases/kafka/log/Dockerfile.satellite create mode 100644 test/e2e-v2/cases/kafka/log/docker-compose.yml create mode 100644 test/e2e-v2/cases/kafka/log/e2e.yaml create mode 100644 test/e2e-v2/cases/kafka/log/expected/logs-filebeat.yml create mode 100644 test/e2e-v2/cases/kafka/log/expected/logs-fluentd.yml create mode 100644 test/e2e-v2/cases/kafka/log/filebeat.yml create mode 100644 test/e2e-v2/cases/kafka/log/fluentd.conf create mode 100644 test/e2e-v2/cases/kafka/log/satellite_config.yaml create mode 100644 test/e2e-v2/cases/kafka/meter/docker-compose.yml create mode 100644 test/e2e-v2/cases/kafka/meter/e2e.yaml create mode 100644 test/e2e-v2/cases/kafka/profile/docker-compose.yml create mode 100644 test/e2e-v2/cases/kafka/profile/e2e.yaml create mode 100644 test/e2e-v2/cases/kafka/simple-so11y/docker-compose.yml create mode 100644 test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml create mode 100644 test/e2e-v2/cases/log/log-cases.yaml create mode 100644 test/e2e-v2/cases/meter/meter-cases.yaml create mode 100644 test/e2e-v2/cases/simple/expected/dependency-endpoint-consumer.yml rename test/e2e-v2/cases/simple/expected/{dependency-endpoint.yml => dependency-endpoint-provider.yml} (100%) create mode 100644 test/e2e-v2/cases/simple/expected/dependency-services-consumer.yml rename test/e2e-v2/cases/simple/expected/{dependency-services.yml => dependency-services-provider.yml} (100%) create mode 100644 test/e2e-v2/cases/simple/expected/metrics-has-value-percentile.yml create mode 100644 test/e2e-v2/cases/simple/expected/service-endpoint-consumer.yml rename test/e2e-v2/cases/simple/expected/{service-endpoint.yml => service-endpoint-provider.yml} (100%) create mode 100644 test/e2e-v2/cases/simple/expected/service-instance-consumer.yml rename test/e2e-v2/cases/simple/expected/{service-instance.yml => service-instance-provider.yml} (100%) create mode 100644 test/e2e-v2/cases/simple/simple-cases.yaml create mode 100644 test/e2e-v2/cases/so11y/so11y-cases.yaml diff --git a/.github/workflows/e2e.kafka.yaml b/.github/workflows/e2e.kafka.yaml index fdbbd81dc8..84207ba708 100644 --- a/.github/workflows/e2e.kafka.yaml +++ b/.github/workflows/e2e.kafka.yaml @@ -26,7 +26,6 @@ on: env: SW_AGENT_JDK_VERSION: 8 - SW_STORAGE: influxdb concurrency: group: e2e-kafka-${{ github.event.pull_request.number || github.ref }} @@ -42,13 +41,13 @@ jobs: matrix: case: - name: Base - class: org.apache.skywalking.e2e.kafka.KafkaE2E + config-file: kafka/simple-so11y/e2e.yaml - name: Profiling - class: org.apache.skywalking.e2e.kafka.KafkaProfileE2E + config-file: kafka/profile/e2e.yaml - name: Meter - class: org.apache.skywalking.e2e.kafka.KafkaMeterE2E + config-file: kafka/meter/e2e.yaml - name: Log - class: org.apache.skywalking.e2e.kafka.KafkaLogE2E + config-file: kafka/log/e2e.yaml steps: - uses: actions/checkout@v2 with: @@ -57,6 +56,6 @@ jobs: uses: ./.github/actions/skip - name: Run E2E Test if: env.SKIP_CI != 'true' - uses: ./.github/actions/e2e-test + uses: ./.github/actions/infra-e2e-test with: - test_class: ${{ matrix.case.class }} + config-file: ${{ matrix.case.config-file }} diff --git a/CHANGES.md b/CHANGES.md index 6a3e1e9fb5..b9d95b381f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ Release Notes. - Cluster - Event - Profile + - Kafka: Base, Meter, Log, Profile #### OAP Server @@ -48,6 +49,7 @@ Release Notes. * Fix setting wrong field when combining `Event`s. * Support search browser service. * Add `getProfileTaskLogs` to profile query protocol. +* Set `SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG`, `SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG` default `true`. #### UI diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 1e48ed0bee..8a02dd3240 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -210,8 +210,8 @@ core|default|role|Option values: `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | isSharding | True when OAP Server is in cluster. | SW_KAFKA_FETCHER_IS_SHARDING | false | | - | - | createTopicIfNotExist | If true, this creates Kafka topic (if it does not already exist). | - | true | | - | - | partitions | The number of partitions for the topic being created. | SW_KAFKA_FETCHER_PARTITIONS | 3 | -| - | - | enableNativeProtoLog | Enables fetching and handling native proto log data. | SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG | false | -| - | - | enableNativeJsonLog | Enables fetching and handling native json log data. | SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG | false | +| - | - | enableNativeProtoLog | Enables fetching and handling native proto log data. | SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG | true | +| - | - | enableNativeJsonLog | Enables fetching and handling native json log data. | SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG | true | | - | - | replicationFactor | The replication factor for each partition in the topic being created. | SW_KAFKA_FETCHER_PARTITIONS_FACTOR | 2 | | - | - | kafkaHandlerThreadPoolSize | Pool size of Kafka message handler executor. | SW_KAFKA_HANDLER_THREAD_POOL_SIZE | CPU core * 2 | | - | - | kafkaHandlerThreadPoolQueueSize | Queue size of Kafka message handler executor. | SW_KAFKA_HANDLER_THREAD_POOL_QUEUE_SIZE | 10000 | diff --git a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java index 424d77f763..40358aa81f 100644 --- a/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java +++ b/oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java @@ -67,9 +67,9 @@ public class KafkaFetcherConfig extends ModuleConfig { */ private int replicationFactor = 2; - private boolean enableNativeProtoLog = false; + private boolean enableNativeProtoLog = true; - private boolean enableNativeJsonLog = false; + private boolean enableNativeJsonLog = true; private String configPath = "meter-analyzer-config"; diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 979b17454d..cdce04059e 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -347,8 +347,8 @@ kafka-fetcher: namespace: ${SW_NAMESPACE:""} partitions: ${SW_KAFKA_FETCHER_PARTITIONS:3} replicationFactor: ${SW_KAFKA_FETCHER_PARTITIONS_FACTOR:2} - enableNativeProtoLog: ${SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG:false} - enableNativeJsonLog: ${SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG:false} + enableNativeProtoLog: ${SW_KAFKA_FETCHER_ENABLE_NATIVE_PROTO_LOG:true} + enableNativeJsonLog: ${SW_KAFKA_FETCHER_ENABLE_NATIVE_JSON_LOG:true} isSharding: ${SW_KAFKA_FETCHER_IS_SHARDING:false} consumePartitions: ${SW_KAFKA_FETCHER_CONSUME_PARTITIONS:""} kafkaHandlerThreadPoolSize: ${SW_KAFKA_HANDLER_THREAD_POOL_SIZE:-1} diff --git a/test/e2e-v2/cases/kafka/Dockerfile.provider b/test/e2e-v2/cases/kafka/Dockerfile.provider new file mode 100644 index 0000000000..45e81c3aef --- /dev/null +++ b/test/e2e-v2/cases/kafka/Dockerfile.provider @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SW_AGENT_JDK_VERSION +ARG SW_AGENT_JAVA_COMMIT + +FROM ghcr.io/apache/skywalking-java/skywalking-java:${SW_AGENT_JAVA_COMMIT}-java${SW_AGENT_JDK_VERSION} + +WORKDIR /skywalking + +RUN echo 'plugin.kafka.producer_config[compression.type]=snappy' >> agent/config/agent.config + +# activate kafka-reporter plugin +RUN mv agent/optional-reporter-plugins/kafka-reporter-plugin-*.jar agent/plugins/ || true +RUN mv agent/optional-reporter-plugins/snappy-java-*.jar agent/plugins/ || true + +VOLUME /services + +ADD java-test-service/e2e-service-provider/target/e2e-service-provider-2.0.0.jar /services/ + +CMD ["sh", "-c", "java -jar /services/e2e-service-provider-2.0.0.jar"] diff --git a/test/e2e-v2/cases/kafka/log/Dockerfile.fluentd b/test/e2e-v2/cases/kafka/log/Dockerfile.fluentd new file mode 100644 index 0000000000..01d09480ff --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/Dockerfile.fluentd @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM bitnami/fluentd:1.12.1 + +## Install output kafka plugins +RUN fluent-gem install 'fluent-plugin-kafka' diff --git a/test/e2e-v2/cases/kafka/log/Dockerfile.satellite b/test/e2e-v2/cases/kafka/log/Dockerfile.satellite new file mode 100644 index 0000000000..3a67dcf298 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/Dockerfile.satellite @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SW_AGENT_SATELLITE_COMMIT +FROM ghcr.io/apache/skywalking-satellite/skywalking-satellite:v${SW_AGENT_SATELLITE_COMMIT} as base + +FROM alpine:3.10 + +COPY --from=base /skywalking-satellite /sw-satellite +COPY --from=base /skywalking/configs /skywalking/ + +ENTRYPOINT ["/sw-satellite", "start", "--config", "/skywalking/configs/satellite_config.yaml"] diff --git a/test/e2e-v2/cases/kafka/log/docker-compose.yml b/test/e2e-v2/cases/kafka/log/docker-compose.yml new file mode 100644 index 0000000000..05ace1c906 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/docker-compose.yml @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + SW_SEARCHABLE_LOGS_TAG_KEYS: level,logger,agent + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + ports: + - 12800 + + provider: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: provider + environment: + SW_GRPC_LOG_SERVER_HOST: sw-satellite + volumes: + - /tmp/skywalking-logs/:/tmp/skywalking-logs/ + ports: + - 9090 + depends_on: + oap: + condition: service_healthy + sw-satellite: + condition: service_healthy + filebeat: + condition: service_healthy + fluentd: + condition: service_healthy + + sw-satellite: + build: + context: ../../../ + dockerfile: cases/kafka/log/Dockerfile.satellite + args: + - SW_AGENT_SATELLITE_COMMIT=${SW_AGENT_SATELLITE_COMMIT} + networks: + - e2e + expose: + - 11800 + volumes: + - ./satellite_config.yaml:/skywalking/configs/satellite_config.yaml + command: ["start"] + healthcheck: + test: [ "CMD", "sh", "-c", "nc -z 127.0.0.1 11800" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + filebeat: + image: elastic/filebeat:7.12.0 + command: -e --strict.perms=false + networks: + - e2e + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml + - /tmp/skywalking-logs/:/tmp/skywalking-logs/ + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5066"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + fluentd: + build: + context: ../../../ + dockerfile: cases/kafka/log/Dockerfile.fluentd + networks: + - e2e + volumes: + - ./fluentd.conf:/opt/bitnami/fluentd/conf/fluentd.conf + - /tmp/skywalking-logs/:/tmp/skywalking-logs/ + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/24220"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/kafka/log/e2e.yaml b/test/e2e-v2/cases/kafka/log/e2e.yaml new file mode 100644 index 0000000000..d0d324aee8 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/e2e.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + retry: + count: 20 + interval: 3s + cases: + - includes: + - ../../log/log-cases.yaml + # filebeat + - query: | + curl -X POST http://${provider_host}:${provider_9090}/file/logs/trigger > /dev/null; + curl -X POST http://${provider_host}:${provider_9090}/file/logs/trigger > /dev/null; + curl -X POST http://${provider_host}:${provider_9090}/file/logs/trigger > /dev/null; + sleep 10; + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --tags agent=filebeat + expected: expected/logs-filebeat.yml + # fluentd + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --tags agent=fluentd + expected: expected/logs-fluentd.yml diff --git a/test/e2e-v2/cases/kafka/log/expected/logs-filebeat.yml b/test/e2e-v2/cases/kafka/log/expected/logs-filebeat.yml new file mode 100644 index 0000000000..3c3fc87879 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/expected/logs-filebeat.yml @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: +{{- contains .logs }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "logback fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: logfile + value: /tmp/skywalking-logs/logback/e2e-service-provider.log + - key: hostname + value: {{ notEmpty .value }} + - key: agent + value: filebeat + - key: agentVersion + value: {{ notEmpty .value }} + - key: agentId + value: {{ notEmpty .value }} + - key: agentName + value: {{ notEmpty .value }} + {{- end }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "log4j fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: logfile + value: /tmp/skywalking-logs/log4j1/e2e-service-provider.log + - key: hostname + value: {{ notEmpty .value }} + - key: agent + value: filebeat + - key: agentVersion + value: {{ notEmpty .value }} + - key: agentId + value: {{ notEmpty .value }} + - key: agentName + value: {{ notEmpty .value }} + {{- end }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "log4j2 fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: logfile + value: /tmp/skywalking-logs/log4j2/e2e-service-provider.log + - key: hostname + value: {{ notEmpty .value }} + - key: agent + value: filebeat + - key: agentVersion + value: {{ notEmpty .value }} + - key: agentId + value: {{ notEmpty .value }} + - key: agentName + value: {{ notEmpty .value }} + {{- end }} +{{- end }} +total: {{ gt .total 0 }} diff --git a/test/e2e-v2/cases/kafka/log/expected/logs-fluentd.yml b/test/e2e-v2/cases/kafka/log/expected/logs-fluentd.yml new file mode 100644 index 0000000000..a3f39bf564 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/expected/logs-fluentd.yml @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logs: +{{- contains .logs }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "logback fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: agent + value: fluentd + {{- end }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "log4j fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: agent + value: fluentd + {{- end }} +- servicename: e2e-service-provider + serviceid: {{ b64enc "e2e-service-provider" }}.1 + serviceinstancename: provider1 + serviceinstanceid: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "provider1" }} + endpointname: null + endpointid: null + traceid: {{ .traceid }} + timestamp: {{ gt .timestamp 0 }} + contenttype: TEXT + content: {{ regexp .content "log4j2 fileLogger.+"}} + tags: + {{- contains .tags }} + - key: level + value: INFO + - key: logger + value: fileLogger + - key: thread + value: {{ notEmpty .value }} + - key: agent + value: fluentd + {{- end }} +{{- end }} +total: {{ gt .total 0 }} diff --git a/test/e2e-v2/cases/kafka/log/filebeat.yml b/test/e2e-v2/cases/kafka/log/filebeat.yml new file mode 100644 index 0000000000..3c4c9a22d1 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/filebeat.yml @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +http.enabled: true +filebeat.inputs: + - type: log + enabled: true + paths: + - /tmp/skywalking-logs/*/e2e-service-provider.log + +processors: + - dissect: + tokenizer: '[SW_CTX:%{SW_CTX}] [%{level}] %{logtime} [%{thread}] %{logger}:%{line} - %{body}' + field: "message" + target_prefix: "" + trim_values: all + overwrite_keys: true + - dissect: + tokenizer: '[%{service},%{serviceInstance},%{traceContext.traceId},%{traceContext.traceSegmentId},%{traceContext.spanId}]' + field: "SW_CTX" + target_prefix: "" + trim_values: all + - script: + lang: javascript + id: format_datetime + source: > + function process(event) { + var datetime = event.Get("logtime"); + var datetimeArr = datetime.split(" "); + var dateArr = datetimeArr[0].split("-"); + var timeArrs = datetimeArr[1].split("."); + var timeArr = timeArrs[0].split(":"); + var mills = timeArrs[1]; + + var date = new Date(); + date.setUTCFullYear(parseInt(dateArr[0])); + date.setUTCMonth(parseInt(dateArr[1]) - 1); + date.setUTCDate(parseInt(dateArr[2])); + date.setUTCHours(parseInt(timeArr[0])); + date.setUTCMinutes(parseInt(timeArr[1])); + date.setUTCSeconds(parseInt(timeArr[2])); + date.setUTCMilliseconds(parseInt(mills)); + + var timeMillis = date.getTime(); + event.Put("timestamp", timeMillis); + } + - script: + lang: javascript + id: build_logTags_array + source: > + function process(event) { + var logLevel = {"key": "level", "value": event.Get("level")}; + var logThread = {"key": "thread", "value": event.Get("thread")}; + var logLogger = {"key": "logger", "value": event.Get("logger")}; + + var logFile = {"key": "logfile", "value": event.Get("log.file.path")}; + var hostname = {"key": "hostname", "value": event.Get("agent.hostname")}; + var agent = {"key": "agent", "value": event.Get("agent.type")}; + var agentVersion = {"key": "agentVersion", "value": event.Get("agent.version")}; + var agentId = {"key": "agentId", "value": event.Get("agent.id")}; + var agentName = {"key": "agentName", "value": event.Get("agent.name")}; + + var logTags = [logLevel,logThread,logLogger,logFile,hostname,agent,agentVersion,agentId,agentName]; + event.Put("tags.data", logTags); + } + - rename: + fields: + - from: "body" + to: "body.text.text" + ignore_missing: false + fail_on_error: true + - drop_fields: + fields: ["ecs", "host", "input", "agent", "log", "message", "SW_CTX", "level", "thread", "logger", "line", "logtime"] + +output.kafka: + hosts: ["broker-a:9092", "broker-b:9092"] + topic: "skywalking-logs-json" diff --git a/test/e2e-v2/cases/kafka/log/fluentd.conf b/test/e2e-v2/cases/kafka/log/fluentd.conf new file mode 100644 index 0000000000..20f6a7018c --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/fluentd.conf @@ -0,0 +1,66 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + + @type monitor_agent + bind 0.0.0.0 + port 24220 + + + + @type tail + path /tmp/skywalking-logs/*/e2e-service-provider.log + read_from_head true + tag skywalking_log + refresh_interval 1 + + @type regexp + expression /^\[SW_CTX:\s*(?[^ ]*]*)\] \[(?[^ ]*)\] (?[^\]]*) \[(?[^ ]*)\] (?[^\]]*):(?[^\]]*) - (?[^\]]*)$/ + + + + + @type parser + key_name SW_CTX + reserve_data true + remove_key_name_field true + + @type regexp + expression /^\[(?[^\]]*),(?[^\]]*),(?[^\]]*),(?[^\]]*),(?[^\]]*)\]$/ + + + + + @type record_transformer + enable_ruby true + + traceContext ${{"traceId" => record["traceId"], "traceSegmentId" => record["traceSegmentId"], "spanId" => record["spanId"]}} + tags ${{"data" => [{"key" => "level", "value" => record["level"]}, {"key" => "thread", "value" => record["thread"]}, {"key" => "logger", "value" => record["logger"]}, {"key" => "agent", "value" => "fluentd"}]}} + body ${{"text" => {"text" => record["body"]}}} + timestamp ${DateTime.strptime(record["logtime"], '%Y-%m-%d %H:%M:%S.%L').strftime('%Q')} + + remove_keys level,thread,logger,line,traceId,traceSegmentId,spanId,logtime + + + + @type kafka2 + brokers broker-a:9092,broker-b:9092 + default_topic skywalking-logs-json + + @type json + + diff --git a/test/e2e-v2/cases/kafka/log/satellite_config.yaml b/test/e2e-v2/cases/kafka/log/satellite_config.yaml new file mode 100644 index 0000000000..01aca88f45 --- /dev/null +++ b/test/e2e-v2/cases/kafka/log/satellite_config.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +logger: + log_pattern: "%time [%level][%field] - %msg" + time_pattern: "2006-01-02 15:04:05.000" + level: "info" + +telemetry: + cluster: cluster1 + service: service1 + instance: instance1 + +sharing: + common_config: + pipe_name: sharing + clients: + - plugin_name: "kafka-client" + brokers: broker-a:9092,broker-b:9092 + version: 2.4.1 + servers: + - plugin_name: "grpc-server" + address: ":11800" +pipes: + - common_config: + pipe_name: pipe1 + gatherer: + server_name: "grpc-server" + receiver: + plugin_name: "grpc-nativelog-receiver" + queue: + plugin_name: "mmap-queue" + segment_size: 524288 + max_in_mem_segments: 6 + queue_dir: "pipe1-log-grpc-receiver-queue" + processor: + filters: + sender: + fallbacker: + plugin_name: none-fallbacker + flush_time: 1000 + max_buffer_size: 200 + min_flush_events: 5 + client_name: kafka-client + forwarders: + - plugin_name: nativelog-kafka-forwarder + topic: skywalking-logs \ No newline at end of file diff --git a/test/e2e-v2/cases/kafka/meter/docker-compose.yml b/test/e2e-v2/cases/kafka/meter/docker-compose.yml new file mode 100644 index 0000000000..f008442974 --- /dev/null +++ b/test/e2e-v2/cases/kafka/meter/docker-compose.yml @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + SW_METER_ANALYZER_ACTIVE_FILES: spring-sleuth + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + ports: + - 12800 + + provider: + build: + context: ../../../ + dockerfile: cases/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + - SW_AGENT_JAVA_COMMIT=${SW_AGENT_JAVA_COMMIT} + networks: + - e2e + ports: + - 9090 + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + SW_GET_TOPIC_TIMEOUT: 60 + SW_AGENT_NAME: e2e-service-provider + SW_AGENT_INSTANCE_NAME: provider1 + SW_LOGGING_OUTPUT: CONSOLE + SW_METER_REPORT_INTERVAL: 5 + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/kafka/meter/e2e.yaml b/test/e2e-v2/cases/kafka/meter/e2e.yaml new file mode 100644 index 0000000000..dea6f93f36 --- /dev/null +++ b/test/e2e-v2/cases/kafka/meter/e2e.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${provider_host}:${provider_9090}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + retry: + count: 20 + interval: 3s + cases: + - includes: + - ../../meter/meter-cases.yaml diff --git a/test/e2e-v2/cases/kafka/profile/docker-compose.yml b/test/e2e-v2/cases/kafka/profile/docker-compose.yml new file mode 100644 index 0000000000..df34aae9fe --- /dev/null +++ b/test/e2e-v2/cases/kafka/profile/docker-compose.yml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + influxdb: + image: influxdb:1.7.9 + expose: + - 8086 + networks: + - e2e + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8086" ] + interval: 5s + timeout: 60s + retries: 120 + + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + SW_STORAGE: influxdb + depends_on: + influxdb: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + ports: + - 12800 + + provider: + build: + context: ../../../ + dockerfile: cases/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + - SW_AGENT_JAVA_COMMIT=${SW_AGENT_JAVA_COMMIT} + networks: + - e2e + ports: + - 9090 + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + SW_GET_TOPIC_TIMEOUT: 60 + SW_AGENT_NAME: e2e-service-provider + SW_AGENT_INSTANCE_NAME: provider1 + SW_LOGGING_OUTPUT: CONSOLE + SW_METER_REPORT_INTERVAL: 5 + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/kafka/profile/e2e.yaml b/test/e2e-v2/cases/kafka/profile/e2e.yaml new file mode 100644 index 0000000000..9be10b3b01 --- /dev/null +++ b/test/e2e-v2/cases/kafka/profile/e2e.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +verify: + retry: + count: 20 + interval: 3s + cases: + - includes: + - ../../profile/profile-cases.yaml diff --git a/test/e2e-v2/cases/kafka/simple-so11y/docker-compose.yml b/test/e2e-v2/cases/kafka/simple-so11y/docker-compose.yml new file mode 100644 index 0000000000..85e043a6dc --- /dev/null +++ b/test/e2e-v2/cases/kafka/simple-so11y/docker-compose.yml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + oap: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: oap + environment: + SW_PROMETHEUS_FETCHER: "default" + SW_TELEMETRY: prometheus + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + ports: + - 12800 + + provider: + build: + context: ../../../ + dockerfile: cases/kafka/Dockerfile.provider + args: + - SW_AGENT_JDK_VERSION=${SW_AGENT_JDK_VERSION} + - SW_AGENT_JAVA_COMMIT=${SW_AGENT_JAVA_COMMIT} + networks: + - e2e + expose: + - 9090 + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_KAFKA_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + SW_GET_TOPIC_TIMEOUT: 60 + SW_AGENT_NAME: e2e-service-provider + SW_AGENT_INSTANCE_NAME: provider1 + SW_LOGGING_OUTPUT: CONSOLE + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 9090"] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + oap: + condition: service_healthy + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy + + consumer: + extends: + file: ../../../script/docker-compose/base-compose.yml + service: consumer + environment: + PROVIDER_URL: http://provider:9090 + ports: + - 9092 + depends_on: + provider: + condition: service_healthy + +networks: + e2e: diff --git a/test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml b/test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml new file mode 100644 index 0000000000..229f30a15a --- /dev/null +++ b/test/e2e-v2/cases/kafka/simple-so11y/e2e.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is used to show how to write configuration files and can be used to test. + +setup: + env: compose + file: docker-compose.yml + timeout: 1200 + init-system-environment: ../../../script/env + steps: + - name: install yq + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh swctl + - name: install etcdctl + command: bash test/e2e-v2/script/prepare/setup-e2e-shell/install.sh etcdctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${consumer_host}:${consumer_9092}/users + method: POST + body: '{"id":"123","name":"skywalking"}' + headers: + "Content-Type": "application/json" + +verify: + retry: + count: 20 + interval: 3s + cases: + - includes: + - ../../simple/simple-cases.yaml + - ../../so11y/so11y-cases.yaml diff --git a/test/e2e-v2/cases/log/es/e2e.yaml b/test/e2e-v2/cases/log/es/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/es/e2e.yaml +++ b/test/e2e-v2/cases/log/es/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/fluent-bit/e2e.yaml b/test/e2e-v2/cases/log/fluent-bit/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/fluent-bit/e2e.yaml +++ b/test/e2e-v2/cases/log/fluent-bit/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/h2/e2e.yaml b/test/e2e-v2/cases/log/h2/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/h2/e2e.yaml +++ b/test/e2e-v2/cases/log/h2/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/influxdb/e2e.yaml b/test/e2e-v2/cases/log/influxdb/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/influxdb/e2e.yaml +++ b/test/e2e-v2/cases/log/influxdb/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/log-cases.yaml b/test/e2e-v2/cases/log/log-cases.yaml new file mode 100644 index 0000000000..05d011663f --- /dev/null +++ b/test/e2e-v2/cases/log/log-cases.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: expected/service-endpoint.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: expected/service-instance.yml + # logs + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: expected/logs.yml diff --git a/test/e2e-v2/cases/log/mysql/e2e.yaml b/test/e2e-v2/cases/log/mysql/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/mysql/e2e.yaml +++ b/test/e2e-v2/cases/log/mysql/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/log/postgres/e2e.yaml b/test/e2e-v2/cases/log/postgres/e2e.yaml index 6f98247250..5be83f6f42 100644 --- a/test/e2e-v2/cases/log/postgres/e2e.yaml +++ b/test/e2e-v2/cases/log/postgres/e2e.yaml @@ -46,19 +46,5 @@ verify: # the interval between two retries, in millisecond. interval: 10s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # logs - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql logs list --service-name=e2e-service-provider --trace-id=$( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/logs.yml \ No newline at end of file + - includes: + - ../log-cases.yaml diff --git a/test/e2e-v2/cases/meter/e2e.yaml b/test/e2e-v2/cases/meter/e2e.yaml index 5495ce1df5..26b47c3c6c 100644 --- a/test/e2e-v2/cases/meter/e2e.yaml +++ b/test/e2e-v2/cases/meter/e2e.yaml @@ -43,16 +43,5 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: expected/service-instance.yml - # service instance metrics linear - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_jvm_memory_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_jvm_threads_live --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_process_files_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - - expected: expected/metrics-has-value.yml + - includes: + - meter-cases.yaml diff --git a/test/e2e-v2/cases/meter/meter-cases.yaml b/test/e2e-v2/cases/meter/meter-cases.yaml new file mode 100644 index 0000000000..d1eeb7c0ad --- /dev/null +++ b/test/e2e-v2/cases/meter/meter-cases.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: expected/service-instance.yml + # service instance metrics linear + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_jvm_memory_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_jvm_threads_live --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_process_files_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml diff --git a/test/e2e-v2/cases/simple/auth/e2e.yaml b/test/e2e-v2/cases/simple/auth/e2e.yaml index b50769888d..2eb370e4a5 100644 --- a/test/e2e-v2/cases/simple/auth/e2e.yaml +++ b/test/e2e-v2/cases/simple/auth/e2e.yaml @@ -43,37 +43,5 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # service endpoint metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # dependency service - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider - expected: ../expected/dependency-services.yml - # dependency instance - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider - expected: ../expected/dependency-instance.yml - # dependency endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users - expected: ../expected/dependency-endpoint.yml - # trace segment list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls - expected: ../expected/traces-list.yml - # trace detail - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/trace-users-detail.yml \ No newline at end of file + - includes: + - ../simple-cases.yaml diff --git a/test/e2e-v2/cases/simple/expected/dependency-endpoint-consumer.yml b/test/e2e-v2/cases/simple/expected/dependency-endpoint-consumer.yml new file mode 100644 index 0000000000..8f5a8b26b4 --- /dev/null +++ b/test/e2e-v2/cases/simple/expected/dependency-endpoint-consumer.yml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "User" }}.0_{{ b64enc "User" }} + name: User + serviceid: {{ b64enc "User" }}.0 + servicename: User + type: "" + isreal: false +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-consumer" }}.1 + servicename: e2e-service-consumer + type: "" + isreal: true +- id: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users + serviceid: {{ b64enc "e2e-service-provider" }}.1 + servicename: e2e-service-provider + type: "" + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "User" }}.0_{{ b64enc "User" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + targetcomponents: [] + id: {{ b64enc "User" }}.0-{{ b64enc "User" }}-{{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }} + detectpoints: + - SERVER +- source: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider" }}.1_{{ b64enc "POST:/users" }} + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer" }}.1-{{ b64enc "POST:/users" }}-{{ b64enc "e2e-service-provider" }}.1-{{ b64enc "POST:/users" }} + detectpoints: + - SERVER +{{- end }} diff --git a/test/e2e-v2/cases/simple/expected/dependency-endpoint.yml b/test/e2e-v2/cases/simple/expected/dependency-endpoint-provider.yml similarity index 100% rename from test/e2e-v2/cases/simple/expected/dependency-endpoint.yml rename to test/e2e-v2/cases/simple/expected/dependency-endpoint-provider.yml diff --git a/test/e2e-v2/cases/simple/expected/dependency-services-consumer.yml b/test/e2e-v2/cases/simple/expected/dependency-services-consumer.yml new file mode 100644 index 0000000000..11ba084ec2 --- /dev/null +++ b/test/e2e-v2/cases/simple/expected/dependency-services-consumer.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: +{{- contains .nodes }} +- id: {{ b64enc "User"}}.0 + name: User + type: USER + isreal: false +- id: {{ b64enc "e2e-service-consumer"}}.1 + name: e2e-service-consumer + type: Tomcat + isreal: true +- id: {{ b64enc "e2e-service-provider"}}.1 + name: e2e-service-provider + type: Tomcat + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer"}}.1 + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider"}}.1 + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer"}}.1-{{ b64enc "e2e-service-provider"}}.1 + detectpoints: + - CLIENT + - SERVER +- source: {{ b64enc "User" }}.0 + sourcecomponents: [] + target: {{ b64enc "e2e-service-consumer"}}.1 + targetcomponents: [] + id: {{ b64enc "User" }}.0-{{ b64enc "e2e-service-consumer"}}.1 + detectpoints: + - SERVER +{{- end }} diff --git a/test/e2e-v2/cases/simple/expected/dependency-services.yml b/test/e2e-v2/cases/simple/expected/dependency-services-provider.yml similarity index 100% rename from test/e2e-v2/cases/simple/expected/dependency-services.yml rename to test/e2e-v2/cases/simple/expected/dependency-services-provider.yml diff --git a/test/e2e-v2/cases/simple/expected/metrics-has-value-percentile.yml b/test/e2e-v2/cases/simple/expected/metrics-has-value-percentile.yml new file mode 100644 index 0000000000..ef17490250 --- /dev/null +++ b/test/e2e-v2/cases/simple/expected/metrics-has-value-percentile.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- key: 0 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 1 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 2 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 3 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +- key: 4 + value: + {{- contains .value }} + - key: {{ notEmpty .key }} + value: {{ ge .value 1 }} + {{- end }} +{{- end }} diff --git a/test/e2e-v2/cases/simple/expected/service-endpoint-consumer.yml b/test/e2e-v2/cases/simple/expected/service-endpoint-consumer.yml new file mode 100644 index 0000000000..cedf79a14e --- /dev/null +++ b/test/e2e-v2/cases/simple/expected/service-endpoint-consumer.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "POST:/users" }} + name: POST:/users +{{- end}} diff --git a/test/e2e-v2/cases/simple/expected/service-endpoint.yml b/test/e2e-v2/cases/simple/expected/service-endpoint-provider.yml similarity index 100% rename from test/e2e-v2/cases/simple/expected/service-endpoint.yml rename to test/e2e-v2/cases/simple/expected/service-endpoint-provider.yml diff --git a/test/e2e-v2/cases/simple/expected/service-instance-consumer.yml b/test/e2e-v2/cases/simple/expected/service-instance-consumer.yml new file mode 100644 index 0000000000..7d1d199a7a --- /dev/null +++ b/test/e2e-v2/cases/simple/expected/service-instance-consumer.yml @@ -0,0 +1,40 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +- id: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} + name: consumer1 + attributes: + {{- contains .attributes }} + - name: OS Name + value: Linux + - name: hostname + value: {{ notEmpty .value }} + - name: Process No. + value: "1" + - name: Start Time + value: {{ notEmpty .value }} + - name: JVM Arguments + value: '{{ notEmpty .value }}' + - name: Jar Dependencies + value: '{{ notEmpty .value }}' + - name: ipv4s + value: {{ notEmpty .value }} + {{- end}} + language: JAVA + instanceuuid: {{ b64enc "e2e-service-consumer" }}.1_{{ b64enc "consumer1" }} +{{- end}} diff --git a/test/e2e-v2/cases/simple/expected/service-instance.yml b/test/e2e-v2/cases/simple/expected/service-instance-provider.yml similarity index 100% rename from test/e2e-v2/cases/simple/expected/service-instance.yml rename to test/e2e-v2/cases/simple/expected/service-instance-provider.yml diff --git a/test/e2e-v2/cases/simple/jdk/e2e.yaml b/test/e2e-v2/cases/simple/jdk/e2e.yaml index b50769888d..2eb370e4a5 100644 --- a/test/e2e-v2/cases/simple/jdk/e2e.yaml +++ b/test/e2e-v2/cases/simple/jdk/e2e.yaml @@ -43,37 +43,5 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # service endpoint metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # dependency service - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider - expected: ../expected/dependency-services.yml - # dependency instance - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider - expected: ../expected/dependency-instance.yml - # dependency endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users - expected: ../expected/dependency-endpoint.yml - # trace segment list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls - expected: ../expected/traces-list.yml - # trace detail - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/trace-users-detail.yml \ No newline at end of file + - includes: + - ../simple-cases.yaml diff --git a/test/e2e-v2/cases/simple/mtls/e2e.yaml b/test/e2e-v2/cases/simple/mtls/e2e.yaml index b50769888d..2eb370e4a5 100644 --- a/test/e2e-v2/cases/simple/mtls/e2e.yaml +++ b/test/e2e-v2/cases/simple/mtls/e2e.yaml @@ -43,37 +43,5 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # service endpoint metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # dependency service - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider - expected: ../expected/dependency-services.yml - # dependency instance - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider - expected: ../expected/dependency-instance.yml - # dependency endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users - expected: ../expected/dependency-endpoint.yml - # trace segment list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls - expected: ../expected/traces-list.yml - # trace detail - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/trace-users-detail.yml \ No newline at end of file + - includes: + - ../simple-cases.yaml diff --git a/test/e2e-v2/cases/simple/simple-cases.yaml b/test/e2e-v2/cases/simple/simple-cases.yaml new file mode 100644 index 0000000000..f6549c1459 --- /dev/null +++ b/test/e2e-v2/cases/simple/simple-cases.yaml @@ -0,0 +1,131 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider + expected: expected/service-instance-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-consumer + expected: expected/service-instance-consumer.yml + # service endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider + expected: expected/service-endpoint-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-consumer + expected: expected/service-endpoint-consumer.yml + # dependency service + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider + expected: expected/dependency-services-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-consumer + expected: expected/dependency-services-consumer.yml + # dependency instance + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider + expected: expected/dependency-instance.yml + # dependency endpoint + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users + expected: expected/dependency-endpoint-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-consumer --endpoint-name=POST:/users + expected: expected/dependency-endpoint-consumer.yml + # trace segment list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls + expected: expected/traces-list.yml + # trace detail + - query: | + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ + swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ + | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - + ) + expected: expected/trace-users-detail.yml + # service metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_cpm --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_resp_time --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_apdex --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service instance metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_resp_time --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_cpm --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_instance_sla --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service instance JVM metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=consumer1 --service-name=e2e-service-consumer |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_heap_max --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_memory_noheap --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_live_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_daemon_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_peak_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_thread_runnable_state_thread_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=instance_jvm_class_total_loaded_class_count --instance-name=provider1 --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service endpoint metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_avg --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_avg --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_sla --endpoint-name=POST:/users --service-name=e2e-service-consumer | yq e 'to_entries' - + expected: expected/metrics-has-value.yml + # service endpoint metrics percentile + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-percentile.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics multiple-linear --name=endpoint_percentile --endpoint-name=POST:/users --service-name=e2e-service-consumer |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' - + expected: expected/metrics-has-value-percentile.yml diff --git a/test/e2e-v2/cases/simple/ssl/e2e.yaml b/test/e2e-v2/cases/simple/ssl/e2e.yaml index b50769888d..2eb370e4a5 100644 --- a/test/e2e-v2/cases/simple/ssl/e2e.yaml +++ b/test/e2e-v2/cases/simple/ssl/e2e.yaml @@ -43,37 +43,5 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: ../expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=e2e-service-provider - expected: ../expected/service-instance.yml - # service endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --keyword=users --service-name=e2e-service-provider - expected: ../expected/service-endpoint.yml - # service metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=service_sla --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # service endpoint metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=endpoint_cpm --endpoint-name=POST:/users --service-name=e2e-service-provider | yq e 'to_entries' - - expected: ../expected/metrics-has-value.yml - # dependency service - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider - expected: ../expected/dependency-services.yml - # dependency instance - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency instance --service-name=e2e-service-consumer --dest-service-name=e2e-service-provider - expected: ../expected/dependency-instance.yml - # dependency endpoint - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency endpoint --service-name=e2e-service-provider --endpoint-name=POST:/users - expected: ../expected/dependency-endpoint.yml - # trace segment list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls - expected: ../expected/traces-list.yml - # trace detail - - query: | - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace $( \ - swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql trace ls \ - | yq e '.traces | select(.[].endpointnames[0]=="POST:/users") | .[0].traceids[0]' - - ) - expected: ../expected/trace-users-detail.yml \ No newline at end of file + - includes: + - ../simple-cases.yaml diff --git a/test/e2e-v2/cases/so11y/e2e.yaml b/test/e2e-v2/cases/so11y/e2e.yaml index 6f19a556b6..c2f00fb5bf 100644 --- a/test/e2e-v2/cases/so11y/e2e.yaml +++ b/test/e2e-v2/cases/so11y/e2e.yaml @@ -43,31 +43,6 @@ verify: count: 20 interval: 3s cases: - # service list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls - expected: expected/service.yml - # service instance list - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=oap::oap-server - expected: expected/service-instance.yml - # service instance metrics - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_cpu_percentage --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name meter_oap_instance_jvm_memory_bytes_used --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_trace_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_metrics_first_aggregation --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_persistence_prepare_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_persistence_execute_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_thread_live_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_thread_runnable_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_class_loaded_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml - - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_class_total_loaded_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - - expected: expected/metrics-has-value.yml + - includes: + - so11y-cases.yaml diff --git a/test/e2e-v2/cases/so11y/so11y-cases.yaml b/test/e2e-v2/cases/so11y/so11y-cases.yaml new file mode 100644 index 0000000000..3e2276a78e --- /dev/null +++ b/test/e2e-v2/cases/so11y/so11y-cases.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + cases: + # service list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql service ls + expected: expected/service.yml + # service instance list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name=oap::oap-server + expected: expected/service-instance.yml + # service instance metrics + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_cpu_percentage --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name meter_oap_instance_jvm_memory_bytes_used --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_trace_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_metrics_first_aggregation --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_persistence_prepare_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_instance_persistence_execute_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_thread_live_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_thread_runnable_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_class_loaded_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name=meter_oap_jvm_class_total_loaded_count --instance-name=http://localhost:1234 --service-name=oap::oap-server |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml index 3e44264f0c..27171d571b 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml +++ b/test/e2e-v2/java-test-service/e2e-service-provider/pom.xml @@ -70,11 +70,15 @@ log4j ${log4j.version} + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + org.apache.logging.log4j log4j-core ${log4j2.version} - provided ch.qos.logback diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/FileLogController.java b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/FileLogController.java index 34e293e577..b02295ad45 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/FileLogController.java +++ b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/FileLogController.java @@ -33,9 +33,9 @@ public class FileLogController { @RequestMapping(value = "/file/logs/trigger") public String trigger() { - LOG4J_LOGGER.info("log4j fileLogger ==> mills: " + System.currentTimeMillis()); - LOG4J2_LOGGER.info("log4j2 fileLogger ==> mills: " + System.currentTimeMillis()); - LOGBACK_LOGGER.info("logback fileLogger ==> mills: {}", System.currentTimeMillis()); + LOG4J_LOGGER.info("log4j fileLogger ==> mills-> " + System.currentTimeMillis()); + LOG4J2_LOGGER.info("log4j2 fileLogger ==> mills->" + System.currentTimeMillis()); + LOGBACK_LOGGER.info("logback fileLogger ==> mills-> {}", System.currentTimeMillis()); return TraceContext.traceId(); } } diff --git a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java index a84d99b04a..16970f106d 100644 --- a/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java +++ b/test/e2e-v2/java-test-service/e2e-service-provider/src/main/java/org/apache/skywalking/e2e/controller/LogController.java @@ -34,9 +34,9 @@ public class LogController { @RequestMapping(value = "/logs/trigger") public String trigger() { - LOG4J_LOGGER.info("log4j message==> now: " + System.currentTimeMillis()); - LOG4J2_LOGGER.info("log4j2 message==> now: " + System.currentTimeMillis()); - LOGBACK_LOGGER.info("logback message==> now: {}", System.currentTimeMillis()); + LOG4J_LOGGER.info("log4j message==> now-> " + System.currentTimeMillis()); + LOG4J2_LOGGER.info("log4j2 message==> now-> " + System.currentTimeMillis()); + LOGBACK_LOGGER.info("logback message==> now-> {}", System.currentTimeMillis()); return TraceContext.traceId(); } } diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index ceef242e95..65123c1d89 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -SW_AGENT_JAVA_COMMIT=3997f0256056788bd054ee37e4603c11c0fd6756 +SW_AGENT_JAVA_COMMIT=fbdfc42b4d825ba33205d646ddaaaad20c005cb8 SW_AGENT_SATELLITE_COMMIT=1f3c08a5af19f8522f2a40d9339c45fa816bfe07 SW_AGENT_NGINX_LUA_COMMIT=c3cee4841798a147d83b96a10914d4ac0e11d0aa SW_AGENT_NODEJS_COMMIT=e755659c7f308d3b5589619778c8360308cb14f8