From 39815388a310319424071bd1e618bbc01f8a53cb Mon Sep 17 00:00:00 2001 From: Humbertzhang <504490160@qq.com> Date: Sat, 10 Oct 2020 16:41:18 +0800 Subject: [PATCH] Add E2E test for python Kafka report protocol (#5640) --- .../e2e-test/docker/python/Dockerfile.python | 4 +- test/e2e/e2e-test/docker/python/consumer.py | 6 ++ .../e2e-test/docker/python/docker-compose.yml | 92 +++++++++++++++++++ .../e2e-test/docker/python/provider-kafka.py | 47 ++++++++++ .../org/apache/skywalking/e2e/PythonE2E.java | 9 +- .../python/provider-kafka-endpoints.yml | 18 ++++ .../python/provider-kafka-instances.yml | 18 ++++ .../resources/expected/python/services.yml | 2 + .../test/resources/expected/python/topo.yml | 10 ++ .../test/resources/expected/python/traces.yml | 8 ++ 10 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 test/e2e/e2e-test/docker/python/provider-kafka.py create mode 100644 test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml create mode 100644 test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml diff --git a/test/e2e/e2e-test/docker/python/Dockerfile.python b/test/e2e/e2e-test/docker/python/Dockerfile.python index cdd675c11..4d986384a 100644 --- a/test/e2e/e2e-test/docker/python/Dockerfile.python +++ b/test/e2e/e2e-test/docker/python/Dockerfile.python @@ -15,7 +15,7 @@ FROM python:3.7 -ENV COMMIT_HASH=2056144925a574d04c4a90914a82b8efce2375be +ENV COMMIT_HASH=50388c55428d742d73d9733278f04173585de80d WORKDIR /app @@ -24,6 +24,8 @@ RUN git clone https://github.com/apache/skywalking-python.git $(pwd) RUN git reset --hard ${COMMIT_HASH} && git submodule update --init RUN make setup install +RUN python3 -m pip install kafka-python ADD ./consumer.py /consumer.py ADD ./provider.py /provider.py +ADD ./provider-kafka.py /provider-kafka.py diff --git a/test/e2e/e2e-test/docker/python/consumer.py b/test/e2e/e2e-test/docker/python/consumer.py index 67a7be479..4a611c023 100644 --- a/test/e2e/e2e-test/docker/python/consumer.py +++ b/test/e2e/e2e-test/docker/python/consumer.py @@ -43,6 +43,12 @@ if __name__ == '__main__': with request.urlopen(req, data): self.wfile.write(data) + req2 = request.Request("http://provider-kafka:9089/users") + req2.add_header('Content-Type', 'application/json; charset=utf-8') + req2.add_header('Content-Length', str(len(data))) + with request.urlopen(req2, data): + self.wfile.write(data) + PORT = 9090 Handler = SimpleHTTPRequestHandler diff --git a/test/e2e/e2e-test/docker/python/docker-compose.yml b/test/e2e/e2e-test/docker/python/docker-compose.yml index 68eaf0391..6fff91317 100644 --- a/test/e2e/e2e-test/docker/python/docker-compose.yml +++ b/test/e2e/e2e-test/docker/python/docker-compose.yml @@ -16,10 +16,77 @@ version: '2.1' services: + zookeeper: + image: zookeeper:3.4 + hostname: zookeeper + expose: + - 2181 + networks: + - e2e + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + healthcheck: + test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-a: + image: bitnami/kafka:2.4.1 + hostname: broker-a + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=10 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + + broker-b: + image: bitnami/kafka:2.4.1 + hostname: broker-b + expose: + - 9092 + networks: + - e2e + environment: + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_BROKER_ID=24 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"] + interval: 5s + timeout: 60s + retries: 120 + oap: extends: file: ../base-compose.yml service: oap + environment: + SW_PROMETHEUS_FETCHER_ACTIVE: "true" + SW_TELEMETRY: prometheus + SW_KAFKA_FETCHER: default + SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092 + SW_KAFKA_FETCHER_PARTITIONS: 2 + SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1 + depends_on: + broker-a: + condition: service_healthy + broker-b: + condition: service_healthy ui: extends: @@ -62,6 +129,29 @@ services: provider: condition: service_healthy + provider-kafka: + build: + context: . + dockerfile: Dockerfile.python + networks: + - e2e + expose: + - 9089 + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_AGENT_INSTANCE: provider-kafka-instance + SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092 + depends_on: + oap: + condition: service_healthy + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9089"] + interval: 5s + timeout: 60s + retries: 120 + entrypoint: ['python3', '/provider-kafka.py'] + + consumer: build: context: . @@ -79,6 +169,8 @@ services: condition: service_healthy medium: condition: service_healthy + provider-kafka: + condition: service_healthy healthcheck: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"] interval: 5s diff --git a/test/e2e/e2e-test/docker/python/provider-kafka.py b/test/e2e/e2e-test/docker/python/provider-kafka.py new file mode 100644 index 000000000..75084ddb9 --- /dev/null +++ b/test/e2e/e2e-test/docker/python/provider-kafka.py @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time + +from urllib import request + +from skywalking import agent, config + +if __name__ == '__main__': + config.service_name = 'provider-kafka' + config.logging_level = 'DEBUG' + config.protocol = "kafka" + agent.start() + + import socketserver + from http.server import BaseHTTPRequestHandler + + class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + + def do_POST(self): + time.sleep(0.15) + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"name": "whatever"}'.encode('ascii')) + + PORT = 9089 + Handler = SimpleHTTPRequestHandler + + with socketserver.TCPServer(("", PORT), Handler) as httpd: + print("serving at port", PORT) + httpd.serve_forever() diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java index 254fc4e16..6a0d08d73 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java @@ -18,7 +18,6 @@ package org.apache.skywalking.e2e; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.e2e.annotation.ContainerHostAndPort; import org.apache.skywalking.e2e.annotation.DockerCompose; @@ -55,6 +54,9 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.DockerComposeContainer; +import java.util.List; +import java.util.stream.Collectors; + import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS; import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS; @@ -95,8 +97,8 @@ public class PythonE2E extends SkyWalkingTestAdapter { @RetryableTest void services() throws Exception { - final List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - + List services = graphql.services(new ServicesQuery().start(startTime).end(now())); + services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/python/services.yml").as(ServicesMatcher.class).verify(services); @@ -105,6 +107,7 @@ public class PythonE2E extends SkyWalkingTestAdapter { if ("Your_ApplicationName".equals(service.getLabel())) { continue; } + LOGGER.info("verifying service instances: {}", service); verifyServiceMetrics(service); diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml new file mode 100644 index 000000000..9a5a4774d --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +endpoints: + - key: not null + label: /users diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml new file mode 100644 index 000000000..7de572a77 --- /dev/null +++ b/test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +instances: + - key: not null + label: not null diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml b/test/e2e/e2e-test/src/test/resources/expected/python/services.yml index 5aea5776e..d2404d10d 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/python/services.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/python/services.yml @@ -16,6 +16,8 @@ services: - key: not null label: provider + - key: not null + label: provider-kafka - key: not null label: consumer - key: not null diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml b/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml index efdea724e..d9f12ef5b 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/python/topo.yml @@ -26,6 +26,10 @@ nodes: name: provider type: Python isReal: true + - id: not null + name: provider-kafka + type: Python + isReal: true - id: not null name: Your_ApplicationName type: Tomcat @@ -42,6 +46,12 @@ calls: - CLIENT - SERVER target: ${Your_ApplicationName[0]} + - id: not null + source: ${consumer[0]} + detectPoints: + - CLIENT + - SERVER + target: ${provider-kafka[0]} - id: not null source: ${Your_ApplicationName[0]} detectPoints: diff --git a/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml b/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml index 6fc49bad2..a26eb66f5 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/python/traces.yml @@ -22,6 +22,14 @@ traces: isError: false traceIds: - not null + - key: not null + endpointNames: + - /users + duration: ge 0 + start: gt 0 + isError: false + traceIds: + - not null - key: not null endpointNames: - /test