kazusa
/
skywalking
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add E2E test for python Kafka report protocol (#5640)

This commit is contained in:
Humbertzhang 2020-10-10 16:41:18 +08:00 committed by GitHub
parent 77a16c50ce
commit 39815388a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 210 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/e2e/e2e-test/docker/python/Dockerfile.python
View File

@ -15,7 +15,7 @@
FROM python:3.7
ENV COMMIT_HASH=2056144925a574d04c4a90914a82b8efce2375be
ENV COMMIT_HASH=50388c55428d742d73d9733278f04173585de80d
WORKDIR /app
@ -24,6 +24,8 @@ RUN git clone https://github.com/apache/skywalking-python.git $(pwd)
RUN git reset --hard ${COMMIT_HASH} && git submodule update --init
RUN make setup install
RUN python3 -m pip install kafka-python
ADD ./consumer.py /consumer.py
ADD ./provider.py /provider.py
ADD ./provider-kafka.py /provider-kafka.py

6
test/e2e/e2e-test/docker/python/consumer.py
View File

@ -43,6 +43,12 @@ if __name__ == '__main__':
with request.urlopen(req, data):
self.wfile.write(data)
req2 = request.Request("http://provider-kafka:9089/users")
req2.add_header('Content-Type', 'application/json; charset=utf-8')
req2.add_header('Content-Length', str(len(data)))
with request.urlopen(req2, data):
self.wfile.write(data)
PORT = 9090
Handler = SimpleHTTPRequestHandler

92
test/e2e/e2e-test/docker/python/docker-compose.yml
View File

@ -16,10 +16,77 @@
version: '2.1'
services:
zookeeper:
image: zookeeper:3.4
hostname: zookeeper
expose:
- 2181
networks:
- e2e
environment:
- ALLOW_ANONYMOUS_LOGIN=yes
healthcheck:
test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"]
interval: 5s
timeout: 60s
retries: 120
broker-a:
image: bitnami/kafka:2.4.1
hostname: broker-a
expose:
- 9092
networks:
- e2e
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_BROKER_ID=10
- ALLOW_PLAINTEXT_LISTENER=yes
depends_on:
zookeeper:
condition: service_healthy
healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
interval: 5s
timeout: 60s
retries: 120
broker-b:
image: bitnami/kafka:2.4.1
hostname: broker-b
expose:
- 9092
networks:
- e2e
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_BROKER_ID=24
- ALLOW_PLAINTEXT_LISTENER=yes
depends_on:
zookeeper:
condition: service_healthy
healthcheck:
test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
interval: 5s
timeout: 60s
retries: 120
oap:
extends:
file: ../base-compose.yml
service: oap
environment:
SW_PROMETHEUS_FETCHER_ACTIVE: "true"
SW_TELEMETRY: prometheus
SW_KAFKA_FETCHER: default
SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092
SW_KAFKA_FETCHER_PARTITIONS: 2
SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1
depends_on:
broker-a:
condition: service_healthy
broker-b:
condition: service_healthy
ui:
extends:
@ -62,6 +129,29 @@ services:
provider:
condition: service_healthy
provider-kafka:
build:
context: .
dockerfile: Dockerfile.python
networks:
- e2e
expose:
- 9089
environment:
SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800
SW_AGENT_INSTANCE: provider-kafka-instance
SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092
depends_on:
oap:
condition: service_healthy
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9089"]
interval: 5s
timeout: 60s
retries: 120
entrypoint: ['python3', '/provider-kafka.py']
consumer:
build:
context: .
@ -79,6 +169,8 @@ services:
condition: service_healthy
medium:
condition: service_healthy
provider-kafka:
condition: service_healthy
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"]
interval: 5s

47
test/e2e/e2e-test/docker/python/provider-kafka.py Normal file
View File

@ -0,0 +1,47 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import time
from urllib import request
from skywalking import agent, config
if __name__ == '__main__':
config.service_name = 'provider-kafka'
config.logging_level = 'DEBUG'
config.protocol = "kafka"
agent.start()
import socketserver
from http.server import BaseHTTPRequestHandler
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
def do_POST(self):
time.sleep(0.15)
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.end_headers()
self.wfile.write('{"name": "whatever"}'.encode('ascii'))
PORT = 9089
Handler = SimpleHTTPRequestHandler
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print("serving at port", PORT)
httpd.serve_forever()

9
test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java
View File

@ -18,7 +18,6 @@
package org.apache.skywalking.e2e;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
import org.apache.skywalking.e2e.annotation.DockerCompose;
@ -55,6 +54,9 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.DockerComposeContainer;
import java.util.List;
import java.util.stream.Collectors;
import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
@ -95,8 +97,8 @@ public class PythonE2E extends SkyWalkingTestAdapter {
@RetryableTest
void services() throws Exception {
final List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList());
LOGGER.info("services: {}", services);
load("expected/python/services.yml").as(ServicesMatcher.class).verify(services);
@ -105,6 +107,7 @@ public class PythonE2E extends SkyWalkingTestAdapter {
if ("Your_ApplicationName".equals(service.getLabel())) {
continue;
}
LOGGER.info("verifying service instances: {}", service);
verifyServiceMetrics(service);

18
test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
endpoints:
- key: not null
label: /users

18
test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml Normal file
View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
instances:
- key: not null
label: not null

2
test/e2e/e2e-test/src/test/resources/expected/python/services.yml
View File

@ -16,6 +16,8 @@
services:
- key: not null
label: provider
- key: not null
label: provider-kafka
- key: not null
label: consumer
- key: not null

10
test/e2e/e2e-test/src/test/resources/expected/python/topo.yml
View File

@ -26,6 +26,10 @@ nodes:
name: provider
type: Python
isReal: true
- id: not null
name: provider-kafka
type: Python
isReal: true
- id: not null
name: Your_ApplicationName
type: Tomcat
@ -42,6 +46,12 @@ calls:
- CLIENT
- SERVER
target: ${Your_ApplicationName[0]}
- id: not null
source: ${consumer[0]}
detectPoints:
- CLIENT
- SERVER
target: ${provider-kafka[0]}
- id: not null
source: ${Your_ApplicationName[0]}
detectPoints:

8
test/e2e/e2e-test/src/test/resources/expected/python/traces.yml
View File

@ -22,6 +22,14 @@ traces:
isError: false
traceIds:
- not null
- key: not null
endpointNames:
- /users
duration: ge 0
start: gt 0
isError: false
traceIds:
- not null
- key: not null
endpointNames:
- /test