kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

iimmigrate kafak scenario (#3653)

This commit is contained in:
zhangwei 2019-10-18 21:08:08 +08:00 committed by 吴晟 Wu Sheng
parent ef91bcf45f
commit 19288d85e9
12 changed files with 683 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Jenkinsfile-Agent-Test-3
View File

@ -58,9 +58,9 @@ pipeline {
parallel {
stage('Group1') {
stages {
stage('reserve stages') {
stage('kafka 0.11.0.0-2.3.0 (16)') {
steps {
echo "reserve."
sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} kafka-scenario'
}
}
}

4
test/plugin/runner-helper/src/main/resources/docker-compose.template
View File

@ -57,9 +57,9 @@ services:
- ${volume}
</#list>
</#if>
<#if service.environments??>
<#if service.environment??>
environment:
<#list service.environments as environment>
<#list service.environment as environment>
- ${environment}
</#list>
</#if>

21
test/plugin/scenarios/kafka-scenario/bin/startup.sh Normal file
View File

@ -0,0 +1,21 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
home="$(cd "$(dirname $0)"; pwd)"
java -Dbootstrap.servers=${BOOTSTRAP_SERVERS} -jar ${agent_opts} "-Dskywalking.agent.service_name=kafka-scenario" ${home}/../libs/kafka-scenario.jar &

130
test/plugin/scenarios/kafka-scenario/config/expectedData.yaml Normal file
View File

@ -0,0 +1,130 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
registryItems:
applications:
- {kafka-scenario: 2}
instances:
- {kafka-scenario: 1}
operationNames:
- kafka-scenario: [Kafka/test/Producer, Kafka/test/Consumer/testGroup,
/case/healthCheck, /case/kafka-case]
heartbeat: []
segmentItems:
- applicationCode: kafka-scenario
segmentSize: gt 4
segments:
- segmentId: not null
spans:
- operationName: /case/healthCheck
operationId: 0
parentSpanId: -1
spanId: 0
spanLayer: Http
startTime: nq 0
endTime: nq 0
componentId: 14
componentName: ''
isError: false
spanType: Entry
peer: ''
peerId: 0
tags:
- {key: url, value: 'http://localhost:8080/kafka-scenario/case/healthCheck'}
- {key: http.method, value: HEAD}
- segmentId: not null
spans:
- operationName: Kafka/Producer/Callback
operationId: 0
parentSpanId: -1
spanId: 0
spanLayer: Unknown
startTime: nq 0
endTime: nq 0
componentId: 40
componentName: ''
isError: false
spanType: Local
peer: ''
peerId: 0
tags:
- {key: mq.topic, value: test}
refs:
- {parentEndpointId: 0, parentEndpoint: /case/kafka-case, networkAddressId: 0,
entryEndpointId: 0, refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
parentServiceInstanceId: 1, networkAddress: '', entryEndpoint: /case/kafka-case,
entryServiceInstanceId: 1}
- segmentId: not null
spans:
- operationName: Kafka/test/Producer
operationId: 0
parentSpanId: 0
spanId: 1
spanLayer: MQ
startTime: nq 0
endTime: nq 0
componentId: 40
componentName: ''
isError: false
spanType: Exit
peer: kafka-server:9092
peerId: 0
tags:
- {key: mq.broker, value: 'kafka-server:9092'}
- {key: mq.topic, value: test}
- operationName: /case/kafka-case
operationId: 0
parentSpanId: -1
spanId: 0
spanLayer: Http
startTime: nq 0
endTime: nq 0
componentId: 14
componentName: ''
isError: false
spanType: Entry
peer: ''
peerId: 0
tags:
- {key: url, value: 'http://localhost:8080/kafka-scenario/case/kafka-case'}
- {key: http.method, value: GET}
- segmentId: not null
spans:
- operationName: Kafka/test/Consumer/testGroup
operationId: 0
parentSpanId: -1
spanId: 0
spanLayer: MQ
startTime: nq 0
endTime: nq 0
componentId: 41
componentName: ''
isError: false
spanType: Entry
peer: ''
peerId: 0
tags:
- {key: mq.broker, value: 'kafka-server:9092'}
- {key: mq.topic, value: test}
refs:
- {parentEndpointId: 0, parentEndpoint: /case/kafka-case, networkAddressId: 0,
entryEndpointId: 0, refType: CrossProcess, parentSpanId: 1, parentTraceSegmentId: not null,
parentServiceInstanceId: 1, networkAddress: 'kafka-server:9092', entryEndpoint: /case/kafka-case,
entryServiceInstanceId: 1
}

40
test/plugin/scenarios/kafka-scenario/configuration.yml Normal file
View File

@ -0,0 +1,40 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
type: jvm
entryService: http://localhost:8080/kafka-scenario/case/kafka-case
healthCheck: http://localhost:8080/kafka-scenario/case/healthCheck
startScript: ./bin/startup.sh
framework: kafka
environment:
- BOOTSTRAP_SERVERS=kafka-server:9092
depends_on:
- zookeeper-server
- kafka-server
dependencies:
zookeeper-server:
image: zookeeper:3.4
hostname: zookeeper-server
kafka-server:
image: bitnami/kafka:2.1.1
hostname: kafka-server
environment:
- KAFKA_ZOOKEEPER_CONNECT=zookeeper-server:2181
- KAFKA_BROKER_ID=1
- ALLOW_PLAINTEXT_LISTENER=yes
- KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092
depends_on:
- zookeeper-server

139
test/plugin/scenarios/kafka-scenario/pom.xml Normal file
View File

@ -0,0 +1,139 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.skywalking</groupId>
<artifactId>kafka-scenario</artifactId>
<version>5.0.0</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<compiler.version>1.8</compiler.version>
<test.framework.version>1.0.0</test.framework.version>
<test.framework>spring</test.framework>
<log4j.version>2.6.2</log4j.version>
<spring.version>4.3.8.RELEASE</spring.version>
<spring-boot-version>1.5.2.RELEASE</spring-boot-version>
</properties>
<name>skywalking-kafka-scenario</name>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>${test.framework.version}</version>
</dependency>
<!-- Spring Boot-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${spring-boot-version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jcl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
<version>${spring-boot-version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${spring-boot-version}</version>
</dependency>
</dependencies>
<build>
<finalName>kafka-scenario</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${compiler.version}</source>
<target>${compiler.version}</target>
<encoding>${project.build.sourceEncoding}</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>assemble</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
<outputDirectory>./target/</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<pluginRepositories>
<pluginRepository>
<id>spring-snapshots</id>
<url>http://repo.spring.io/snapshot</url>
</pluginRepository>
<pluginRepository>
<id>spring-milestones</id>
<url>http://repo.spring.io/milestone</url>
</pluginRepository>
</pluginRepositories>
</project>

41
test/plugin/scenarios/kafka-scenario/src/main/assembly/assembly.xml Normal file
View File

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>./bin</directory>
<fileMode>0775</fileMode>
</fileSet>
</fileSets>
<files>
<file>
<source>${project.build.directory}/kafka-scenario.jar</source>
<outputDirectory>./libs</outputDirectory>
<fileMode>0775</fileMode>
</file>
</files>
</assembly>

33
test/plugin/scenarios/kafka-scenario/src/main/java/test/org/apache/skywalking/apm/testcase/kafka/Application.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package test.org.apache.skywalking.apm.testcase.kafka;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author zhangwei
*/
@SpringBootApplication
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

193
test/plugin/scenarios/kafka-scenario/src/main/java/test/org/apache/skywalking/apm/testcase/kafka/controller/CaseController.java Normal file
View File

@ -0,0 +1,193 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package test.org.apache.skywalking.apm.testcase.kafka.controller;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import javax.annotation.PostConstruct;
import java.util.Arrays;
import java.util.Properties;
import java.util.function.Consumer;
import static java.util.Objects.isNull;
@Controller
@RequestMapping("/case")
@PropertySource("classpath:application.properties")
public class CaseController {
private static final Logger logger = LogManager.getLogger(CaseController.class);
private static final String SUCCESS = "Success";
@Value("${bootstrap.servers:127.0.0.1:9092}")
private String bootstrapServers;
private String topicName;
private static volatile boolean KAFKA_STATUS = false;
@PostConstruct
private void setUp() {
topicName = "test";
new CheckKafkaProducerThread(bootstrapServers).start();
}
@RequestMapping("/kafka-case")
@ResponseBody
public String kafkaCase() {
wrapProducer(producer -> {
ProducerRecord<String, String> record = new ProducerRecord<String, String>(topicName, "testKey", Integer.toString(1));
record.headers().add("TEST", "TEST".getBytes());
producer.send(record, new Callback() {
@Override
public void onCompletion(RecordMetadata metadata, Exception exception) {
logger.info("send success metadata={}", metadata);
}
});
}, bootstrapServers);
Thread thread = new ConsumerThread();
thread.start();
try {
thread.join();
} catch (InterruptedException e) {
// ignore
}
return SUCCESS;
}
@RequestMapping("/healthCheck")
@ResponseBody
public String healthCheck() {
if (KAFKA_STATUS) {
return SUCCESS;
}
throw new RuntimeException("kafka not ready");
}
private static void wrapProducer(Consumer<Producer<String, String>> consFunc, String bootstrapServers) {
Properties producerProperties = new Properties();
producerProperties.put("bootstrap.servers", bootstrapServers);
producerProperties.put("acks", "all");
producerProperties.put("retries", 0);
producerProperties.put("batch.size", 16384);
producerProperties.put("linger.ms", 1);
producerProperties.put("buffer.memory", 33554432);
producerProperties.put("auto.create.topics.enable", "true");
producerProperties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producerProperties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
Producer<String, String> producer = new KafkaProducer<>(producerProperties);
try {
consFunc.accept(producer);
} finally {
producer.close();
}
}
public static class CheckKafkaProducerThread extends Thread {
private final String bootstrapServers;
public CheckKafkaProducerThread(String bootstrapServers) {
setDaemon(true);
this.bootstrapServers = bootstrapServers;
}
@Override
public void run() {
while (true) {
try {
if (KAFKA_STATUS) {
return;
}
wrapProducer(producer -> {
ProducerRecord<String, String> record = new ProducerRecord<String, String>("check", "checkKey", Integer.toString(1));
record.headers().add("CHECK", "CHECK".getBytes());
producer.send(record, new Callback() {
@Override
public void onCompletion(RecordMetadata recordMetadata, Exception e) {
if (isNull(e)) {
KAFKA_STATUS = true;
}
}
});
}, bootstrapServers);
} catch (Exception e) {
logger.error("check " + bootstrapServers + " " + e.getMessage(), e);
}
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
// ignore
}
}
}
}
public class ConsumerThread extends Thread {
@Override
public void run() {
Properties consumerProperties = new Properties();
consumerProperties.put("bootstrap.servers", bootstrapServers);
consumerProperties.put("group.id", "testGroup");
consumerProperties.put("enable.auto.commit", "true");
consumerProperties.put("auto.commit.interval.ms", "1000");
consumerProperties.put("auto.offset.reset", "earliest");
consumerProperties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
consumerProperties.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(consumerProperties);
consumer.subscribe(Arrays.asList(topicName));
int i = 0;
while (i++ <= 10) {
try {
Thread.sleep(1 * 1000);
} catch (InterruptedException e) {
}
ConsumerRecords<String, String> records = consumer.poll(100);
if (!records.isEmpty()) {
for (ConsumerRecord<String, String> record : records) {
logger.info("header: {}", new String(record.headers().headers("TEST").iterator().next().value()));
logger.info("offset = {}, key = {}, value = {}", record.offset(), record.key(), record.value());
}
break;
}
}
consumer.close();
}
}
}

20
test/plugin/scenarios/kafka-scenario/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,20 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
server.port=8080
server.contextPath=/kafka-scenario

30
test/plugin/scenarios/kafka-scenario/src/main/resources/log4j2.xml Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_ERR">
<PatternLayout charset="UTF-8" pattern="[%d{yyyy-MM-dd HH:mm:ss:SSS}] [%p] - %l - %m%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="WARN">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>

32
test/plugin/scenarios/kafka-scenario/support-version.list Normal file
View File

@ -0,0 +1,32 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
0.11.0.0
0.11.0.1
0.11.0.2
0.11.0.3
1.0.0
1.0.1
1.0.2
1.1.0
1.1.1
2.0.0
2.0.1
2.1.0
2.1.1
2.2.0
2.2.1
2.3.0