kazusa
/
skywalking-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test: add a tool to select a suitable group for a new plugin test and remove the outdated group size of plugin tests (#5676)

This commit is contained in:
kezhenxu94 2020-10-16 08:44:55 +08:00 committed by GitHub
parent 31a217c944
commit d113f58645
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 164 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
.github/workflows/plugins-test.0.yaml vendored
View File

@ -28,32 +28,32 @@ jobs:
fail-fast: true fail-fast: true
matrix: matrix:
case: case:
- { name: 'activemq-scenario', title: 'ActiveMQ 5.10.0-5.15.4 (22)' } - activemq-scenario
- { name: 'apm-toolkit-trace-scenario', title: 'apm-toolkit-trace (1)' } - apm-toolkit-trace-scenario
- { name: 'armeria-0.96minus-scenario', title: 'Armeria 0.63.0-0.95.0 (20)' } - armeria-0.96minus-scenario
- { name: 'armeria-0.96plus-scenario', title: 'Armeria 0.96.0-0.97.0 (2)' } - armeria-0.96plus-scenario
- { name: 'avro-scenario', title: 'Avro 1.7.0-1.8.2 (10)' } - avro-scenario
- { name: 'canal-scenario', title: 'Canal 1.0.24-1.1.2 (5)' } - canal-scenario
- { name: 'cassandra-java-driver-3.x-scenario', title: 'Cassandra 3.7.0-3.7.2 (3)' } - cassandra-java-driver-3.x-scenario
- { name: 'customize-scenario', title: 'customize (1)' } - customize-scenario
- { name: 'dubbo-2.5.x-scenario', title: 'Dubbo 2.5.x-2.6.x (10)' } - dubbo-2.5.x-scenario
- { name: 'dubbo-2.7.x-scenario', title: 'Dubbo 2.7.x (4)' } - dubbo-2.7.x-scenario
- { name: 'ehcache-2.x-scenario', title: 'Ehcache 2.8.x-2.10.x (19)' } - ehcache-2.x-scenario
- { name: 'elasticsearch-5.x-scenario', title: 'ElasticSearch 5.x (3)' } - elasticsearch-5.x-scenario
- { name: 'elasticsearch-6.x-scenario', title: 'ElasticSearch 6.7.1-6.8.4 (7)' } - elasticsearch-6.x-scenario
- { name: 'elasticsearch-7.x-scenario', title: 'ElasticSearch 7.0.0-7.5.2 (14)' } - elasticsearch-7.x-scenario
- { name: 'exception-checker-spring-scenario', title: 'SpringBoot exception checker 1.0.0 (1)' } - exception-checker-spring-scenario
- { name: 'exception-checker-tomcat-scenario', title: 'Tomcat exception checker 1.0.0 (1)' } - exception-checker-tomcat-scenario
- { name: 'feign-scenario', title: 'Feign 9.0.0-9.5.1 (8)' } - feign-scenario
- { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0(19)' } - finagle-17.10.x-scenario
- { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0 (18)' } - finagle-6.25.x-scenario
- { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0 (4)' } - finagle-6.44.x-scenario
- { name: 'gateway-2.1.x-scenario', title: 'Spring-Cloud-Gateway 2.1.x (7)' } - gateway-2.1.x-scenario
- { name: 'gateway-2.0.x-scenario', title: 'Spring-Cloud-Gateway 2.0.x (3)' } - gateway-2.0.x-scenario
- { name: 'grpc-scenario', title: 'gRPC 1.6.0-1.25.0 (22)' } - grpc-scenario
- { name: 'gson-scenario', title: 'Gson (7)' } - gson-scenario
- { name: 'elasticjob-3.x-scenario', title: 'elasticjob-3.x-scenario (1)' } - elasticjob-3.x-scenario
- { name: 'springmvc-reactive-scenario', title: 'springmvc-reactive-scenario (7)' } - springmvc-reactive-scenario
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -72,7 +72,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image - name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- name: ${{ matrix.case.title }} - name: ${{ matrix.case }}
run: bash test/plugin/run.sh ${{ matrix.case.name }} run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage - name: Report Coverage
run: bash -x tools/coverage/report.sh run: bash -x tools/coverage/report.sh

38
.github/workflows/plugins-test.1.yaml vendored
View File

@ -28,23 +28,23 @@ jobs:
fail-fast: true fail-fast: true
matrix: matrix:
case: case:
- { name: 'h2-scenario', title: 'H2 (24)' } - h2-scenario
- { name: 'httpasyncclient-scenario', title: 'HttpAsyncClient 4.0-4.1.3 (7)' } - httpasyncclient-scenario
- { name: 'httpclient-3.x-scenario', title: 'HttpClient 2.0-3.1 (5)' } - httpclient-3.x-scenario
- { name: 'httpclient-4.3.x-scenario', title: 'HttpClient 4.3.x-4.5.x (14)' } - httpclient-4.3.x-scenario
- { name: 'hystrix-scenario', title: 'Hystrix 1.4.20-1.5.12 (20)' } - hystrix-scenario
- { name: 'influxdb-scenario', title: 'InfluxDB Java 2.5-2.17 (12)' } - influxdb-scenario
- { name: 'jdk-http-scenario', title: 'JDK http (1)' } - jdk-http-scenario
- { name: 'jdk-threading-scenario', title: 'JDK Threading (1)' } - jdk-threading-scenario
- { name: 'jedis-scenario', title: 'Jedis 2.4.0-2.9.0 (18)' } - jedis-scenario
- { name: 'jetty-scenario', title: 'Jetty 9.x (63)' } - jetty-scenario
- { name: 'kafka-scenario', title: 'Kafka 0.11.0.0-2.3.0 (16)' } - kafka-scenario
- { name: 'kotlin-coroutine-scenario', title: 'Kotlin Coroutine 1.0.1-1.3.3 (4)' } - kotlin-coroutine-scenario
- { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' } - lettuce-scenario
- { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.12.7 (31)' } - mongodb-3.x-scenario
- { name: 'mongodb-4.x-scenario', title: 'Mongodb 4.0.0-4.1.0 (7)' } - mongodb-4.x-scenario
- { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' } - netty-socketio-scenario
- { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' } - postgresql-above9.4.1207-scenario
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -63,7 +63,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image - name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- name: ${{ matrix.case.title }} - name: ${{ matrix.case }}
run: bash test/plugin/run.sh ${{ matrix.case.name }} run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage - name: Report Coverage
run: bash -x tools/coverage/report.sh run: bash -x tools/coverage/report.sh

50
.github/workflows/plugins-test.2.yaml vendored
View File

@ -28,29 +28,29 @@ jobs:
fail-fast: true fail-fast: true
matrix: matrix:
case: case:
- { name: 'okhttp-scenario', title: 'OKHttp 3.0.x-3.14.x (34)' } - okhttp-scenario
- { name: 'play-scenario', title: 'Play! Framework (12)' } - play-scenario
- { name: 'postgresql-scenario', title: 'PostgreSQL 9.2.x-9.4.x (36)' } - postgresql-scenario
- { name: 'pulsar-scenario', title: 'Pulsar 2.2.0-2.4.1 (7)' } - pulsar-scenario
- { name: 'rabbitmq-scenario', title: 'RabbitMQ (12)' } - rabbitmq-scenario
- { name: 'redisson-scenario', title: 'Redisson 3.x (37)' } - redisson-scenario
- { name: 'resttemplate-4.x-scenario', title: 'RestTemplate 4.0.0.RELEASE-4.3.26.RELEASE (57)' } - resttemplate-4.x-scenario
- { name: 'servicecomb-0.x-scenario', title: 'ServiceComb 0.x (5)' } - servicecomb-0.x-scenario
- { name: 'servicecomb-1.x-scenario', title: 'ServiceComb 1.x (5)' } - servicecomb-1.x-scenario
- { name: 'shardingsphere-3.x-scenario', title: 'ShardingSphere 3.0.0 (1)' } - shardingsphere-3.x-scenario
- { name: 'shardingsphere-4.x-RC1-RC2-scenario', title: 'ShardingSphere 4.0.0-RC1-4.0.0-RC2 (2)' } - shardingsphere-4.x-RC1-RC2-scenario
- { name: 'shardingsphere-4.x-RC3-scenario', title: 'ShardingSphere 4.0.0-RC3 (1)' } - shardingsphere-4.x-RC3-scenario
- { name: 'shardingsphere-4.0.x-scenario', title: 'ShardingSphere 4.0.0-4.0.1 (2)' } - shardingsphere-4.0.x-scenario
- { name: 'shardingsphere-4.x-scenario', title: 'ShardingSphere 4.1.0-4.1.1 (2)' } - shardingsphere-4.x-scenario
- { name: 'sofarpc-scenario', title: 'SofaRPC 5.4.0-5.6.2 (23)' } - sofarpc-scenario
- { name: 'solrj-7.x-scenario', title: 'SolrJ 7.x (12)' } - solrj-7.x-scenario
- { name: 'spring-3.0.x-scenario', title: 'Spring 3.0.x (8)' } - spring-3.0.x-scenario
- { name: 'spring-cloud-feign-1.1.x-scenario', title: 'SpringCloud Feign 1.1.x (8)' } - spring-cloud-feign-1.1.x-scenario
- { name: 'spring-cloud-feign-1.2.x-scenario', title: 'SpringCloud Feign 1.2.x-1.4.x (23)' } - spring-cloud-feign-1.2.x-scenario
- { name: 'spring-cloud-feign-2.x-scenario', title: 'SpringCloud Feign 2.x (14)' } - spring-cloud-feign-2.x-scenario
- { name: 'spring-tx-scenario', title: 'Spring-tx 4.x+ (10)' } - spring-tx-scenario
- { name: 'struts2.3-scenario', title: 'Struts2.3 (35)' } - struts2.3-scenario
- { name: 'struts2.5-scenario', title: 'Struts2.5 (14)' } - struts2.5-scenario
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -69,7 +69,7 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image - name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- name: ${{ matrix.case.title }} - name: ${{ matrix.case }}
run: bash test/plugin/run.sh ${{ matrix.case.name }} run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage - name: Report Coverage
run: bash -x tools/coverage/report.sh run: bash -x tools/coverage/report.sh

56
.github/workflows/plugins-test.3.yaml vendored
View File

@ -28,32 +28,32 @@ jobs:
fail-fast: true fail-fast: true
matrix: matrix:
case: case:
- { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } - mysql-scenario
- { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' } - undertow-scenario
- { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' } - webflux-scenario
- { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' } - zookeeper-scenario
- { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' } - spring-3.1.x-scenario
- { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' } - spring-4.1.x-scenario
- { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' } - spring-4.3.x-scenario
- { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' } - spring-async-scenario
- { name: 'vertx-eventbus-3.x-scenario', title: 'Vert.x EventBus 3.2.0-3.9.1 (28)' } - vertx-eventbus-3.x-scenario
- { name: 'vertx-web-3.54minus-scenario', title: 'Vert.x Web 3.0.0-3.5.4 (16)' } - vertx-web-3.54minus-scenario
- { name: 'vertx-web-3.6plus-scenario', title: 'Vert.x Web 3.6.0-3.9.1 (14)' } - vertx-web-3.6plus-scenario
- { name: 'mariadb-scenario', title: 'Mariadb 2.x (8)' } - mariadb-scenario
- { name: 'quasar-scenario', title: 'quasar 0.7.x (5)' } - quasar-scenario
- { name: 'baidu-brpc-scenario', title: 'baidu-brpc 2.3.7-2.5.3 (12)' } - baidu-brpc-scenario
- { name: 'retransform-class-scenario', title: 'Retransform class (1)' } - retransform-class-scenario
- { name: 'retransform-class-tomcat-scenario', title: 'Retransform class (1)' } - retransform-class-tomcat-scenario
- { name: 'graphql-8.x-scenario', title: 'graphql-8.x 8.0 (1)' } - graphql-8.x-scenario
- { name: 'graphql-9.x-scenario', title: 'graphql-9.x 9.0-11.0 (3)' } - graphql-9.x-scenario
- { name: 'graphql-12.x-scenario', title: 'graphql-12.x 12.0-15.0 (4)' } - graphql-12.x-scenario
- { name: 'hbase-scenario', title: 'hbase-scenario (5)' } - hbase-scenario
- { name: 'spring-kafka-2.2.x-scenario', title: 'Spring-Kafka 2.2.x (7)' } - spring-kafka-2.2.x-scenario
- { name: 'spring-kafka-2.3.x-scenario', title: 'Spring-Kafka 2.3.x (7)' } - spring-kafka-2.3.x-scenario
- { name: 'spring-scheduled-scenario', title: 'Spring Scheduled 3.1.x-5.2.x (9)' } - spring-scheduled-scenario
- { name: 'elasticjob-2.x-scenario', title: 'elasticjob-2.x-scenario (2)' } - elasticjob-2.x-scenario
- { name: 'quartz-scheduler-2.x-scenario', title: 'quartz-scheduler-2.x-scenario (5)' } - quartz-scheduler-2.x-scenario
- { name: 'xxl-job-2.x-scenario', title: 'xxl-job-2.x-scenario (1)' } - xxl-job-2.x-scenario
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -72,8 +72,8 @@ jobs:
run: ./mvnw --batch-mode clean package -DskipTests -Pagent run: ./mvnw --batch-mode clean package -DskipTests -Pagent
- name: Build Docker image - name: Build Docker image
run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests run: ./mvnw --batch-mode -f test/plugin/pom.xml clean package -DskipTests
- name: ${{ matrix.case.title }} - name: ${{ matrix.case }}
run: bash test/plugin/run.sh ${{ matrix.case.name }} run: bash test/plugin/run.sh ${{ matrix.case }}
- name: Report Coverage - name: Report Coverage
run: bash -x tools/coverage/report.sh run: bash -x tools/coverage/report.sh

7
docs/en/guides/Plugin-test.md
View File

@ -618,11 +618,12 @@ rather than recompiling it every time.
Use `${SKYWALKING_HOME}/test/plugin/run.sh -h` to know more command options. Use `${SKYWALKING_HOME}/test/plugin/run.sh -h` to know more command options.
If the local test passed, then you could add it to `.github/workflows/plugins-test.<n>.yaml` file, which will drive the tests running on the Github Actions of official SkyWalking repository. If the local test passed, then you could add it to `.github/workflows/plugins-test.<n>.yaml` file, which will drive the tests running on the GitHub Actions of official SkyWalking repository.
Based on your plugin's name, please add the test case into file `.github/workflows/plugins-test.<n>.yaml`, by alphabetical orders. Based on your plugin's name, please add the test case into file `.github/workflows/plugins-test.<n>.yaml`, by alphabetical orders.
Every test case is a Github Actions Job. Please use the `<scenario name> + <version range> + (<supported version count>)` as the Job `title`, and the scenario directory as the Job `name`, Every test case is a GitHub Actions Job. Please use the scenario directory name as the case `name`,
mostly you'll just need to decide which file (`plugins-test.<n>.yaml`) to add your test case, and simply put one line (as follows) in it, take the existed cases as examples. mostly you'll just need to decide which file (`plugins-test.<n>.yaml`) to add your test case, and simply put one line (as follows) in it, take the existed cases as examples.
You can run `python3 tools/select-group.py` to see which file contains the least cases and add your cases into it, in order to balance the running time of each group.
If a test case required to run in JDK 14 environment, please add you test case into file `plugins-jdk14-test.<n>.yaml`. If a test case required to run in JDK 14 environment, please add you test case into file `plugins-jdk14-test.<n>.yaml`.
@ -637,6 +638,6 @@ jobs:
matrix: matrix:
case: case:
# ... # ...
- { name: '<your case name>', title: '<PluginName, i.e. Spring> (<Supported Version Count, i.e 12>)' } # <<== insert one line by alphabetical orders - <your scenario test directory name>
# ... # ...
``` ```

60
tools/select-group.py Normal file
View File

@ -0,0 +1,60 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import re
from os import path
from os.path import dirname
import yaml
def main():
root_dir = path.realpath(path.join(dirname(__file__), '..'))
group_count = {}
for root, sub_dirs, files in os.walk(root_dir):
for filename in files:
if re.match('plugins-test\\.\\d+\\.ya?ml', filename):
count_plugin_tests(filename, group_count, root_dir)
group_count = sorted(group_count.items(), key=lambda x: x[1])
for gc in group_count:
print(gc[0], gc[1])
def count_plugin_tests(filename, group_count, root_dir):
yaml_file = path.join(root_dir, '.github/workflows/', filename)
content = yaml.safe_load(open(yaml_file))
group_count[yaml_file] = 0
for job in content['jobs'].values():
if 'strategy' in job and 'matrix' in job['strategy'] and 'case' in job['strategy']['matrix']:
cases = job['strategy']['matrix']['case']
for case in cases:
count_scenario_cases(case, group_count, root_dir, yaml_file)
def count_scenario_cases(case, group_count, root_dir, yaml_file):
version_filename = path.join(root_dir, 'test/plugin/scenarios', case, 'support-version.list')
with open(version_filename) as version_file:
for line in version_file.readlines():
if not line.startswith('#') and len(line.strip()) > 0:
group_count[yaml_file] += 1
if __name__ == '__main__':
main()