From 6657e6ec1702cea093cc29f93b5fdda4efbd59a2 Mon Sep 17 00:00:00 2001 From: Daming Date: Tue, 22 Oct 2019 03:21:05 +0800 Subject: [PATCH] [test/plugin] fast-fail and reliablity (#3666) * fast-fail and reliablity * add workload no. as a part of the build_id tag * print system info --- Jenkinsfile-Agent-Test | 18 +++++---- Jenkinsfile-Agent-Test-2 | 20 +++++----- Jenkinsfile-Agent-Test-3 | 14 ++++--- test/plugin/run.sh | 2 +- .../resources/compose-start-script.template | 17 ++++++--- .../resources/container-start-script.template | 5 ++- .../src/main/resources/scenario.sh | 1 + test/plugin/script/systeminfo.sh | 38 +++++++++++++++++++ 8 files changed, 83 insertions(+), 32 deletions(-) create mode 100644 test/plugin/script/systeminfo.sh diff --git a/Jenkinsfile-Agent-Test b/Jenkinsfile-Agent-Test index 1af47995b..f404cd5e6 100755 --- a/Jenkinsfile-Agent-Test +++ b/Jenkinsfile-Agent-Test @@ -24,6 +24,7 @@ pipeline { options { timestamps() timeout(time: 5, unit: 'HOURS') + parallelsAlwaysFailFast() } tools { @@ -37,6 +38,7 @@ pipeline { stages { stage('Checkout Source Code') { steps { + sh 'bash ./test/plugin/script/systeminfo.sh' deleteDir() checkout scm sh 'git submodule update --init' @@ -51,7 +53,7 @@ pipeline { stage('Compile plugin-test tools Codes') { steps { - sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=${BUILD_ID} docker:build' + sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl1_${BUILD_ID} docker:build' } } stage('Test Cases Report (135)') { @@ -65,25 +67,25 @@ pipeline { stages { stage('httpclient 4.3.x-4.5.x (14)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} httpclient-4.3.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} httpclient-4.3.x-scenario' } } stage('ehcache 2.8.x-2.10.x (19)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} ehcache-2.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} ehcache-2.x-scenario' } } stage('dubbo 2.7.x (4)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} dubbo-2.7.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} dubbo-2.7.x-scenario' } } stage('spring async 4.3.x-5.1.x (35)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} spring-async-scenario' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} spring-async-scenario' } } } @@ -92,7 +94,7 @@ pipeline { stages { stage('jetty 9.x (63)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} jetty-scenario' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} jetty-scenario' } } } @@ -103,8 +105,8 @@ pipeline { post { cleanup { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} --cleanup' + sh 'bash test/plugin/run.sh --build_id=wl1_${BUILD_ID} --cleanup' deleteDir() } } -} +} \ No newline at end of file diff --git a/Jenkinsfile-Agent-Test-2 b/Jenkinsfile-Agent-Test-2 index 2011ad831..d4dac655f 100755 --- a/Jenkinsfile-Agent-Test-2 +++ b/Jenkinsfile-Agent-Test-2 @@ -24,6 +24,7 @@ pipeline { options { timestamps() timeout(time: 5, unit: 'HOURS') + parallelsAlwaysFailFast() } tools { @@ -37,6 +38,7 @@ pipeline { stages { stage('Checkout Source Code') { steps { + sh 'bash ./test/plugin/script/systeminfo.sh' deleteDir() checkout scm sh 'git submodule update --init' @@ -51,7 +53,7 @@ pipeline { stage('Compile plugin-test tools Codes') { steps { - sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=${BUILD_ID} docker:build' + sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl2_${BUILD_ID} docker:build' } } stage('Test Cases Report (132)') { @@ -65,31 +67,31 @@ pipeline { stages { stage('spring-cloud-gateway 2.1.x (3)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} gateway-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} gateway-scenario' } } stage('dubbo 2.5.x-2.6.x (10)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} dubbo-2.5.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} dubbo-2.5.x-scenario' } } stage('spring 3.0.x (8)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} spring-3.0.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} spring-3.0.x-scenario' } } stage('spring 3.1.x-4.0.x (25)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} spring-3.1.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} spring-3.1.x-scenario' } } stage('spring 4.1.x-4.2.x (20)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} spring-4.1.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} spring-4.1.x-scenario' } } } @@ -98,13 +100,13 @@ pipeline { stages { stage('solrj 7.x (12)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} solrj-7.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} solrj-7.x-scenario' } } stage('spring 4.3.x-5.2.x (54)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} spring-4.3.x-scenario' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} spring-4.3.x-scenario' } } } @@ -115,7 +117,7 @@ pipeline { post { cleanup { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} --cleanup' + sh 'bash test/plugin/run.sh --build_id=wl2_${BUILD_ID} --cleanup' deleteDir() } } diff --git a/Jenkinsfile-Agent-Test-3 b/Jenkinsfile-Agent-Test-3 index dfc4312a3..24db4da73 100755 --- a/Jenkinsfile-Agent-Test-3 +++ b/Jenkinsfile-Agent-Test-3 @@ -24,6 +24,7 @@ pipeline { options { timestamps() timeout(time: 5, unit: 'HOURS') + parallelsAlwaysFailFast() } tools { @@ -37,6 +38,7 @@ pipeline { stages { stage('Checkout Source Code') { steps { + sh 'bash ./test/plugin/script/systeminfo.sh' deleteDir() checkout scm sh 'git submodule update --init' @@ -51,7 +53,7 @@ pipeline { stage('Compile plugin-test tools Codes') { steps { - sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=${BUILD_ID} docker:build' + sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl3_${BUILD_ID} docker:build' } } stage('Test Cases Report (106)') { @@ -65,12 +67,12 @@ pipeline { stages { stage('kafka 0.11.0.0-2.3.0 (16)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} kafka-scenario' + sh 'bash test/plugin/run.sh --build_id=wl3_${BUILD_ID} kafka-scenario' } } stage('canal 1.0.24-1.1.2 (5)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} canal-scenario' + sh 'bash test/plugin/run.sh --build_id=wl3_${BUILD_ID} canal-scenario' } } } @@ -79,12 +81,12 @@ pipeline { stages { stage('postgresql 9.2.x-9.4.x (36)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} postgresql-scenario' + sh 'bash test/plugin/run.sh --build_id=wl3_${BUILD_ID} postgresql-scenario' } } stage('postgresql-9.4.1207+ (49)') { steps { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} postgresql-above9.4.1207-scenario' + sh 'bash test/plugin/run.sh --build_id=wl3_${BUILD_ID} postgresql-above9.4.1207-scenario' } } } @@ -95,7 +97,7 @@ pipeline { post { cleanup { - sh 'bash test/plugin/run.sh --build_id=${BUILD_ID} --cleanup' + sh 'bash test/plugin/run.sh --build_id=wl3_${BUILD_ID} --cleanup' deleteDir() } } diff --git a/test/plugin/run.sh b/test/plugin/run.sh index 52ebb4d63..66b3a124b 100644 --- a/test/plugin/run.sh +++ b/test/plugin/run.sh @@ -20,7 +20,7 @@ home="$(cd "$(dirname $0)"; pwd)" scenario_name="" parallel_run_size=1 force_build="off" -build_id="latest" +build_id="local" cleanup="off" mvnw=${home}/../../mvnw diff --git a/test/plugin/runner-helper/src/main/resources/compose-start-script.template b/test/plugin/runner-helper/src/main/resources/compose-start-script.template index 7a15280e3..d12cbfd2d 100644 --- a/test/plugin/runner-helper/src/main/resources/compose-start-script.template +++ b/test/plugin/runner-helper/src/main/resources/compose-start-script.template @@ -24,11 +24,16 @@ container_name="${project_name}_${docker_container_name}" docker-compose -p ${project_name} -f ${compose_file} up -d container=`docker ps -qf "name=${container_name}"` -status=`docker wait ${container}` -if [[ $status -ne 0 ]]; then - docker logs ${container} >&2 -fi +if [[ -z "${container}" ]]; then + echo "docker startup failure!" >&2 + status=1 +else + status=`docker wait ${container}` + if [[ $status -ne 0 ]]; then + docker logs ${container} >&2 + fi -docker-compose -p ${project_name} -f ${compose_file} kill -docker-compose -p ${project_name} -f ${compose_file} rm -f + docker-compose -p ${project_name} -f ${compose_file} kill + docker-compose -p ${project_name} -f ${compose_file} rm -f +fi \ No newline at end of file diff --git a/test/plugin/runner-helper/src/main/resources/container-start-script.template b/test/plugin/runner-helper/src/main/resources/container-start-script.template index 4c599af3b..7215c2d30 100644 --- a/test/plugin/runner-helper/src/main/resources/container-start-script.template +++ b/test/plugin/runner-helper/src/main/resources/container-start-script.template @@ -36,8 +36,9 @@ container_name=`docker ps -aqf "name=${docker_container_name}"` <#noparse> if [[ -z ${container_name} ]]; then - status=1 + echo "docker startup failure!" >&2 + status=1 else - docker container rm -f $container_name + docker container rm -f $container_name fi diff --git a/test/plugin/runner-helper/src/main/resources/scenario.sh b/test/plugin/runner-helper/src/main/resources/scenario.sh index 710c35189..3c0c6fe4d 100644 --- a/test/plugin/runner-helper/src/main/resources/scenario.sh +++ b/test/plugin/runner-helper/src/main/resources/scenario.sh @@ -23,6 +23,7 @@ PRGDIR=`dirname "$PRG"` state_house=$1 testcase_name=${scenario_name}-${scenario_version} +status=1 <#noparse>touch ${state_house}/${testcase_name}.RUNNING ${running_script} diff --git a/test/plugin/script/systeminfo.sh b/test/plugin/script/systeminfo.sh new file mode 100644 index 000000000..ccee064ea --- /dev/null +++ b/test/plugin/script/systeminfo.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hostname=$(hostname) +ip_address=$(hostname --ip-address) +echo -e "hostname: ${hostname}(${ip_address}) \n" + +cpu_usage_rate=$(grep 'cpu ' /proc/stat | awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage "%"}') +load_average=$(uptime |grep -o --color=never "load average:.*") + +echo -e "CPU usage rate: ${cpu_usage_rate}, ${load_average} \n" + +running=$(docker ps -q |wc -l) +all=$(docker ps -aq |wc -l) +volumes=$(docker volume ls |wc -l) + +echo -e "docker stats: running=${running:=0}, all=${all:=0}, volumes=${volumes:=0}\n" + +echo -e "Memory usage:" +free -m + +echo "Disk usage:" +df -h \ No newline at end of file