From 4c3a3ec12899b65cca4634b608dd6d9218965ee1 Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Thu, 24 Oct 2019 01:13:08 +0800 Subject: [PATCH] Add CI build job barriers to prevent unnecessary builds (#3662) * Add CI build job barriers to prevent unnecessary builds * Complete change file sets to trigger build jobs * Polish --- Jenkinsfile-Agent-Test | 16 +++++++++ Jenkinsfile-Agent-Test-2 | 16 +++++++++ Jenkinsfile-Agent-Test-3 | 16 +++++++++ Jenkinsfile-E2E | 15 ++++++++ tools/ci/agent-build-condition.sh | 27 ++++++++++++++ tools/ci/changed.sh | 58 +++++++++++++++++++++++++++++++ tools/ci/e2e-build-condition.sh | 28 +++++++++++++++ 7 files changed, 176 insertions(+) create mode 100644 tools/ci/agent-build-condition.sh create mode 100644 tools/ci/changed.sh create mode 100644 tools/ci/e2e-build-condition.sh diff --git a/Jenkinsfile-Agent-Test b/Jenkinsfile-Agent-Test index 563556c03..69df4c499 100755 --- a/Jenkinsfile-Agent-Test +++ b/Jenkinsfile-Agent-Test @@ -46,12 +46,22 @@ pipeline { } stage('Compile agent Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -Pagent clean package -DskipTests' } } stage('Compile plugin-test tools Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl1_${BUILD_ID} docker:build' } @@ -61,7 +71,13 @@ pipeline { echo "reserve." } } + stage('Run Agent Plugin Tests') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } parallel { stage('Group1') { stages { diff --git a/Jenkinsfile-Agent-Test-2 b/Jenkinsfile-Agent-Test-2 index d4dac655f..af0cbed26 100755 --- a/Jenkinsfile-Agent-Test-2 +++ b/Jenkinsfile-Agent-Test-2 @@ -46,12 +46,22 @@ pipeline { } stage('Compile agent Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -Pagent clean package -DskipTests' } } stage('Compile plugin-test tools Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl2_${BUILD_ID} docker:build' } @@ -61,7 +71,13 @@ pipeline { echo "reserve." } } + stage('Run Agent Plugin Tests') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } parallel { stage('Group1') { stages { diff --git a/Jenkinsfile-Agent-Test-3 b/Jenkinsfile-Agent-Test-3 index 85671c67d..6469a9dd6 100755 --- a/Jenkinsfile-Agent-Test-3 +++ b/Jenkinsfile-Agent-Test-3 @@ -46,12 +46,22 @@ pipeline { } stage('Compile agent Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -Pagent clean package -DskipTests' } } stage('Compile plugin-test tools Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } steps { sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl3_${BUILD_ID} docker:build' } @@ -61,7 +71,13 @@ pipeline { echo "reserve." } } + stage('Run Agent Plugin Tests') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh') + } + } parallel { stage('Group1') { stages { diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E index 63c9163b5..63f2ca994 100755 --- a/Jenkinsfile-E2E +++ b/Jenkinsfile-E2E @@ -44,6 +44,11 @@ pipeline { } stage('Prepare Distribution Package') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh') + } + } steps { // although these checks are done in ci-it, since they are lightweight/cheap // we're using them as a barrier here to filter out some invalid PRs (fast-fail) @@ -54,12 +59,22 @@ pipeline { } stage('Compile Test Codes') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh') + } + } steps { sh './mvnw -f test/e2e/pom.xml -pl e2e-base clean install' } } stage('Run End-to-End Tests') { + when { + expression { + return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh') + } + } parallel { stage('Group 1') { stages { diff --git a/tools/ci/agent-build-condition.sh b/tools/ci/agent-build-condition.sh new file mode 100644 index 000000000..7d708e308 --- /dev/null +++ b/tools/ci/agent-build-condition.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash -x $(dirname "$0")/changed.sh --any-of '^pom.xml$' \ + '^Jenkinsfile-Agent-.*$' \ + '^apm-application-toolkit/.*$' \ + '^apm-commons/.*$' \ + '^apm-protocol/.*$' \ + '^apm-sniffer/.*$' \ + '^test/plugin/.*$' \ + '^tools/ci/agent-build-condition.sh$' \ + '^tools/ci/changed.sh$' diff --git a/tools/ci/changed.sh b/tools/ci/changed.sh new file mode 100644 index 000000000..cc107576c --- /dev/null +++ b/tools/ci/changed.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +patterns=() +any_of=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --any-of) + any_of=1 + ;; + *) + patterns+=($1) + ;; + esac + shift +done + +[[ ${#patterns[@]} -eq 0 ]] && echo 'No file pattern is specified, exiting' && exit 1 + +changed_files=$(git diff --name-only origin/${ghprbTargetBranch:-master}..${ghprbActualCommit:-HEAD}) + +test_results=() + +for file in ${changed_files}; do + for pattern in ${patterns[@]}; do + if [[ ${file} =~ ${pattern} ]]; then + test_results+=("Hit: ${file} matches pattern ${pattern}") + else + test_results+=("Miss: ${file} does not match pattern ${pattern}") + fi + done +done + +IFS=$'\n' ; echo "${test_results[*]}" + +if [[ ${any_of} -eq 1 ]]; then + for test_result in ${test_results[@]}; do + [[ ${test_result} =~ ^Hit:.+$ ]] && echo ${test_result} && exit 1 + done +fi + +exit 0 \ No newline at end of file diff --git a/tools/ci/e2e-build-condition.sh b/tools/ci/e2e-build-condition.sh new file mode 100644 index 000000000..5de72553a --- /dev/null +++ b/tools/ci/e2e-build-condition.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bash -x $(dirname "$0")/changed.sh --any-of '^pom.xml$' \ + '^Jenkinsfile-E2E$' \ + '^apm-commons/.*$' \ + '^apm-protocol/.*$' \ + '^apm-sniffer/.*$' \ + '^apm-webapp/.*$' \ + '^oap-server/.*$' \ + '^test/e2e/.*$' \ + '^tools/ci/e2e-build-condition.sh$' \ + '^tools/ci/changed.sh$'