From 96114ad307d12345b1f546cacf750df2f12ba402 Mon Sep 17 00:00:00 2001 From: Daniel Agar Date: Wed, 14 Feb 2018 20:06:16 -0500 Subject: [PATCH] WIP Jenkins cleanup --- Jenkinsfile | 562 +++++++--------------------------------------------- 1 file changed, 74 insertions(+), 488 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 74014e95053a..70b82a18fa8e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -7,230 +7,46 @@ pipeline { script { def builds = [:] + def docker_base = "px4io/px4-dev-base:2017-12-30" + def docker_nuttx = "px4io/px4-dev-nuttx:2017-12-30" + def docker_rpi = "px4io/px4-dev-raspi:2017-12-30" + def docker_armhf = "px4io/px4-dev-armhf:2017-12-30" + def docker_arch = "px4io/px4-dev-base-archlinux:2017-12-30" + + builds["px4fmu-v2"] = createBuildNode(docker_nuttx, "nuttx_px4fmu-v2_default") // nuttx default targets that are archived and uploaded to s3 - for (def option in ["px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1"]) { + for (def option in ["px4fmu-v3", "px4fmu-v4", "px4fmu-v4pro", "px4fmu-v5", "aerofc-v1"]) { def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-nuttx:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "git fetch --tags" - sh "make nuttx_${node_name}_default" - sh "make nuttx_${node_name}_rtps" - sh "make sizes" - sh "ccache -s" - archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true) - } - } - } - } - } + builds[node_name] = createBuildNode(docker_nuttx, "nuttx_${node_name}_default") } - - // special case for fmu-v2/fmu-v3 - builds["px4fmu-v2"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-nuttx:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "git fetch --tags" - sh "make px4io-v2_default" - sh "make nuttx_px4fmu-v2_default" - sh "make nuttx_px4fmu-v2_lpe" - sh "make nuttx_px4fmu-v3_default" - sh "make nuttx_px4fmu-v3_rtps" - sh "make sizes" - sh "ccache -s" - archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true) - } - } - } - } - } - - // nuttx default targets that are archived and uploaded to s3 for (def option in ["aerocore2", "auav-x21", "crazyflie", "mindpx-v2", "nxphlite-v3", "tap-v1"]) { def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-nuttx:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "git fetch --tags" - sh "make nuttx_${node_name}_default" - sh "make sizes" - sh "ccache -s" - archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true) - } - } - } - } - } + builds[node_name] = createBuildNode(docker_nuttx, "nuttx_${node_name}_default") } - // other nuttx default targets for (def option in ["px4-same70xplained-v1", "px4-stm32f4discovery", "px4cannode-v1", "px4esc-v1", "px4nucleoF767ZI-v1", "s2740vc-v1"]) { def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-nuttx:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make nuttx_${node_name}_default" - sh "make sizes" - sh "ccache -s" - } - } - } - } - } + builds[node_name] = createBuildNode(docker_nuttx, "nuttx_${node_name}_default") } + builds["posix_sitl"] = createBuildNode(docker_base, "posix_sitl_default") + builds["posix_sitl_default (GCC 7)"] = createBuildNode(docker_arch, "posix_sitl_default") - // posix_sitl - for (def option in ["sitl_default", "sitl_rtps"]) { - def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-base:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_${node_name}" - sh "ccache -s" - } - } - } - } - } - } - + builds["rpi"] = createBuildNode(docker_rpi, "posix_rpi_cross") + builds["bebop"] = createBuildNode(docker_rpi, "posix_bebop_default") - // raspberry pi and bebop (armhf) - for (def option in ["rpi_cross", "bebop_default"]) { - def node_name = "${option}" + builds["ocpoc"] = createBuildNode(docker_armhf, "posix_ocpoc_ubuntu") - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-raspi:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_${node_name}" - sh "ccache -s" - } - } - } - } - } - } - - - // other armhf (to be merged with raspi and bebop) - for (def option in ["ocpoc_ubuntu"]) { - def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.image('px4io/px4-dev-armhf:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_${node_name}" - sh "ccache -s" - } - } - } - } - } - } - - - // snapdragon eagle (posix + qurt) - for (def option in ["eagle_default"]) { - def node_name = "${option}" - - builds["${node_name}"] = { - node { - stage("Build Test ${node_name}") { - docker.withRegistry('https://registry.hub.docker.com', 'docker_hub_dagar') { - docker.image("lorenzmeier/px4-dev-snapdragon:2017-12-29").inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make ${node_name}" - sh "ccache -s" - } - } - } - } - } - } - } - - - // GCC7 posix - for (def option in ["sitl_default"]) { - def node_name = "${option}" - - builds["${node_name} (GCC7)"] = { - node { - stage("Build Test ${node_name} (GCC7)") { - docker.image('px4io/px4-dev-base-archlinux:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { - stage("${node_name}") { - checkout scm - sh "export" - sh "make distclean" - sh "ccache -z" - sh "make posix_${node_name}" - sh "ccache -s" - } - } - } - } - } - } + //builds["snapdragon"] = createBuildNode(docker_snapdragon, "eagle_default") parallel builds - } - } - } + } // script + } // steps + } // stage Builds stage('Test') { parallel { @@ -339,268 +155,8 @@ pipeline { } } - stage('ROS vtol mission new 1') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_new_1.txt vehicle:=standard_vtol' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_new_1.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_new_1.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-vtol_new_1.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS vtol mission new 2') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_new_2.txt vehicle:=standard_vtol' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_new_2.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_new_2.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-vtol_new_2.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS vtol mission old 1') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_old_1.txt vehicle:=standard_vtol' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_1.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_1.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-vtol_old_1.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS vtol mission old 2') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_old_2.txt vehicle:=standard_vtol' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_2.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_2.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-vtol_old_2.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS vtol mission old 3') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - //sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - //sh 'git fetch --tags' - //sh 'make posix_sitl_default' - //sh 'make posix_sitl_default sitl_gazebo' - //sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=vtol_old_3.txt vehicle:=standard_vtol' - } - post { - //success { - // sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_3.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - //} - failure { - //sh './Tools/upload_log.py -q --description "ROS mission test vtol_old_3.txt: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-vtol_old_3.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS MC mission box') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=multirotor_box.mission vehicle:=iris' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS mission test multirotor_box.mission: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS mission test multirotor_box.mission: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - sh'''#!/bin/bash -xe - find . -type f -name "ros*.xml" | \ - while read f - do mv "$f" "${f/.xml/-multirotor_box.xml}" - done - ''' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS offboard att') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_tests_offboard_attctl.test' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS offboard attitude test: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS offboard attitude test: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - stage('ROS offboard pos') { - agent { - docker { - image 'px4io/px4-dev-ros:2017-12-31' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } - steps { - sh 'export' - sh 'make distclean; rm -rf .ros; rm -rf .gazebo' - sh 'git fetch --tags' - sh 'make posix_sitl_default' - sh 'make posix_sitl_default sitl_gazebo' - sh './test/rostest_px4_run.sh mavros_posix_tests_offboard_posctl.test' - } - post { - success { - sh './Tools/upload_log.py -q --description "ROS offboard position test: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg' - } - failure { - sh './Tools/upload_log.py -q --description "ROS offboard position test: ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI --email "${CHANGE_AUTHOR_EMAIL}" .ros/rootfs/fs/microsd/log/*/*.ulg' - archiveArtifacts '**/*.ulg' - archiveArtifacts '.ros/*/px4/**.xml' - archiveArtifacts '.ros/log/**.log' - } - } - } - - } - } + } // steps + } // stage Builds stage('Generate Metadata') { @@ -611,9 +167,10 @@ pipeline { docker { image 'px4io/px4-dev-base:2017-12-30' } } steps { + sh 'export' sh 'make distclean' sh 'make airframe_metadata' - archiveArtifacts(artifacts: 'airframes.md, airframes.xml', fingerprint: true) + archiveArtifacts(artifacts: 'airframes.md, airframes.xml') } } @@ -622,9 +179,10 @@ pipeline { docker { image 'px4io/px4-dev-base:2017-12-30' } } steps { + sh 'export' sh 'make distclean' sh 'make parameters_metadata' - archiveArtifacts(artifacts: 'parameters.md, parameters.xml', fingerprint: true) + archiveArtifacts(artifacts: 'parameters.md, parameters.xml') } } @@ -633,9 +191,10 @@ pipeline { docker { image 'px4io/px4-dev-base:2017-12-30' } } steps { + sh 'export' sh 'make distclean' sh 'make module_documentation' - archiveArtifacts(artifacts: 'modules/*.md', fingerprint: true) + archiveArtifacts(artifacts: 'modules/*.md') } } @@ -655,31 +214,58 @@ pipeline { } } } - - stage('S3 Upload') { - agent { - docker { image 'px4io/px4-dev-base:2017-12-30' } - } - - when { - anyOf { - branch 'master' - branch 'beta' - branch 'stable' - } - } - - steps { - sh 'echo "uploading to S3"' - } - } } environment { CCACHE_DIR = '/tmp/ccache' CI = true } options { - buildDiscarder(logRotator(numToKeepStr: '5')) + buildDiscarder(logRotator(numToKeepStr: '10')) timeout(time: 60, unit: 'MINUTES') } } + +def createBuildNode(String docker_repo, String target) { + return { + node { + docker.image(docker_repo).inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { + stage(target) { + sh('export') + checkout scm + sh('make distclean') + sh('git fetch --tags') + sh('ccache -z') + sh('make ' + target) + sh('ccache -s') + sh('make sizes') + archiveArtifacts(artifacts: 'build/*/*.px4', fingerprint: true) + } + } + } + } +} + +def createROSMissionTest(String test_file, String mission_file, String vehicle_model) { + return { + node { + docker.image('px4io/px4-dev-ros:2017-12-30').inside('-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw') { + stage(target) { + sh('export') + checkout scm + sh('make distclean') + sh('rm -rf .ros; rm -rf .gazebo') + sh('git fetch --tags') + sh('ccache -z') + sh('make posix_sitl_default') + sh('ccache -s') + sh('ccache -z') + sh('make posix_sitl_default sitl_gazebo') + sh('ccache -s') + sh('./test/rostest_px4_run.sh ' + test_file + ' mission:=' + mission_file + ' vehicle:=' + vehicle_model) + sh('./Tools/upload_log.py -q --description "ROS mission test:' + mission_file + ' ${CHANGE_ID}" --feedback "${CHANGE_TITLE} - ${CHANGE_URL}" --source CI .ros/rootfs/fs/microsd/log/*/*.ulg') + archiveArtifacts(artifacts: '**/*.ulg') + } + } + } + } +}