diff --git a/.gitignore b/.gitignore index 15584cace..25435a095 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ .pyc +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store +jenkins-scripts/dsl/job-dsl-core-1.77-standalone.jar diff --git a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy index 037a59c06..2c2705375 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFUNIXBase.groovy @@ -10,10 +10,10 @@ import javaposse.jobdsl.dsl.Job - allow concurrent builds - bash: RTOOLS checkout */ -class OSRFUNIXBase extends OSRFBase +class OSRFUNIXBase extends OSRFBase { - static void create(Job job) - { + static void create(Job job) + { OSRFBase.create(job) job.with @@ -38,6 +38,63 @@ class OSRFUNIXBase extends OSRFBase git clone https://github.com/gazebo-tooling/release-tools scripts -b \$RTOOLS_BRANCH """.stripIndent()) } + publishers { + postBuildScripts { + steps{ + conditionalSteps { + condition { + expression('(.)*gpu-nvidia(.)*','${NODE_LABELS}') + } + steps { + systemGroovyCommand('''\ + import hudson.model.Cause.UpstreamCause; + import hudson.model.*; + + def node = build.getBuiltOn() + def old_labels = node.getLabelString() + + println("# BEGIN SECTION: NVIDIA MISMATCH RECOVERY") + if (!(build.getLog(1000) =~ "nvml error: driver/library version mismatch")) { + println("Build failed for other reason than NVIDIA error - Not performing any more steps") + return 0; + } else { + try { + println("Removing labels and adding 'recovery-process' label to node") + node.setLabelString("recovery-process") + + println("Requeuing job :" + build.project.name) + def job = Hudson.instance.getJob(build.project.name) + + def params = build.getAllActions().find{ it instanceof ParametersAction }?.parameters + def cause = new UpstreamCause(build) + // wait 70s to build again so that the computer has time to reboot (e.g only one agent available) + def scheduled = job.scheduleBuild2(70, cause, new ParametersAction(params)) + if(!scheduled) { + throw new Exception("Job could not be requeued!") + } + println("Job requeued!") + + } catch (Exception ex) { + println("ERROR - CANNOT PERFORM RECOVERY ACTIONS FOR NVIDIA ERROR") + println("Restoring to previous state") + node.setLabelString(old_labels) + throw ex + } + } + println("# END SECTION: NVIDIA MISMATCH RECOVERY") + '''.stripIndent() + ) + shell("""sudo shutdown -r +1""") + } + } + } + onlyIfBuildSucceeds(false) + onlyIfBuildFails(true) + } + + } + } } -} + +} \ No newline at end of file