Make the shared ccache optional

doc/configuration_options.rst

@@ -113,6 +113,10 @@ A set of options which can be used in any build file.
   platform the associated docker image is based on, therefore no targets can
   be specified.
 
+* ``shared_ccache``: when set to ``true``, the executing user's ccache directory
+  is shared in the build container, which is configured to use ccache where
+  appropriate (default: ``false``).
+
 Description of common options
 -----------------------------
 

ros_buildfarm/ci_job.py

@@ -309,6 +309,8 @@ def _get_ci_job_config(
 
         'benchmark_patterns': build_file.benchmark_patterns,
         'benchmark_schema': build_file.benchmark_schema,
+
+        'shared_ccache': build_file.shared_ccache,
     }
     job_config = expand_template(template_name, job_data)
     return job_config

ros_buildfarm/config/build_file.py

@@ -73,6 +73,10 @@ def __init__(self, name, data):  # noqa: D107
                     self.targets[os_name][os_code_name][arch] = \
                         data['targets'][os_name][os_code_name][arch]
 
+        self.shared_ccache = False
+        if 'shared_ccache' in data:
+            self.shared_ccache = bool(data['shared_ccache'])
+
     def filter_distribution_files_by_tags(self, dist_files):
         res = []
 

ros_buildfarm/devel_job.py

@@ -448,6 +448,8 @@ def _get_devel_job_config(
 
         'benchmark_patterns': build_file.benchmark_patterns,
         'benchmark_schema': build_file.benchmark_schema,
+
+        'shared_ccache': build_file.shared_ccache,
     }
     job_config = expand_template(template_name, job_data)
     return job_config

ros_buildfarm/release_job.py

@@ -714,6 +714,8 @@ def _get_binarydeb_job_config(
         'timeout_minutes': build_file.jenkins_binary_job_timeout,
 
         'credential_id': build_file.upload_credential_id,
+
+        'shared_ccache': build_file.shared_ccache,
     }
     job_config = expand_template(template_name, job_data)
     return job_config

ros_buildfarm/templates/ci/ci_job.xml.em

@@ -298,6 +298,7 @@ parameters = [
         'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name),
         'echo "# END SECTION"',
         '',
+    ] + ([
         'echo "# BEGIN SECTION: ccache stats (before)"',
         'mkdir -p $HOME/.ccache',
         'docker run' +
@@ -309,6 +310,7 @@ parameters = [
         ' "ccache -s"',
         'echo "# END SECTION"',
         '',
+    ] if shared_ccache else []) + [
         'echo "# BEGIN SECTION: Run Dockerfile - build and install"',
     ] + [
         'export UNDERLAY%d_JOB_SPACE=$WORKSPACE/underlay%d/ros%d-linux' % (i + 1, i + 1, local_ros_version)
@@ -317,8 +319,8 @@ parameters = [
         'docker run' +
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_build_and_install/docker.cid' +
-        ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
+        ((' -e CCACHE_DIR=/home/buildfarm/.ccache' +
+          ' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ''.join([
             ' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '')
@@ -328,6 +330,7 @@ parameters = [
         ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
+    ] + ([
         '',
         'echo "# BEGIN SECTION: ccache stats (after)"',
         'docker run' +
@@ -338,7 +341,7 @@ parameters = [
         ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) +
         ' "ccache -s"',
         'echo "# END SECTION"',
-    ]),
+    ] if shared_ccache else [])),
 ))@
 @(SNIPPET(
     'builder_shell',
@@ -369,6 +372,7 @@ parameters = [
         'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name),
         'echo "# END SECTION"',
         '',
+    ] + ([
         'echo "# BEGIN SECTION: ccache stats (before)"',
         'mkdir -p $HOME/.ccache',
         'docker run' +
@@ -380,6 +384,7 @@ parameters = [
         ' "ccache -s"',
         'echo "# END SECTION"',
         '',
+    ] if shared_ccache else []) + [
         'echo "# BEGIN SECTION: Run Dockerfile - build and test"',
     ] + [
         'export UNDERLAY%d_JOB_SPACE=$WORKSPACE/underlay%d/ros%d-linux' % (i + 1, i + 1, local_ros_version)
@@ -390,8 +395,8 @@ parameters = [
         'docker run' +
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_build_and_test/docker.cid' +
-        ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
+        ((' -e CCACHE_DIR=/home/buildfarm/.ccache' +
+          ' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ''.join([
             ' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '')
@@ -401,6 +406,7 @@ parameters = [
         ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
+    ] + ([
         '',
         'echo "# BEGIN SECTION: ccache stats (after)"',
         'docker run' +
@@ -411,7 +417,7 @@ parameters = [
         ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) +
         ' "ccache -s"',
         'echo "# END SECTION"',
-    ]),
+    ] if shared_ccache else [])),
 ))@
 @(SNIPPET(
     'builder_shell',

ros_buildfarm/templates/devel/devel_job.xml.em

@@ -201,7 +201,9 @@ if pull_request:
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build and install"',
+    ] + ([
         'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
+    ]  if shared_ccache else []) + [
         ('if [ ! -c /dev/nvidia[0-9] ]; then echo "--require-gpu-support is enabled but can not detect nvidia support installed" && exit 1; fi' if require_gpu_support else ''),
         'docker run' +
         (' --env=DISPLAY=:0.0 --env=QT_X11_NO_MITSHM=1 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw --gpus all' if require_gpu_support else '') +
@@ -210,7 +212,7 @@ if pull_request:
         ' -e=TRAVIS=$TRAVIS' +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/ws:/tmp/ws' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
+        (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
         ' devel_build_and_install.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
@@ -232,16 +234,18 @@ if pull_request:
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build and test"',
-        ''
+        '',
+    ] + ([
         'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
+    ] if shared_ccache else []) + [
         'docker run' +
         (' --env=DISPLAY=:0.0 --env=QT_X11_NO_MITSHM=1 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw  --gpus all' if require_gpu_support else '') +
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_build_and_test/docker.cid' +
         ' -e=TRAVIS=$TRAVIS' +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/ws:/tmp/ws' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
+        (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
         ' devel_build_and_test.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',

ros_buildfarm/templates/release/deb/binarypkg_job.xml.em

@@ -127,7 +127,9 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'rm -fr $WORKSPACE/docker_build_binarydeb',
         'mkdir -p $WORKSPACE/binarydeb',
         'mkdir -p $WORKSPACE/docker_build_binarydeb',
+    ] + ([
         'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
+    ] if shared_ccache else []) + [
         'docker run' +
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_generating_docker/docker.cid' +
@@ -136,7 +138,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' +
         ' -v $WORKSPACE/docker_build_binarydeb:/tmp/docker_build_binarydeb' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' + \
+        (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
         ' binarydeb_task_generation.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
     ]),
@@ -159,7 +161,9 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Run Dockerfile - build binarydeb"',
         '# -e=HOME= is required to set a reasonable HOME for the user (not /)',
         '# otherwise apt-src will fail',
+    ] + ([
         'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
+    ] if shared_ccache else []) + [
         'docker run' +
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_build_binarydeb/docker.cid' +
@@ -168,7 +172,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         ' --net=host' +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' +
-        ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
+        (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
         ' binarydeb_build.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
     ]),

ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em

@@ -123,6 +123,9 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Run Dockerfile - build binaryrpm"',
         'rm -fr $WORKSPACE/binarypkg',
         'mkdir -p $WORKSPACE/binarypkg/source',
+    ] + ([
+        'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
+    ] if shared_ccache else []) + [
         'docker run' +
         ' --rm' +
         ' --privileged' +
@@ -131,7 +134,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         ' --net=host' +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/binarypkg:/tmp/binarypkg' +
-        ' -v ~/.ccache:/home/buildfarm/.ccache' +
+        (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
         ' binaryrpm.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
     ]),