From 0c142121815b42a0d4103ab422b7e31abd9f0aa0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 4 Jun 2024 18:40:36 +0200 Subject: [PATCH 01/11] Implement custom_rosdep_urls for ci jobs Signed-off-by: Jose Luis Rivero --- ros_buildfarm/ci_job.py | 1 + ros_buildfarm/config/build_file.py | 8 ++++++++ .../scripts/ci/create_workspace_task_generator.py | 4 +++- ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em | 1 + 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ros_buildfarm/ci_job.py b/ros_buildfarm/ci_job.py index 7bd68594a..1d07f9344 100644 --- a/ros_buildfarm/ci_job.py +++ b/ros_buildfarm/ci_job.py @@ -284,6 +284,7 @@ def _get_ci_job_config( 'repository_names': repository_names, 'package_names': package_names, 'package_dependencies': package_dependencies, + 'custom_rosdep_urls': build_file.custom_rosdep_urls, 'skip_rosdep_keys': build_file.skip_rosdep_keys, 'install_packages': build_file.install_packages, diff --git a/ros_buildfarm/config/build_file.py b/ros_buildfarm/config/build_file.py index fff8fd7f0..3d549ea38 100644 --- a/ros_buildfarm/config/build_file.py +++ b/ros_buildfarm/config/build_file.py @@ -73,6 +73,14 @@ def __init__(self, name, data): # noqa: D107 self.targets[os_name][os_code_name][arch] = \ data['targets'][os_name][os_code_name][arch] + self.custom_rosdep_urls = [] + if '_config' in data['targets']: + if 'custom_rosdep_urls' in data['targets']['_config']: + self.custom_rosdep_urls = \ + data['targets']['_config']['custom_rosdep_urls'] + assert isinstance(self.custom_rosdep_urls, list) + + self.shared_ccache = False if 'shared_ccache' in data: self.shared_ccache = bool(data['shared_ccache']) diff --git a/ros_buildfarm/scripts/ci/create_workspace_task_generator.py b/ros_buildfarm/scripts/ci/create_workspace_task_generator.py index 1e0bc7920..7c9db6e6f 100644 --- a/ros_buildfarm/scripts/ci/create_workspace_task_generator.py +++ b/ros_buildfarm/scripts/ci/create_workspace_task_generator.py @@ -21,6 +21,7 @@ from ros_buildfarm.argument import add_argument_arch from ros_buildfarm.argument import \ add_argument_distribution_repository_key_files +from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars @@ -50,6 +51,7 @@ def main(argv=sys.argv[1:]): add_argument_os_code_name(parser) add_argument_arch(parser) + add_argument_custom_rosdep_urls(parser) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) @@ -105,7 +107,7 @@ def main(argv=sys.argv[1:]): 'rosdistro_name': args.rosdistro_name, - 'custom_rosdep_urls': [], + 'custom_rosdep_urls': args.custom_rosdep_urls, 'uid': get_user_id(), diff --git a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em index 7a4f36be2..29545aa7c 100644 --- a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em +++ b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em @@ -80,6 +80,7 @@ cmds = [ ' --repository-names ' + ' '.join(repository_names) + \ ((' --package-names ' + ' '.join(package_names)) if package_names else '') + \ (' --package-dependencies' if package_dependencies else '') + \ + ' --custom_rosdep_urls ' + ' '.join(custom_rosdep_urls) + \ ' --test-branch "%s"' % (test_branch) + \ ' --skip-rosdep-keys ' + ' '.join(skip_rosdep_keys) + \ ' --package-selection-args ' + ' '.join(package_selection_args), From 8d25b5c5f169324d54e5428d2c867097ab51ddb8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 4 Jun 2024 19:50:15 +0200 Subject: [PATCH 02/11] Include rosdep_urls in run_ci_job.py Signed-off-by: Jose Luis Rivero --- ros_buildfarm/scripts/ci/run_ci_job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ros_buildfarm/scripts/ci/run_ci_job.py b/ros_buildfarm/scripts/ci/run_ci_job.py index 2e76265ef..e206ccc3e 100644 --- a/ros_buildfarm/scripts/ci/run_ci_job.py +++ b/ros_buildfarm/scripts/ci/run_ci_job.py @@ -24,6 +24,7 @@ from ros_buildfarm.argument import add_argument_build_tool_test_args from ros_buildfarm.argument import \ add_argument_distribution_repository_key_files +from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars @@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]): add_argument_arch(parser) add_argument_build_tool(parser, required=True) + add_argument_custom_rosdep_urls(parser) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) From b67789df4bd903be9ce9761c00eee79b2ab7469d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 4 Jun 2024 20:13:35 +0200 Subject: [PATCH 03/11] Fix parameter invocation Signed-off-by: Jose Luis Rivero --- ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em index 29545aa7c..f70958234 100644 --- a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em +++ b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em @@ -80,7 +80,7 @@ cmds = [ ' --repository-names ' + ' '.join(repository_names) + \ ((' --package-names ' + ' '.join(package_names)) if package_names else '') + \ (' --package-dependencies' if package_dependencies else '') + \ - ' --custom_rosdep_urls ' + ' '.join(custom_rosdep_urls) + \ + ' --custom-rosdep-urls ' + ' '.join(custom_rosdep_urls) + \ ' --test-branch "%s"' % (test_branch) + \ ' --skip-rosdep-keys ' + ' '.join(skip_rosdep_keys) + \ ' --package-selection-args ' + ' '.join(package_selection_args), From 28c75afa3ecfe55d0271029727e19d1827d2e347 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 4 Jun 2024 21:01:13 +0200 Subject: [PATCH 04/11] Need wget for custom urls Signed-off-by: Jose Luis Rivero --- ros_buildfarm/templates/ci/create_workspace.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em index e6d6601a2..f80179426 100644 --- a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em +++ b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em @@ -72,7 +72,7 @@ RUN echo "@today_str" ))@ # needed for 'vcs custom --git --args merge' invocation -RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q --no-install-recommends sudo +RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q --no-install-recommends sudo wget RUN sudo -H -u buildfarm -- git config --global user.email "jenkins@@ros.invalid" && sudo -H -u buildfarm -- git config --global user.name "Jenkins ROS" @(TEMPLATE( From 0e136fab9ea1b93369794dbfa93e14c685a8e4e1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 5 Jun 2024 17:35:06 +0200 Subject: [PATCH 05/11] Missing -y Signed-off-by: Jose Luis Rivero --- ros_buildfarm/templates/ci/create_workspace.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em index f80179426..b28f56a9b 100644 --- a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em +++ b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em @@ -72,7 +72,7 @@ RUN echo "@today_str" ))@ # needed for 'vcs custom --git --args merge' invocation -RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q --no-install-recommends sudo wget +RUN python3 -u /tmp/wrapper_scripts/apt.py update-install-clean -q -y --no-install-recommends sudo wget RUN sudo -H -u buildfarm -- git config --global user.email "jenkins@@ros.invalid" && sudo -H -u buildfarm -- git config --global user.name "Jenkins ROS" @(TEMPLATE( From 371553db0bae4ca50461a2a641df88573b7de329 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 6 Jun 2024 15:53:52 +0200 Subject: [PATCH 06/11] Include documentation for custom_rosdep_urls in build files Signed-off-by: Jose Luis Rivero --- doc/configuration_options.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/configuration_options.rst b/doc/configuration_options.rst index acccc0489..12f356cb8 100644 --- a/doc/configuration_options.rst +++ b/doc/configuration_options.rst @@ -597,3 +597,15 @@ The following options are valid in version ``1`` (beside the generic options): to, if desired. By default, the resulting archives are only available to other jobs within Jenkins. + +The following options are valid as keys in the ``_config`` dict under +``targets``: + +* ``custom_rosdep_urls``: a list of URLs containing rosdep sources.list.d entry + files that are downloaded into /etc/ros/rosdep/sources.list.d at the beginning + of the doc job after running *rosdep init*. + Note that *rosdep init* will add the 20-default.list file from the public + rosdistro by default. + To override this, add an entry to this list corresponding to the + 20-default.list file from your forked rosdistro repository. + From e08db8a74e0efb399f2aebbb2362ae32bf5353b5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 7 Jun 2024 12:22:55 +0200 Subject: [PATCH 07/11] Fix linting Signed-off-by: Jose Luis Rivero --- ros_buildfarm/config/build_file.py | 1 - ros_buildfarm/scripts/ci/run_ci_job.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ros_buildfarm/config/build_file.py b/ros_buildfarm/config/build_file.py index 3d549ea38..61f2c34c2 100644 --- a/ros_buildfarm/config/build_file.py +++ b/ros_buildfarm/config/build_file.py @@ -80,7 +80,6 @@ def __init__(self, name, data): # noqa: D107 data['targets']['_config']['custom_rosdep_urls'] assert isinstance(self.custom_rosdep_urls, list) - self.shared_ccache = False if 'shared_ccache' in data: self.shared_ccache = bool(data['shared_ccache']) diff --git a/ros_buildfarm/scripts/ci/run_ci_job.py b/ros_buildfarm/scripts/ci/run_ci_job.py index e206ccc3e..5485e97a6 100644 --- a/ros_buildfarm/scripts/ci/run_ci_job.py +++ b/ros_buildfarm/scripts/ci/run_ci_job.py @@ -22,9 +22,9 @@ from ros_buildfarm.argument import add_argument_build_tool from ros_buildfarm.argument import add_argument_build_tool_args from ros_buildfarm.argument import add_argument_build_tool_test_args +from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import \ add_argument_distribution_repository_key_files -from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars From c3159bb1a363771bc74ea6b404cb437cc295d05a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 7 Jun 2024 12:38:52 +0200 Subject: [PATCH 08/11] Do not fail in no _config is defined Signed-off-by: Jose Luis Rivero --- ros_buildfarm/config/build_file.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ros_buildfarm/config/build_file.py b/ros_buildfarm/config/build_file.py index 61f2c34c2..cc7fd1371 100644 --- a/ros_buildfarm/config/build_file.py +++ b/ros_buildfarm/config/build_file.py @@ -63,9 +63,13 @@ def __init__(self, name, data): # noqa: D107 assert isinstance(self.tag_blacklist, list) self.targets = {} + self.custom_rosdep_urls = [] for os_name in data.get('targets', {}).keys(): if os_name == '_config': - continue + if 'custom_rosdep_urls' in data['targets']['_config']: + self.custom_rosdep_urls = \ + data['targets']['_config']['custom_rosdep_urls'] + assert isinstance(self.custom_rosdep_urls, list) self.targets[os_name] = {} for os_code_name in data['targets'][os_name].keys(): self.targets[os_name][os_code_name] = {} @@ -73,13 +77,6 @@ def __init__(self, name, data): # noqa: D107 self.targets[os_name][os_code_name][arch] = \ data['targets'][os_name][os_code_name][arch] - self.custom_rosdep_urls = [] - if '_config' in data['targets']: - if 'custom_rosdep_urls' in data['targets']['_config']: - self.custom_rosdep_urls = \ - data['targets']['_config']['custom_rosdep_urls'] - assert isinstance(self.custom_rosdep_urls, list) - self.shared_ccache = False if 'shared_ccache' in data: self.shared_ccache = bool(data['shared_ccache']) From 01cc3f032c2d0d4407e9b99a4faa7f47fc1a2587 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 7 Jun 2024 17:14:31 +0200 Subject: [PATCH 09/11] Right else clause for non _config targets Signed-off-by: Jose Luis Rivero --- ros_buildfarm/config/build_file.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ros_buildfarm/config/build_file.py b/ros_buildfarm/config/build_file.py index cc7fd1371..f94a11951 100644 --- a/ros_buildfarm/config/build_file.py +++ b/ros_buildfarm/config/build_file.py @@ -70,12 +70,13 @@ def __init__(self, name, data): # noqa: D107 self.custom_rosdep_urls = \ data['targets']['_config']['custom_rosdep_urls'] assert isinstance(self.custom_rosdep_urls, list) - self.targets[os_name] = {} - for os_code_name in data['targets'][os_name].keys(): - self.targets[os_name][os_code_name] = {} - for arch in data['targets'][os_name][os_code_name].keys(): - self.targets[os_name][os_code_name][arch] = \ - data['targets'][os_name][os_code_name][arch] + else: + self.targets[os_name] = {} + for os_code_name in data['targets'][os_name].keys(): + self.targets[os_name][os_code_name] = {} + for arch in data['targets'][os_name][os_code_name].keys(): + self.targets[os_name][os_code_name][arch] = \ + data['targets'][os_name][os_code_name][arch] self.shared_ccache = False if 'shared_ccache' in data: From 895706c0b4c3848adbe04b0f3dbdf787dd263570 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 10 Jun 2024 16:18:30 +0200 Subject: [PATCH 10/11] Fix linting Signed-off-by: Jose Luis Rivero --- ros_buildfarm/scripts/ci/create_workspace_task_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/scripts/ci/create_workspace_task_generator.py b/ros_buildfarm/scripts/ci/create_workspace_task_generator.py index 7c9db6e6f..d2a558637 100644 --- a/ros_buildfarm/scripts/ci/create_workspace_task_generator.py +++ b/ros_buildfarm/scripts/ci/create_workspace_task_generator.py @@ -19,9 +19,9 @@ from apt import Cache from ros_buildfarm.argument import add_argument_arch +from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import \ add_argument_distribution_repository_key_files -from ros_buildfarm.argument import add_argument_custom_rosdep_urls from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars From 7884b1a1f5e17e1f9adf7bfce800c09223acd462 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Sun, 16 Jun 2024 18:10:09 +0200 Subject: [PATCH 11/11] Add code into the ci_build_file instead of the build_file Signed-off-by: Jose Luis Rivero --- ros_buildfarm/config/build_file.py | 19 +++++++------------ ros_buildfarm/config/ci_build_file.py | 7 +++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/ros_buildfarm/config/build_file.py b/ros_buildfarm/config/build_file.py index f94a11951..fff8fd7f0 100644 --- a/ros_buildfarm/config/build_file.py +++ b/ros_buildfarm/config/build_file.py @@ -63,20 +63,15 @@ def __init__(self, name, data): # noqa: D107 assert isinstance(self.tag_blacklist, list) self.targets = {} - self.custom_rosdep_urls = [] for os_name in data.get('targets', {}).keys(): if os_name == '_config': - if 'custom_rosdep_urls' in data['targets']['_config']: - self.custom_rosdep_urls = \ - data['targets']['_config']['custom_rosdep_urls'] - assert isinstance(self.custom_rosdep_urls, list) - else: - self.targets[os_name] = {} - for os_code_name in data['targets'][os_name].keys(): - self.targets[os_name][os_code_name] = {} - for arch in data['targets'][os_name][os_code_name].keys(): - self.targets[os_name][os_code_name][arch] = \ - data['targets'][os_name][os_code_name][arch] + continue + self.targets[os_name] = {} + for os_code_name in data['targets'][os_name].keys(): + self.targets[os_name][os_code_name] = {} + for arch in data['targets'][os_name][os_code_name].keys(): + self.targets[os_name][os_code_name][arch] = \ + data['targets'][os_name][os_code_name][arch] self.shared_ccache = False if 'shared_ccache' in data: diff --git a/ros_buildfarm/config/ci_build_file.py b/ros_buildfarm/config/ci_build_file.py index 35f2506a4..12b32a078 100644 --- a/ros_buildfarm/config/ci_build_file.py +++ b/ros_buildfarm/config/ci_build_file.py @@ -152,3 +152,10 @@ def __init__(self, name, data): # noqa: D107 self.upload_directory = None if 'upload_directory' in data: self.upload_directory = data['upload_directory'] + + self.custom_rosdep_urls = [] + if '_config' in data['targets']: + if 'custom_rosdep_urls' in data['targets']['_config']: + self.custom_rosdep_urls = \ + data['targets']['_config']['custom_rosdep_urls'] + assert isinstance(self.custom_rosdep_urls, list)