From 432091b397173f8784b238b31c9bcaab0730b6f2 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 09:47:07 -0300 Subject: [PATCH 01/30] avoid unintentionally canceling the scheduled crate publishing job because publish-crates and publish-crates-manual share the resource group "crates-publishing", any instance of publish-crates-manual cancels a running instance of publish-crates, as demonstrated by https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2212179 a workaround for that unintended interaction is to avoid creating instances of publish-crates-manual and instead require pipelines to be triggered manually by checking $CI_JOB_MANUAL == "true" --- .gitlab-ci.yml | 4 ++++ scripts/ci/gitlab/pipeline/publish.yml | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0b1464bae23e..543cce232c4f3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -217,6 +217,10 @@ default: rules: - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" +.manual-crate-publishing-pipeline: + rules: + - if: $CI_COMMIT_REF_NAME == "master" && $CI_JOB_MANUAL == "true" && $PIPELINE == "crate-publishing" + .crates-publishing-template: stage: test extends: .docker-env diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index cc7451a9fbb22..7c170b6f4bb7b 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -198,6 +198,9 @@ update-node-template: .publish-crates-template: stage: publish + needs: + - job: publish-crates-locally + artifacts: false extends: .crates-publishing-template # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite # the releases of one another. Use resource_group to ensure that at most one instance of this job @@ -214,7 +217,7 @@ update-node-template: timeout: 9h # A custom publishing environment is used for us to be able to set up protected secrets # specifically for it - environment: publish-crates + environment: publish-crates script: - rusty-cachier snapshot create - git clone @@ -228,11 +231,8 @@ publish-crates: extends: - .publish-crates-template - .scheduled-crate-publishing-pipeline - needs: - - job: publish-crates-locally - artifacts: false publish-crates-manual: - extends: .publish-crates-template - when: manual - allow_failure: true + extends: + - .publish-crates-template + - .manual-crate-publishing-pipeline From 533e1ffcf9ea8f278faa6b9a933e3e2d6469847e Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 10:17:37 -0300 Subject: [PATCH 02/30] check manual pipelines by $CI_PIPELINE_SOURCE instead of $CI_JOB_MANUAL --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 543cce232c4f3..5966ac22d6f8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -219,7 +219,7 @@ default: .manual-crate-publishing-pipeline: rules: - - if: $CI_COMMIT_REF_NAME == "master" && $CI_JOB_MANUAL == "true" && $PIPELINE == "crate-publishing" + - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "web" && $PIPELINE == "crate-publishing" .crates-publishing-template: stage: test From ae6493fead49b12ba49f9421dfa2a8c2a716862c Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:26:24 -0300 Subject: [PATCH 03/30] make crate-publishing pipelines uninterruptible --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5966ac22d6f8e..d909b7ec1f965 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ default: - runner_system_failure - unknown_failure - api_failure - interruptible: true + interruptible: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" cache: {} .collect-artifacts: From a7a522cc355a0ed3b2b0c04afc1ddc1c9c81c35f Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:45:16 -0300 Subject: [PATCH 04/30] use conditional includes to work around interruptible limitations --- .gitlab-ci.yml | 23 +++++++++++-------- scripts/ci/gitlab/common-pipeline.yml | 9 ++++++++ .../ci/gitlab/crate-publishing-pipeline.yml | 5 ++++ scripts/ci/gitlab/default-pipeline.yml | 5 ++++ 4 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 scripts/ci/gitlab/common-pipeline.yml create mode 100644 scripts/ci/gitlab/crate-publishing-pipeline.yml create mode 100644 scripts/ci/gitlab/default-pipeline.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d909b7ec1f965..0ff774efda211 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,15 +55,20 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" -default: - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - interruptible: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" - cache: {} +# The crate-publishing pipeline should not be interruptible so that we'll be able to run the +# publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant +# because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might +# cancel it). Unfortunately `interruptible` can't currently be conditional as per: +# - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 +# - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 +# Thus we work around that limitation by using conditional includes. +include: + - local: ./scripts/ci/gitlab/crate-publishing-pipeline.yml + rules: + - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" + - local: ./scripts/ci/gitlab/default-pipeline.yml + rules: + - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" .collect-artifacts: artifacts: diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml new file mode 100644 index 0000000000000..2e730b54c882c --- /dev/null +++ b/scripts/ci/gitlab/common-pipeline.yml @@ -0,0 +1,9 @@ +.default: + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} + interruptible: true diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml new file mode 100644 index 0000000000000..85ec9d9c439c2 --- /dev/null +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -0,0 +1,5 @@ +include: "./common-pipeline.yml" + +default: + extends: .default + interruptible: false diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml new file mode 100644 index 0000000000000..fe485ffb6c15c --- /dev/null +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -0,0 +1,5 @@ +include: "./common-pipeline.yml" + +default: + extends: .default + interruptible: true From 70ef904a63a3f21bf83aece502dfbb40e8072edf Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:50:33 -0300 Subject: [PATCH 05/30] organize comments --- .gitlab-ci.yml | 6 ++---- scripts/ci/gitlab/crate-publishing-pipeline.yml | 4 ++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ff774efda211..41c588a39418d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,10 +55,8 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" -# The crate-publishing pipeline should not be interruptible so that we'll be able to run the -# publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant -# because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might -# cancel it). Unfortunately `interruptible` can't currently be conditional as per: +# The crate-publishing pipeline requires a customized `interruptible` configuration. Unfortunately +# `interruptible` can't currently be dynamically set based on variables as per: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index 85ec9d9c439c2..07bde56581b0e 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -2,4 +2,8 @@ include: "./common-pipeline.yml" default: extends: .default + # The crate-publishing pipeline should not be interruptible so that we'll be able to run the + # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant + # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might + # cancel it). interruptible: false From f16de79d02d884051647dd2b2790630d2f6dea4d Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:51:20 -0300 Subject: [PATCH 06/30] remove interruptible from common pipeline --- scripts/ci/gitlab/common-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml index 2e730b54c882c..ec3715962027a 100644 --- a/scripts/ci/gitlab/common-pipeline.yml +++ b/scripts/ci/gitlab/common-pipeline.yml @@ -6,4 +6,3 @@ - unknown_failure - api_failure cache: {} - interruptible: true From 8d5ea86c710780d264cd05b0b7cd0f7c5f7d3ad7 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:57:10 -0300 Subject: [PATCH 07/30] wip: check include --- scripts/ci/gitlab/default-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index fe485ffb6c15c..edd677d813c57 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,4 +1,4 @@ -include: "./common-pipeline.yml" +include: "./common-pipelie.yml" default: extends: .default From 370b140068b0b3984e109afd0927975094b29d57 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 11:59:22 -0300 Subject: [PATCH 08/30] wip: check include --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41c588a39418d..97c2a74142e20 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,12 +61,10 @@ variables: # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. include: + - local: ./scripts/ci/gitlab/default-pipeline.yml - local: ./scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" - - local: ./scripts/ci/gitlab/default-pipeline.yml - rules: - - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" .collect-artifacts: artifacts: From 0bc595861dba4c86700e191da6d6833eb883651d Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:00:41 -0300 Subject: [PATCH 09/30] fix include --- .gitlab-ci.yml | 23 ++++++++++++----------- scripts/ci/gitlab/default-pipeline.yml | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97c2a74142e20..cbcceb8b726dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,17 +55,6 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" -# The crate-publishing pipeline requires a customized `interruptible` configuration. Unfortunately -# `interruptible` can't currently be dynamically set based on variables as per: -# - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 -# - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 -# Thus we work around that limitation by using conditional includes. -include: - - local: ./scripts/ci/gitlab/default-pipeline.yml - - local: ./scripts/ci/gitlab/crate-publishing-pipeline.yml - rules: - - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" - .collect-artifacts: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" @@ -268,6 +257,18 @@ include: - scripts/ci/gitlab/pipeline/publish.yml # zombienet jobs - scripts/ci/gitlab/pipeline/zombienet.yml + # The crate-publishing pipeline requires a customized `interruptible` configuration. Unfortunately + # `interruptible` can't currently be dynamically set based on variables as per: + # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 + # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 + # Thus we work around that limitation by using conditional includes. + - local: ./scripts/ci/gitlab/crate-publishing-pipeline.yml + rules: + - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" + - local: ./scripts/ci/gitlab/default-pipeline.yml + rules: + - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" + #### stage: deploy diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index edd677d813c57..fe485ffb6c15c 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,4 +1,4 @@ -include: "./common-pipelie.yml" +include: "./common-pipeline.yml" default: extends: .default From 618c509fb49b1fa7278a07eadff0ba20135eb308 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:01:41 -0300 Subject: [PATCH 10/30] fix include --- .gitlab-ci.yml | 4 ++-- scripts/ci/gitlab/crate-publishing-pipeline.yml | 2 +- scripts/ci/gitlab/default-pipeline.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbcceb8b726dd..14471f6018ff3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -262,10 +262,10 @@ include: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. - - local: ./scripts/ci/gitlab/crate-publishing-pipeline.yml + - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" - - local: ./scripts/ci/gitlab/default-pipeline.yml + - local: scripts/ci/gitlab/default-pipeline.yml rules: - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index 07bde56581b0e..cd3bf434ba9ea 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,4 +1,4 @@ -include: "./common-pipeline.yml" +include: common-pipeline.yml default: extends: .default diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index fe485ffb6c15c..a45bfa9419732 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,4 +1,4 @@ -include: "./common-pipeline.yml" +include: common-pipeline.yml default: extends: .default From 6e4421f4b2f42ecb87dc4e1cab264e76316f8fbd Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:02:59 -0300 Subject: [PATCH 11/30] fix include --- scripts/ci/gitlab/crate-publishing-pipeline.yml | 2 +- scripts/ci/gitlab/default-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index cd3bf434ba9ea..ad81eb92936a8 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,4 +1,4 @@ -include: common-pipeline.yml +include: scripts/ci/gitlab/common-pipeline.yml default: extends: .default diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index a45bfa9419732..8cbf33ba16d21 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,4 +1,4 @@ -include: common-pipeline.yml +include: scripts/ci/gitlab/common-pipeline.yml default: extends: .default From 716fa112a95468575da368525b69319817e07bdb Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:04:19 -0300 Subject: [PATCH 12/30] fix yaml --- scripts/ci/gitlab/common-pipeline.yml | 2 +- scripts/ci/gitlab/crate-publishing-pipeline.yml | 2 +- scripts/ci/gitlab/default-pipeline.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml index ec3715962027a..03ef04abf9042 100644 --- a/scripts/ci/gitlab/common-pipeline.yml +++ b/scripts/ci/gitlab/common-pipeline.yml @@ -1,4 +1,4 @@ -.default: +.default: &default retry: max: 2 when: diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index ad81eb92936a8..d71881026bb17 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,7 +1,7 @@ include: scripts/ci/gitlab/common-pipeline.yml default: - extends: .default + <<: *default # The crate-publishing pipeline should not be interruptible so that we'll be able to run the # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 8cbf33ba16d21..3970228b87286 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,5 +1,5 @@ include: scripts/ci/gitlab/common-pipeline.yml default: - extends: .default + <<: *default interruptible: true From 261d47c62e4547f360e61b54929689888263a540 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:05:55 -0300 Subject: [PATCH 13/30] fix yaml --- scripts/ci/gitlab/common-pipeline.yml | 2 +- scripts/ci/gitlab/crate-publishing-pipeline.yml | 2 +- scripts/ci/gitlab/default-pipeline.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml index 03ef04abf9042..19bb1d904aaca 100644 --- a/scripts/ci/gitlab/common-pipeline.yml +++ b/scripts/ci/gitlab/common-pipeline.yml @@ -1,4 +1,4 @@ -.default: &default +.default-template: &default-template retry: max: 2 when: diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index d71881026bb17..f314ea18321e7 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,7 +1,7 @@ include: scripts/ci/gitlab/common-pipeline.yml default: - <<: *default + <<: *default-template # The crate-publishing pipeline should not be interruptible so that we'll be able to run the # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 3970228b87286..7fd398a03cddf 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,5 +1,5 @@ include: scripts/ci/gitlab/common-pipeline.yml default: - <<: *default + <<: *default-template interruptible: true From 6ef01204749143ab14992056063676c92f303d59 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:12:30 -0300 Subject: [PATCH 14/30] remove shared common-pipeline --- scripts/ci/gitlab/common-pipeline.yml | 8 -------- scripts/ci/gitlab/crate-publishing-pipeline.yml | 16 ++++++++-------- scripts/ci/gitlab/default-pipeline.yml | 10 +++++++--- 3 files changed, 15 insertions(+), 19 deletions(-) delete mode 100644 scripts/ci/gitlab/common-pipeline.yml diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml deleted file mode 100644 index 19bb1d904aaca..0000000000000 --- a/scripts/ci/gitlab/common-pipeline.yml +++ /dev/null @@ -1,8 +0,0 @@ -.default-template: &default-template - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index f314ea18321e7..4937003124733 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,9 +1,9 @@ -include: scripts/ci/gitlab/common-pipeline.yml - default: - <<: *default-template - # The crate-publishing pipeline should not be interruptible so that we'll be able to run the - # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant - # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might - # cancel it). - interruptible: false + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} + interruptible: true diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 7fd398a03cddf..4937003124733 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,5 +1,9 @@ -include: scripts/ci/gitlab/common-pipeline.yml - default: - <<: *default-template + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} interruptible: true From 7553643f12117fed4074ad9ad4ab4291b457dcfa Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:17:24 -0300 Subject: [PATCH 15/30] wip: retry common-pipeline --- .gitlab-ci.yml | 1 - scripts/ci/gitlab/common-pipeline.yml | 8 ++++++++ scripts/ci/gitlab/crate-publishing-pipeline.yml | 17 +++++++++-------- scripts/ci/gitlab/default-pipeline.yml | 11 ++++------- 4 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 scripts/ci/gitlab/common-pipeline.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14471f6018ff3..6d2512f61d2c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -269,7 +269,6 @@ include: rules: - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" - #### stage: deploy deploy-prometheus-alerting-rules: diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml new file mode 100644 index 0000000000000..84ac4210be45d --- /dev/null +++ b/scripts/ci/gitlab/common-pipeline.yml @@ -0,0 +1,8 @@ +.default-template: &default-template + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index 4937003124733..009aa8f374666 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,9 +1,10 @@ +include: + - local: scripts/ci/gitlab/common-pipeline.yml + default: - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} - interruptible: true + <<: *default-template + # The crate-publishing pipeline should not be interruptible so that we'll be able to run the + # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant + # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might + # cancel it). + interruptible: false diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 4937003124733..d57fc30c28358 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,9 +1,6 @@ +include: + - local: scripts/ci/gitlab/common-pipeline.yml + default: - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} + <<: *default-template interruptible: true From 211c9efa25cf14a3c71ef0878e1804612b3c6fed Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:18:50 -0300 Subject: [PATCH 16/30] move .default-template to .gitlab-ci.yml --- .gitlab-ci.yml | 9 +++++++++ scripts/ci/gitlab/common-pipeline.yml | 8 -------- scripts/ci/gitlab/crate-publishing-pipeline.yml | 3 --- scripts/ci/gitlab/default-pipeline.yml | 3 --- 4 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 scripts/ci/gitlab/common-pipeline.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d2512f61d2c8..69c73d84be696 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -230,6 +230,15 @@ variables: REPO_OWNER: paritytech SPUB_TMP: artifacts +.default-template: &default-template + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} + #### stage: .pre skip-if-draft: diff --git a/scripts/ci/gitlab/common-pipeline.yml b/scripts/ci/gitlab/common-pipeline.yml deleted file mode 100644 index 84ac4210be45d..0000000000000 --- a/scripts/ci/gitlab/common-pipeline.yml +++ /dev/null @@ -1,8 +0,0 @@ -.default-template: &default-template - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index 009aa8f374666..aaf74cd703fd1 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,6 +1,3 @@ -include: - - local: scripts/ci/gitlab/common-pipeline.yml - default: <<: *default-template # The crate-publishing pipeline should not be interruptible so that we'll be able to run the diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index d57fc30c28358..f1230bfd7da81 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,6 +1,3 @@ -include: - - local: scripts/ci/gitlab/common-pipeline.yml - default: <<: *default-template interruptible: true From e791c05cb100d98d51508a188cc6e03a493c33fc Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:24:50 -0300 Subject: [PATCH 17/30] fix the pipeline add comments --- .gitlab-ci.yml | 9 --------- scripts/ci/gitlab/crate-publishing-pipeline.yml | 10 +++++++++- scripts/ci/gitlab/default-pipeline.yml | 9 ++++++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69c73d84be696..6d2512f61d2c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -230,15 +230,6 @@ variables: REPO_OWNER: paritytech SPUB_TMP: artifacts -.default-template: &default-template - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} - #### stage: .pre skip-if-draft: diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index aaf74cd703fd1..f3c608abdb1d4 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,7 +1,15 @@ default: - <<: *default-template # The crate-publishing pipeline should not be interruptible so that we'll be able to run the # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might # cancel it). interruptible: false + + # From this line onwards the properties are shared with ./default-pipeline.yml + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index f1230bfd7da81..803860934e88e 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,3 +1,10 @@ default: - <<: *default-template + # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} interruptible: true From 74d128e84b2de0ba8df3d22d4a5211970bf8faee Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:25:41 -0300 Subject: [PATCH 18/30] fix default-pipeline.yml --- scripts/ci/gitlab/default-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 803860934e88e..f835d82ff480b 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,4 +1,6 @@ default: + interruptible: true + # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml retry: max: 2 @@ -7,4 +9,3 @@ default: - unknown_failure - api_failure cache: {} - interruptible: true From 2a68e4205daff964e964b47e17fe48e06a638489 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:37:27 -0300 Subject: [PATCH 19/30] revert publish-crates-manual to when: manual --- .gitlab-ci.yml | 9 +++------ scripts/ci/gitlab/pipeline/publish.yml | 9 ++++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d2512f61d2c8..b3013458ef95a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,12 +205,9 @@ variables: .scheduled-crate-publishing-pipeline: rules: + # `$CI_COMMIT_REF_NAME == "master"` according to publish-crates-manual's rule - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" -.manual-crate-publishing-pipeline: - rules: - - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "web" && $PIPELINE == "crate-publishing" - .crates-publishing-template: stage: test extends: .docker-env @@ -264,10 +261,10 @@ include: # Thus we work around that limitation by using conditional includes. - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - - if: $PIPELINE == "automatic-crate-publishing" || $PIPELINE == "crate-publishing" + - if: $PIPELINE == "automatic-crate-publishing" - local: scripts/ci/gitlab/default-pipeline.yml rules: - - if: $PIPELINE != "automatic-crate-publishing" && $PIPELINE != "crate-publishing" + - if: $PIPELINE != "automatic-crate-publishing" #### stage: deploy diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index 7c170b6f4bb7b..48e179532e479 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -233,6 +233,9 @@ publish-crates: - .scheduled-crate-publishing-pipeline publish-crates-manual: - extends: - - .publish-crates-template - - .manual-crate-publishing-pipeline + extends: .publish-crates-template + rules: + # `$CI_COMMIT_REF_NAME == "master"` according to publish-crates' rule + - if: $CI_COMMIT_REF_NAME == "master" + when: manual + allow_failure: true From cf20d132c315bc06b99587408d83cfdff9c6e2d6 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 6 Jan 2023 12:41:35 -0300 Subject: [PATCH 20/30] move "needs:" back to publish-crates --- scripts/ci/gitlab/pipeline/publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index 48e179532e479..1fc99ee0d67fa 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -198,9 +198,6 @@ update-node-template: .publish-crates-template: stage: publish - needs: - - job: publish-crates-locally - artifacts: false extends: .crates-publishing-template # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite # the releases of one another. Use resource_group to ensure that at most one instance of this job @@ -231,6 +228,9 @@ publish-crates: extends: - .publish-crates-template - .scheduled-crate-publishing-pipeline + needs: + - job: publish-crates-locally + artifacts: false publish-crates-manual: extends: .publish-crates-template From 56a3a73583155b0506256084279d34a6d5a9a33a Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 06:53:36 -0300 Subject: [PATCH 21/30] avoid manual repetition --- .gitlab-ci.yml | 8 +++++++- scripts/ci/gitlab/crate-publishing-pipeline.yml | 10 +--------- scripts/ci/gitlab/default-pipeline.yml | 10 +--------- scripts/ci/gitlab/pipeline-definitions.yml | 8 ++++++++ scripts/ci/gitlab/pipeline/publish.yml | 3 +-- 5 files changed, 18 insertions(+), 21 deletions(-) create mode 100644 scripts/ci/gitlab/pipeline-definitions.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3013458ef95a..d61785069fd6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,9 +203,14 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" +.crate-publishing-pipeline-rules: + ref: + - if: $CI_COMMIT_REF_NAME != "master" + when: never + .scheduled-crate-publishing-pipeline: rules: - # `$CI_COMMIT_REF_NAME == "master"` according to publish-crates-manual's rule + - !reference [.crate-publishing-pipeline-rules, ref] - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" .crates-publishing-template: @@ -259,6 +264,7 @@ include: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. + - scripts/ci/gitlab/pipeline-definitions.yml - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index f3c608abdb1d4..aaf74cd703fd1 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,15 +1,7 @@ default: + <<: *default-template # The crate-publishing pipeline should not be interruptible so that we'll be able to run the # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might # cancel it). interruptible: false - - # From this line onwards the properties are shared with ./default-pipeline.yml - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index f835d82ff480b..f1230bfd7da81 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,11 +1,3 @@ default: + <<: *default-template interruptible: true - - # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} diff --git a/scripts/ci/gitlab/pipeline-definitions.yml b/scripts/ci/gitlab/pipeline-definitions.yml new file mode 100644 index 0000000000000..84ac4210be45d --- /dev/null +++ b/scripts/ci/gitlab/pipeline-definitions.yml @@ -0,0 +1,8 @@ +.default-template: &default-template + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index 1fc99ee0d67fa..b923a125edba8 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -235,7 +235,6 @@ publish-crates: publish-crates-manual: extends: .publish-crates-template rules: - # `$CI_COMMIT_REF_NAME == "master"` according to publish-crates' rule - - if: $CI_COMMIT_REF_NAME == "master" + - !reference [.crate-publishing-pipeline-rules, ref] when: manual allow_failure: true From aff91c54fecf685b0ad80913c764a71da9cdb0d5 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 07:07:38 -0300 Subject: [PATCH 22/30] improve previous commit --- .gitlab-ci.yml | 9 ++++----- scripts/ci/gitlab/crate-publishing-pipeline.yml | 11 ++++++++++- scripts/ci/gitlab/default-pipeline.yml | 11 ++++++++++- scripts/ci/gitlab/pipeline-definitions.yml | 8 -------- scripts/ci/gitlab/pipeline/publish.yml | 2 +- 5 files changed, 25 insertions(+), 16 deletions(-) delete mode 100644 scripts/ci/gitlab/pipeline-definitions.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d61785069fd6b..c14e88e531e9c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,15 +203,15 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" -.crate-publishing-pipeline-rules: - ref: +.crate-publishing-pipeline: + rules: - if: $CI_COMMIT_REF_NAME != "master" when: never .scheduled-crate-publishing-pipeline: rules: - - !reference [.crate-publishing-pipeline-rules, ref] - - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" + - !reference [.crate-publishing-pipeline, rules] + - if: $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" .crates-publishing-template: stage: test @@ -264,7 +264,6 @@ include: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. - - scripts/ci/gitlab/pipeline-definitions.yml - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index aaf74cd703fd1..a1851670fdefc 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,7 +1,16 @@ default: - <<: *default-template # The crate-publishing pipeline should not be interruptible so that we'll be able to run the # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might # cancel it). interruptible: false + + # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml + # TODO: find way to avoid manual repetition + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index f1230bfd7da81..7abad568ac69c 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,3 +1,12 @@ default: - <<: *default-template interruptible: true + + # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml + # TODO: find way to avoid manual repetition + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} diff --git a/scripts/ci/gitlab/pipeline-definitions.yml b/scripts/ci/gitlab/pipeline-definitions.yml deleted file mode 100644 index 84ac4210be45d..0000000000000 --- a/scripts/ci/gitlab/pipeline-definitions.yml +++ /dev/null @@ -1,8 +0,0 @@ -.default-template: &default-template - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index b923a125edba8..a9f3bd8b5d7a9 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -235,6 +235,6 @@ publish-crates: publish-crates-manual: extends: .publish-crates-template rules: - - !reference [.crate-publishing-pipeline-rules, ref] + - !reference [.crate-publishing-pipeline, rules] when: manual allow_failure: true From 0dac70d5e8d587a3f13c13ce0553b2a3484bb3ca Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 07:20:31 -0300 Subject: [PATCH 23/30] try to avoid manual repetition --- .gitlab-ci.yml | 23 +++++++++++++++++++ .../ci/gitlab/crate-publishing-pipeline.yml | 17 +------------- scripts/ci/gitlab/default-pipeline.yml | 13 +---------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c14e88e531e9c..658a2f1d82624 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -232,6 +232,29 @@ variables: REPO_OWNER: paritytech SPUB_TMP: artifacts +.default-template: &default-template + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} + +.default-pipeline-definitions: + default: + <<: *default-template + interruptible: true + +.crate-publishing-pipeline-definitions: + default: + <<: *default-template + # The crate-publishing pipeline should not be interruptible so that we'll be able to run the + # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant + # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might + # cancel it). + interruptible: false + #### stage: .pre skip-if-draft: diff --git a/scripts/ci/gitlab/crate-publishing-pipeline.yml b/scripts/ci/gitlab/crate-publishing-pipeline.yml index a1851670fdefc..9d5303952e6ef 100644 --- a/scripts/ci/gitlab/crate-publishing-pipeline.yml +++ b/scripts/ci/gitlab/crate-publishing-pipeline.yml @@ -1,16 +1 @@ -default: - # The crate-publishing pipeline should not be interruptible so that we'll be able to run the - # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant - # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might - # cancel it). - interruptible: false - - # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml - # TODO: find way to avoid manual repetition - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} +default: !reference [.crate-publishing-pipeline-definitions, default] diff --git a/scripts/ci/gitlab/default-pipeline.yml b/scripts/ci/gitlab/default-pipeline.yml index 7abad568ac69c..19f6c320c3c24 100644 --- a/scripts/ci/gitlab/default-pipeline.yml +++ b/scripts/ci/gitlab/default-pipeline.yml @@ -1,12 +1 @@ -default: - interruptible: true - - # From this line onwards the properties are shared with ./crate-publishing-pipeline.yml - # TODO: find way to avoid manual repetition - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} +default: !reference [.default-pipeline-definitions, default] From 9c028810a795202aea6a579569bef47c207dfd97 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 07:24:01 -0300 Subject: [PATCH 24/30] fix indentation --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 658a2f1d82624..ed2193de9a00b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,7 +233,7 @@ variables: SPUB_TMP: artifacts .default-template: &default-template - retry: + retry: max: 2 when: - runner_system_failure From 294c092476c9de50e76c5f3a98df7cf71bf9eb93 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 11:14:13 -0300 Subject: [PATCH 25/30] minor adjustments --- .gitlab-ci.yml | 12 ++++++------ scripts/ci/gitlab/pipeline/publish.yml | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ed2193de9a00b..1d2b5c623a808 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -232,7 +232,7 @@ variables: REPO_OWNER: paritytech SPUB_TMP: artifacts -.default-template: &default-template +.shared-default: &shared-default retry: max: 2 when: @@ -243,16 +243,16 @@ variables: .default-pipeline-definitions: default: - <<: *default-template + <<: *shared-default interruptible: true .crate-publishing-pipeline-definitions: default: - <<: *default-template + <<: *shared-default # The crate-publishing pipeline should not be interruptible so that we'll be able to run the - # publishing automation jobs despite the "Auto-cancel redundant pipelines" CI setting (relevant - # because the crate-publishing pipeline runs on `master`, so future pipelines on `master` might - # cancel it). + # publishing automation jobs to completion despite the "Auto-cancel redundant pipelines" CI + # setting (relevant because the crate-publishing pipeline runs on `master`, so future pipelines + # on `master` might cancel it before the publishing jobs run to completion). interruptible: false #### stage: .pre diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index a9f3bd8b5d7a9..dff1b508625fb 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -234,7 +234,6 @@ publish-crates: publish-crates-manual: extends: .publish-crates-template - rules: - - !reference [.crate-publishing-pipeline, rules] + rules: !reference [.crate-publishing-pipeline, rules] when: manual allow_failure: true From c5952d6141e1a2f2c751af14ab2e1d804db32913 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 11:16:28 -0300 Subject: [PATCH 26/30] move defaults to top of .gitlab-ci.yml --- .gitlab-ci.yml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d2b5c623a808..0ce8aa4ac14f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,29 @@ workflow: - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH +.shared-default: &shared-default + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} + +.default-pipeline-definitions: + default: + <<: *shared-default + interruptible: true + +.crate-publishing-pipeline-definitions: + default: + <<: *shared-default + # The crate-publishing pipeline should not be interruptible so that we'll be able to run the + # publishing automation jobs to completion despite the "Auto-cancel redundant pipelines" CI + # setting (relevant because the crate-publishing pipeline runs on `master`, so future pipelines + # on `master` might cancel it before the publishing jobs run to completion). + interruptible: false + variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 @@ -232,29 +255,6 @@ variables: REPO_OWNER: paritytech SPUB_TMP: artifacts -.shared-default: &shared-default - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} - -.default-pipeline-definitions: - default: - <<: *shared-default - interruptible: true - -.crate-publishing-pipeline-definitions: - default: - <<: *shared-default - # The crate-publishing pipeline should not be interruptible so that we'll be able to run the - # publishing automation jobs to completion despite the "Auto-cancel redundant pipelines" CI - # setting (relevant because the crate-publishing pipeline runs on `master`, so future pipelines - # on `master` might cancel it before the publishing jobs run to completion). - interruptible: false - #### stage: .pre skip-if-draft: From 3f2909a1e33413e0629ca9992996539edf0219ff Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 9 Jan 2023 11:18:02 -0300 Subject: [PATCH 27/30] fix positioning on default in the diff --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ce8aa4ac14f6..a0012c215f54e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,19 @@ workflow: - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH +variables: + GIT_STRATEGY: fetch + GIT_DEPTH: 100 + CARGO_INCREMENTAL: 0 + DOCKER_OS: "debian:stretch" + ARCH: "x86_64" + CI_IMAGE: "paritytech/ci-linux:production" + BUILDAH_IMAGE: "quay.io/buildah/stable:v1.27" + RUSTY_CACHIER_SINGLE_BRANCH: master + RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true" + RUSTY_CACHIER_COMPRESSION_METHOD: zstd + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" + .shared-default: &shared-default retry: max: 2 @@ -65,19 +78,6 @@ workflow: # on `master` might cancel it before the publishing jobs run to completion). interruptible: false -variables: - GIT_STRATEGY: fetch - GIT_DEPTH: 100 - CARGO_INCREMENTAL: 0 - DOCKER_OS: "debian:stretch" - ARCH: "x86_64" - CI_IMAGE: "paritytech/ci-linux:production" - BUILDAH_IMAGE: "quay.io/buildah/stable:v1.27" - RUSTY_CACHIER_SINGLE_BRANCH: master - RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true" - RUSTY_CACHIER_COMPRESSION_METHOD: zstd - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" - .collect-artifacts: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" From 479da9111d6ae7cc8c13015a881a82163f2094a3 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 10 Jan 2023 08:19:09 -0300 Subject: [PATCH 28/30] comments --- .gitlab-ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0012c215f54e..bf384d4e1bc1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,10 +72,11 @@ variables: .crate-publishing-pipeline-definitions: default: <<: *shared-default - # The crate-publishing pipeline should not be interruptible so that we'll be able to run the - # publishing automation jobs to completion despite the "Auto-cancel redundant pipelines" CI - # setting (relevant because the crate-publishing pipeline runs on `master`, so future pipelines - # on `master` might cancel it before the publishing jobs run to completion). + # The crate-publishing pipeline defaults to `interruptible: false` so that we'll be able to + # reach and run the publishing jobs despite the "Auto-cancel redundant pipelines" CI setting. + # The setting is relevant because the crate-publishing pipeline runs on `master`, thus future + # pipelines on `master` (e.g. created for new commits or other schedules) might unintendedly + # cancel the publishing jobs or its dependencies before we get to actually publish the crates. interruptible: false .collect-artifacts: @@ -287,9 +288,14 @@ include: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. + # For crate-publishing pipelines: run it with defaults + `interruptible: false`. The WHOLE + # pipeline is made uninterruptible to ensure that test jobs also get a chance to run to + # completion, because the publishing jobs depends on them AS INTENDED: crates should not be + # published before their source code is checked. - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" + # For normal pipelines: run it with defaults + `interruptible: true` - local: scripts/ci/gitlab/default-pipeline.yml rules: - if: $PIPELINE != "automatic-crate-publishing" From 08d899d62b92ae2f3c35f5a3449ad53639fbe323 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 10 Jan 2023 08:24:10 -0300 Subject: [PATCH 29/30] indentation --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf384d4e1bc1b..811c926984559 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,13 +56,13 @@ variables: ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" .shared-default: &shared-default - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - cache: {} + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + cache: {} .default-pipeline-definitions: default: From c9699981c96a1ba9f74f85ea9f9e0bbb179b9ec2 Mon Sep 17 00:00:00 2001 From: JP <77391175+joao-paulo-parity@users.noreply.github.com> Date: Tue, 10 Jan 2023 08:35:17 -0300 Subject: [PATCH 30/30] Apply suggestions from code review Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 811c926984559..b41cf3af25e1b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,29 +55,29 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.22" -.shared-default: &shared-default +.shared-default: &shared-default retry: max: 2 when: - runner_system_failure - unknown_failure - api_failure - cache: {} + cache: {} .default-pipeline-definitions: default: - <<: *shared-default - interruptible: true + <<: *shared-default + interruptible: true .crate-publishing-pipeline-definitions: default: - <<: *shared-default + <<: *shared-default # The crate-publishing pipeline defaults to `interruptible: false` so that we'll be able to # reach and run the publishing jobs despite the "Auto-cancel redundant pipelines" CI setting. # The setting is relevant because the crate-publishing pipeline runs on `master`, thus future # pipelines on `master` (e.g. created for new commits or other schedules) might unintendedly # cancel the publishing jobs or its dependencies before we get to actually publish the crates. - interruptible: false + interruptible: false .collect-artifacts: artifacts: