From e8d16d53c9cb98ff56800e06c2871ac7a49f7652 Mon Sep 17 00:00:00 2001 From: Aaron Crickenberger Date: Mon, 31 Aug 2020 19:21:26 -0700 Subject: [PATCH] Tune release-blocking kind job resources down The kind release-blocking prowjobs have been ending up in error state over the past few week days, often due to pods being unable to schedule due to insufficient CPU. We've been seeing increased PR traffic due to v1.19 getting released, and a concerted attempt to clear the backlog of v1.20 PRs that were waiting for code thaw. The kind release-blocking jobs were the only jobs migrated to k8s-infra-prow-build that reserved 7300m CPU. The next highest cpu limit is 7, used by the verify presubmit and bazel-test periodic jobs. These jobs have not been ending up in error state. I was originally going to match the kind presubmit jobs, which use 4 cpu, but they have been failing a lot today during times I would correlate with high PR volume. I'm still tuning the memory to match based on graphs in the issue linked to this PR. --- .../kind/kind-release-blocking.yaml | 17 ++++++++--------- .../sig-release/release-branch-jobs/1.16.yaml | 16 ++++++++-------- .../sig-release/release-branch-jobs/1.17.yaml | 16 ++++++++-------- .../sig-release/release-branch-jobs/1.18.yaml | 16 ++++++++-------- .../sig-release/release-branch-jobs/1.19.yaml | 16 ++++++++-------- 5 files changed, 40 insertions(+), 41 deletions(-) diff --git a/config/jobs/kubernetes-sigs/kind/kind-release-blocking.yaml b/config/jobs/kubernetes-sigs/kind/kind-release-blocking.yaml index 8c93256e90a6..599ce6cc79a6 100644 --- a/config/jobs/kubernetes-sigs/kind/kind-release-blocking.yaml +++ b/config/jobs/kubernetes-sigs/kind/kind-release-blocking.yaml @@ -46,14 +46,13 @@ periodics: privileged: true resources: limits: - memory: 14Gi - cpu: 7300m + memory: 9Gi + cpu: 7 requests: # these are both a bit below peak usage during build # this is mostly for building kubernetes - memory: 14Gi - # during the tests more like 3-20m is used - cpu: 7300m + memory: 9Gi + cpu: 7 - interval: 1h cluster: k8s-infra-prow-build name: ci-kubernetes-kind-ipv6-e2e-parallel @@ -107,11 +106,11 @@ periodics: privileged: true resources: limits: - memory: 14Gi - cpu: 7300m + memory: 9Gi + cpu: 7 requests: # these are both a bit below peak usage during build # this is mostly for building kubernetes - memory: 14Gi + memory: 9Gi # during the tests more like 3-20m is used - cpu: 7300m + cpu: 7 diff --git a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.16.yaml b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.16.yaml index 54889cdb128e..323d6069d0a1 100644 --- a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.16.yaml +++ b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.16.yaml @@ -541,11 +541,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true - annotations: @@ -588,11 +588,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true postsubmits: diff --git a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.17.yaml b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.17.yaml index 6c841a33a488..fb35b231d05d 100644 --- a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.17.yaml +++ b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.17.yaml @@ -600,11 +600,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true - annotations: @@ -647,11 +647,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true postsubmits: diff --git a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.18.yaml b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.18.yaml index 36f36389113b..115539bb122a 100644 --- a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.18.yaml +++ b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.18.yaml @@ -655,11 +655,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true - annotations: @@ -709,11 +709,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true postsubmits: diff --git a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.19.yaml b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.19.yaml index 96ae285c7bd6..c55ad228e40f 100644 --- a/config/jobs/kubernetes/sig-release/release-branch-jobs/1.19.yaml +++ b/config/jobs/kubernetes/sig-release/release-branch-jobs/1.19.yaml @@ -605,11 +605,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true - annotations: @@ -659,11 +659,11 @@ periodics: name: "" resources: limits: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi requests: - cpu: 7300m - memory: 14Gi + cpu: 7 + memory: 9Gi securityContext: privileged: true postsubmits: