Run python linter in docker

This is part of effort for build/test without bazel. Running in docker for ensuring toolchain consistency.

.gitignore

@@ -8,6 +8,8 @@
 /_output
 /_bin
 node_modules/
+.python_deps
+.python_virtual_env
 .DS_Store
 # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA
 .idea/

config/jobs/kubernetes/kops/build_jobs.py

@@ -16,8 +16,8 @@
 import math
 import json
 import re
-import jinja2 # pylint: disable=import-error
 import yaml
+import jinja2 # pylint: disable=import-error
 
 
 from helpers import ( # pylint: disable=import-error, no-name-in-module

hack/build/ensure-python_deps.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script ensures node_modules is up to date with yarn in docker
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+cd "${REPO_ROOT}"
+
+# Ensure virtual env
+# Trick from https://pythonspeed.com/articles/activate-virtualenv-dockerfile/
+echo "Ensure virtual env"
+export VIRTUAL_ENV="${REPO_ROOT}/.python_virtual_env"
+hack/run-in-python-container.sh \
+    python3 -m venv "${VIRTUAL_ENV}"
+
+echo "Installing requirements3.txt"
+hack/run-in-python-container.sh \
+    pip3 install -r requirements3.txt

hack/make-rules/verify/all.sh

@@ -60,6 +60,11 @@ if [[ "${VERIFY_TSLINT:-true}" == "true" ]]; then
   hack/make-rules/verify/tslint.sh || res=1
   cd "${REPO_ROOT}"
 fi
+if [[ "${VERIFY_PYLINT:-true}" == "true" ]]; then
+  echo "verifying pylint ..."
+  hack/make-rules/verify/pylint.sh || res=1
+  cd "${REPO_ROOT}"
+fi
 
 # exit based on verify scripts
 if [[ "${res}" = 0 ]]; then

hack/make-rules/verify/pylint.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script runs tslint against the repo using docker
+#
+# TODO: we should be using eslint instead now, but this repo is also
+# on a rather old version of node, first we get the build system migrated.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
+cd "${REPO_ROOT}"
+
+# ensure deps are installed
+source hack/build/ensure-python_deps.sh
+
+echo "Grep files to inspect"
+shopt -s extglob globstar
+# files_to_inspect="$( ls !(gubernator|external|vendor|jenkins|scenarios|triage|boskos|bazel-*)/**/*.py | grep -v analyze-memory-profiles )"
+# echo "Files to inspect: ${files_to_inspect}"
+echo "Linting"
+hack/run-in-python-container.sh \
+    python3 ./hack/pylint_bin.py $( ls !(gubernator|external|vendor|jenkins|scenarios|triage|boskos|bazel-*)/**/*.py | grep -v analyze-memory-profiles )

hack/run-in-python-container.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs $@ in a python container
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "${REPO_ROOT}"
+
+PY_IMAGE='python:3.9-slim-buster'
+PYLINTHOME="$(mktemp -d)"
+
+# Technically we can assume that this is always set, setting here again to
+# ensure consistency
+VIRTUAL_ENV="${VIRTUAL_ENV:-${REPO_ROOT:?}/.python_virtual_env}"
+
+# Setting PATH is a trick learnt from
+# https://pythonspeed.com/articles/activate-virtualenv-dockerfile/, for
+# mimicking the behavior of `source venv_dir/bin/activate`, same as setting
+# PYTHONPATH to empty. Tried several other alternatives:
+#   - Set PATH in docker image. This doesn't work because the workspace is set
+#     afterwards.
+#   - Run `source venv_dir/bin/activate` as part of docker run argument. This
+#     cannot be done directly as `source` is a bash builtin, so have to do `bash
+#     -c `source.., python3..` stuff, and the quotes passing around almost
+#     killed me.
+docker run \
+    --rm -i \
+    -e HOME=/tmp \
+    -e PYLINTHOME=${PYLINTHOME} \
+    -e VIRTUAL_ENV="${VIRTUAL_ENV}" \
+    -e PATH="${VIRTUAL_ENV}/bin:/usr/local/bin:/bin" \
+    -e PYTHONPATH='' \
+    -v "${PYLINTHOME}:${PYLINTHOME}" \
+    -v "${REPO_ROOT:?}:${REPO_ROOT:?}" -w "${REPO_ROOT}" \
+    "${PY_IMAGE}" \
+    "$@"

kettle/stream_test.py

@@ -18,12 +18,12 @@
 
 import unittest
 
+from parameterized import parameterized
+
 import stream
 import make_db_test
 import model
 
-from parameterized import parameterized
-
 class FakePullResponse:
     def __init__(self, messages):
         self.received_messages = messages