Also move out dependency installer functions

dev-support/ci/selective_ci_checks.sh

@@ -247,6 +247,7 @@ function get_count_doc_files() {
     local pattern_array=(
         "^hadoop-hdds/docs"
         "^hadoop-ozone/dev-support/checks/docs.sh"
+        "^hadoop-ozone/dev-support/checks/install/hugo.sh"
     )
     filter_changed_files true
     COUNT_DOC_CHANGED_FILES=${match_count}
@@ -278,6 +279,8 @@ function get_count_kubernetes_files() {
     start_end::group_start "Count kubernetes files"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/kubernetes.sh"
+        "^hadoop-ozone/dev-support/checks/install/flekszible.sh"
+        "^hadoop-ozone/dev-support/checks/install/k3s.sh"
         "^hadoop-ozone/dist"
     )
     local ignore_array=(
@@ -356,7 +359,7 @@ function check_needs_bats() {
     local pattern_array=(
         "\.bash$"
         "\.bats$"
-        "\.sh$" # includes hadoop-ozone/dev-support/checks/bats.sh
+        "\.sh$" # includes hadoop-ozone/dev-support/checks/bats.sh and hadoop-ozone/dev-support/checks/install/bats.sh
     )
     filter_changed_files
 
@@ -416,6 +419,7 @@ function check_needs_findbugs() {
     start_end::group_start "Check if findbugs is needed"
     local pattern_array=(
         "^hadoop-ozone/dev-support/checks/findbugs.sh"
+        "^hadoop-ozone/dev-support/checks/install/spotbugs.sh"
         "findbugsExcludeFile.xml"
         "pom.xml"
         "src/..../java"

hadoop-ozone/dev-support/checks/_lib.sh

@@ -63,106 +63,3 @@ _install_tool() {
     fi
   fi
 }
-
-install_bats() {
-  _install_tool bats "bats-core-${BATS_VERSION}/bin"
-}
-
-_install_bats() {
-  curl -LSs "https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz" | tar -xz -f -
-}
-
-install_k3s() {
-  _install_tool k3s
-}
-
-_install_k3s() {
-  curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="${K3S_VERSION}" sh -
-  sudo chmod a+r $KUBECONFIG
-}
-
-install_flekszible() {
-  _install_tool flekszible bin
-}
-
-_install_flekszible() {
-  mkdir bin
-
-  local os=$(uname -s)
-  local arch=$(uname -m)
-
-  curl -LSs "https://github.com/elek/flekszible/releases/download/v${FLEKSZIBLE_VERSION}/flekszible_${FLEKSZIBLE_VERSION}_${os}_${arch}.tar.gz" | tar -xz -f - -C bin
-
-  chmod +x bin/flekszible
-}
-
-install_hugo() {
-  _install_tool hugo bin
-}
-
-_install_hugo() {
-  local os=$(uname -s)
-  local arch=$(uname -m)
-
-  mkdir bin
-
-  case "${os}" in
-    Darwin)
-      os=macOS
-      ;;
-  esac
-
-  case "${arch}" in
-    x86_64)
-      arch=64bit
-      ;;
-  esac
-
-  curl -LSs "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_${os}-${arch}.tar.gz" | tar -xz -f - -C bin hugo
-  chmod +x bin/hugo
-}
-
-install_virtualenv() {
-  _install_tool virtualenv
-}
-
-_install_virtualenv() {
-  sudo pip3 install virtualenv
-}
-
-install_robot() {
-  _install_tool robot venv/bin
-}
-
-_install_robot() {
-  virtualenv venv
-  source venv/bin/activate
-  pip install robotframework
-}
-
-install_spotbugs() {
-  _install_tool spotbugs "spotbugs-${SPOTBUGS_VERSION}/bin"
-}
-
-_install_spotbugs() {
-  curl -LSs "https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz" | tar -xz -f -
-}
-
-download_hadoop_aws() {
-  local dir="$1"
-
-  if [[ -z ${dir} ]]; then
-    echo "Required argument: target directory for Hadoop AWS sources" >&2
-    return 1
-  fi
-
-  if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then
-    mkdir -p "${dir}"
-    if [[ ! -f "${dir}.tar.gz" ]]; then
-      local url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz"
-      echo "Downloading Hadoop from ${url}"
-      curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1
-    fi
-    tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1
-  fi
-}

hadoop-ozone/dev-support/checks/acceptance.sh

@@ -50,6 +50,25 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
     export HADOOP_AWS_DIR=${OZONE_ROOT}/target/hadoop-src
   fi
 
+  download_hadoop_aws() {
+    local dir="$1"
+
+    if [[ -z ${dir} ]]; then
+      echo "Required argument: target directory for Hadoop AWS sources" >&2
+      return 1
+    fi
+
+    if [[ ! -e "${dir}" ]] || [[ ! -d "${dir}"/src/test/resources ]]; then
+      mkdir -p "${dir}"
+      if [[ ! -f "${dir}.tar.gz" ]]; then
+        local url="https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}-src.tar.gz"
+        echo "Downloading Hadoop from ${url}"
+        curl -LSs --fail -o "${dir}.tar.gz" "$url" || return 1
+      fi
+      tar -x -z -C "${dir}" --strip-components=3 -f "${dir}.tar.gz" --wildcards 'hadoop-*-src/hadoop-tools/hadoop-aws' || return 1
+    fi
+  }
+
   if ! download_hadoop_aws "${HADOOP_AWS_DIR}"; then
     echo "Failed to download Hadoop ${HADOOP_VERSION}" > "${REPORT_FILE}"
     exit 1

hadoop-ozone/dev-support/checks/bats.sh

@@ -21,14 +21,8 @@ set -u -o pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "${DIR}/../../.." || exit 1
 
-: ${BATS_VERSION:="1.2.1"}
-
 source "${DIR}/_lib.sh"
-
-install_bats
-
-git clone https://github.com/bats-core/bats-assert dev-support/ci/bats-assert
-git clone https://github.com/bats-core/bats-support dev-support/ci/bats-support
+source "${DIR}/install/bats.sh"
 
 REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/bats"}
 mkdir -p "${REPORT_DIR}"

hadoop-ozone/dev-support/checks/docs.sh

@@ -21,10 +21,8 @@ set -u -o pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "${DIR}/../../.." || exit 1
 
-: ${HUGO_VERSION:=0.83.1}
-
 source "${DIR}/_lib.sh"
-install_hugo
+source "${DIR}/install/hugo.sh"
 
 REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/docs"}
 mkdir -p "${REPORT_DIR}"

hadoop-ozone/dev-support/checks/findbugs.sh

@@ -22,11 +22,9 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
 : ${OZONE_WITH_COVERAGE:="false"}
-: ${SPOTBUGS_VERSION:=3.1.12}
 
 source "${DIR}/_lib.sh"
-
-install_spotbugs
+source "${DIR}/install/spotbugs.sh"
 
 REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
 mkdir -p "$REPORT_DIR"

hadoop-ozone/dev-support/checks/install/bats.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs bats.
+# Requires _install_tool from _lib.sh.  Use `source` for both scripts, because it modifies $PATH.
+
+: ${BATS_VERSION:="1.2.1"}
+
+_install_bats() {
+  curl -LSs "https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz" | tar -xz -f -
+}
+
+_install_tool bats "bats-core-${BATS_VERSION}/bin"
+
+git clone https://github.com/bats-core/bats-assert dev-support/ci/bats-assert
+git clone https://github.com/bats-core/bats-support dev-support/ci/bats-support

hadoop-ozone/dev-support/checks/install/flekszible.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs Flekszible.
+# Requires _install_tool from _lib.sh.  Use `source` for both scripts, because it modifies $PATH.
+
+: ${FLEKSZIBLE_VERSION:="2.3.0"}
+
+_install_flekszible() {
+  mkdir bin
+
+  local os=$(uname -s)
+  local arch=$(uname -m)
+
+  curl -LSs "https://github.com/elek/flekszible/releases/download/v${FLEKSZIBLE_VERSION}/flekszible_${FLEKSZIBLE_VERSION}_${os}_${arch}.tar.gz" | tar -xz -f - -C bin
+
+  chmod +x bin/flekszible
+}
+
+_install_tool flekszible bin

hadoop-ozone/dev-support/checks/install/hugo.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs Hugo.
+# Requires _install_tool from _lib.sh.  Use `source` for both scripts, because it modifies $PATH.
+
+: ${HUGO_VERSION:=0.83.1}
+
+_install_hugo() {
+  local os=$(uname -s)
+  local arch=$(uname -m)
+
+  mkdir bin
+
+  case "${os}" in
+    Darwin)
+      os=macOS
+      ;;
+  esac
+
+  case "${arch}" in
+    x86_64)
+      arch=64bit
+      ;;
+  esac
+
+  curl -LSs "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_${os}-${arch}.tar.gz" | tar -xz -f - -C bin hugo
+  chmod +x bin/hugo
+}
+
+_install_tool hugo bin

hadoop-ozone/dev-support/checks/install/k3s.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs K3S.
+# Requires _install_tool from _lib.sh.  Use `source` for both scripts, because it modifies $PATH.
+
+: ${K3S_VERSION:="v1.21.2+k3s1"}
+: ${KUBECONFIG:=/etc/rancher/k3s/k3s.yaml}
+
+_install_k3s() {
+  curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="${K3S_VERSION}" sh -
+  sudo chmod a+r $KUBECONFIG
+}
+
+_install_tool k3s

hadoop-ozone/dev-support/checks/install/spotbugs.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs SpotBugs.
+# Requires _install_tool from _lib.sh.  Use `source` for both scripts, because it modifies $PATH.
+
+: ${SPOTBUGS_VERSION:=3.1.12}
+
+_install_spotbugs() {
+  curl -LSs "https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz" | tar -xz -f -
+}
+
+_install_tool spotbugs "spotbugs-${SPOTBUGS_VERSION}/bin"