DRIVERS-3032 Add standalone clis for top level python scripts (#552)

.evergreen/config.yml

@@ -511,19 +511,19 @@ functions:
           cd ${PROJECT_DIRECTORY}
           make test
 
-  "run mongodl test partial":
+  "run cli test partial":
     - command: subprocess.exec
       type: test
       params:
         binary: bash
-        args: [src/.evergreen/tests/test-mongodl.sh, partial]
+        args: [src/.evergreen/tests/test-cli.sh, partial]
 
-  "run mongodl test full":
+  "run cli test full":
     - command: subprocess.exec
       type: test
       params:
         binary: bash
-        args: [src/.evergreen/tests/test-mongodl.sh]
+        args: [src/.evergreen/tests/test-cli.sh]
 
   "teardown assets":
     - command: subprocess.exec
@@ -974,15 +974,15 @@ tasks:
       commands:
         - func: "run install binaries test"
 
-    - name: "test-mongodl-full"
+    - name: "test-cli-full"
       tags: ["pr"]
       commands:
-        - func: "run mongodl test full"
+        - func: "run cli test full"
 
-    - name: "test-mongodl-partial"
+    - name: "test-cli-partial"
       tags: ["pr"]
       commands:
-        - func: "run mongodl test partial"
+        - func: "run cli test partial"
 
 # }}}
 
@@ -1421,7 +1421,7 @@ buildvariants:
       then:
         add_tasks:
           - "test-install-binaries"
-          - "test-mongodl-full"
+          - "test-cli-full"
           - "test-8.0-standalone-require-api"
 
 - matrix_name: "tests-os-requires-50"
@@ -1448,7 +1448,7 @@ buildvariants:
       then:
         add_tasks:
           - "test-install-binaries"
-          - "test-mongodl-partial"
+          - "test-cli-partial"
 
 - matrix_name: "tests-os-requires-70"
   matrix_spec: {"os-requires-70": "*", auth: "*", ssl: "*" }
@@ -1471,7 +1471,7 @@ buildvariants:
       then:
         add_tasks:
           - "test-install-binaries"
-          - "test-mongodl-partial"
+          - "test-cli-partial"
 
 # Storage Engine Tests on Ubuntu 20.04
 - matrix_name: "tests-storage-engines"

.evergreen/download-mongodb.sh

@@ -91,7 +91,7 @@ get_mongodb_download_url_for ()
    # shellcheck disable=SC3028
    _script_dir="$(dirname ${BASH_SOURCE:-$0})"
    _python3=$(bash -c ". $_script_dir/find-python3.sh && ensure_python3 2>/dev/null")
-   MONGOSH_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongosh-dl.py" --no-download | tr -d '\r')
+   MONGOSH_DOWNLOAD_URL=$($_python3 "${_script_dir}/mongosh_dl.py" --no-download | tr -d '\r')
 
    # Get the download url for MongoDB for the given version.
    MONGODB_DOWNLOAD_URL="$($_python3 "${_script_dir}/mongodl.py" --version $_VERSION --component $_component --no-download | tr -d '\r')"

.evergreen/install-cli.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Install the drivers orchestration scripts.
+
+set -eux
+
+if [ -z "$BASH" ]; then
+  echo "install-cli.sh must be run in a Bash shell!" 1>&2
+  return 1
+fi
+
+if [ -z "${1:-}" ]; then
+  echo "Must give a target directory!"
+fi
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/handle-paths.sh
+
+pushd $SCRIPT_DIR
+
+# Ensure pipx is writing assets to a contained location.
+export UV_CACHE_DIR=${DRIVERS_TOOLS}/.local/uv-cache
+export UV_TOOL_DIR=${DRIVERS_TOOLS}/.local/uv-tool
+
+. ./venv-utils.sh
+
+if [ ! -d $SCRIPT_DIR/venv ]; then
+
+  . ./find-python3.sh
+
+  echo "Ensuring python binary..."
+  PYTHON=$(ensure_python3 2>/dev/null)
+  echo "Ensuring python binary... done."
+
+  echo "Creating virtual environment 'venv'..."
+  venvcreate "${PYTHON:?}" venv
+  echo "Creating virtual environment 'venv'... done."
+
+  python -m pip install uv
+else
+  venvactivate venv
+fi
+
+pushd $1
+
+# On Windows, we have to do a bit of path manipulation.
+if [ "Windows_NT" == "${OS:-}" ]; then
+  TMP_DIR=$(cygpath -m "$(mktemp -d)")
+  PATH="$SCRIPT_DIR/venv/Scripts:$PATH"
+  UV_TOOL_BIN_DIR=${TMP_DIR} uv tool install --force --editable .
+  filenames=$(ls ${TMP_DIR})
+  for filename in $filenames; do
+    mv $TMP_DIR/$filename "$1/${filename//.exe/}"
+  done
+  rm -rf $TMP_DIR
+else
+  UV_TOOL_BIN_DIR=$(pwd) uv tool install --python "$(which python)" --force --editable .
+fi
+
+popd
+popd

.evergreen/mongodl.py

@@ -42,7 +42,6 @@
     Iterator,
     NamedTuple,
     Optional,
-    Sequence,
     cast,
 )
 
@@ -991,7 +990,7 @@ def _maybe_extract_member(
     return 1
 
 
-def main(argv: "Sequence[str]"):
+def main():
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
@@ -1103,7 +1102,7 @@ def main(argv: "Sequence[str]"):
         _print_list(
             cache.db, args.version, args.target, args.arch, args.edition, args.component
         )
-        return None
+        return
 
     if args.version is None:
         raise argparse.ArgumentError(None, 'A "--version" is required')
@@ -1140,9 +1139,8 @@ def main(argv: "Sequence[str]"):
         latest_build_branch=args.latest_build_branch,
     )
     if result is ExpandResult.Empty and args.empty_is_error:
-        return 1
-    return 0
+        sys.exit(1)
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
+    main()

.evergreen/mongosh-dl.py → .evergreen/mongosh_dl.py

@@ -15,7 +15,6 @@
 import tempfile
 import urllib.request
 from pathlib import Path
-from typing import Sequence
 
 HERE = Path(__file__).absolute().parent
 sys.path.insert(0, str(HERE))
@@ -111,7 +110,7 @@ def _download(
     return resp
 
 
-def main(argv: "Sequence[str]"):
+def main():
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
@@ -175,7 +174,7 @@ def main(argv: "Sequence[str]"):
         help="Do not extract or place any files/directories. "
         "Only print what will be extracted without placing any files.",
     )
-    args = parser.parse_args(argv)
+    args = parser.parse_args()
 
     if args.out is None and args.test is None and args.no_download is None:
         raise argparse.ArgumentError(None, 'A "--out" directory should be provided')
@@ -199,9 +198,8 @@ def main(argv: "Sequence[str]"):
         no_download=args.no_download,
     )
     if result is ExpandResult.Empty:
-        return 1
-    return 0
+        sys.exit(1)
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
+    main()

.evergreen/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["hatchling>=1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "drivers-evergreen-tools"
+version = "0.1.0"
+description = 'Scripts for drivers-evergreen-tools'
+requires-python = ">=3.8"
+license = "MIT"
+keywords = []
+authors = [
+  { name = "Steven Silvester", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: Implementation :: CPython",
+]
+dependencies = []
+
+[project.scripts]
+mongodl = "mongodl:main"
+mongosh-dl = "mongosh_dl:main"
+socks5srv = "socks5srv:main"
+
+[tool.hatch.build]
+include = ["mongodl.py", "mongosh_dl.py", "socks5srv.py"]

.evergreen/setup.sh

@@ -25,3 +25,6 @@ DRIVERS_TOOLS=$DRIVERS_TOOLS
 OS=${OS:-}
 PATH=$PATH
 EOF
+
+# Install the clis in this folder.
+bash $SCRIPT_DIR/install-cli.sh $SCRIPT_DIR

.evergreen/socks5srv.py

@@ -261,7 +261,7 @@ def raw_proxy(self, a, b):
                         a.sendall(buf)
 
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser(description="Start a Socks5 proxy server.")
     parser.add_argument("--port", type=int, required=True)
     parser.add_argument("--auth", type=str)
@@ -271,3 +271,7 @@ def raw_proxy(self, a, b):
     socketserver.TCPServer.allow_reuse_address = True
     with Socks5Server(("localhost", args.port), Socks5Handler, args) as server:
         server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

.evergreen/tests/test-cli.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Test mongodl and mongosh_dl.
+set -eux
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+
+pushd $SCRIPT_DIR/..
+
+bash install-cli.sh .
+DOWNLOAD_DIR=mongodl_test
+
+./socks5srv --help
+./mongodl --help
+./mongosh-dl --help
+
+# Make sure we can install again.
+bash install-cli.sh .
+
+if [ "${OS:-}" != "Windows_NT" ]; then
+  ./mongodl --edition enterprise --version 7.0 --component archive-debug --no-download
+else
+  DOWNLOAD_DIR=$(cygpath -m $DOWNLOAD_DIR)
+fi
+
+./mongodl --edition enterprise --version 7.0 --component archive --test
+./mongodl --edition enterprise --version 7.0 --component cryptd --out ${DOWNLOAD_DIR} --strip-path-components 1
+./mongosh-dl --no-download
+./mongosh-dl --version 2.1.1 --no-download
+
+export PATH="${DOWNLOAD_DIR}/bin:$PATH"
+if [ "${OS:-}" != "Windows_NT" ]; then
+  ./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1
+  chmod +x ./mongodl_test/bin/mongosh
+  ./mongodl_test/bin/mongosh --version
+else
+  ./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR} --strip-path-components 1
+fi
+
+if [ ${1:-} == "partial" ]; then
+  popd
+  make -C ${DRIVERS_TOOLS} test
+  exit 0
+fi
+
+# Ensure that all distros are accounted for in DISTRO_ID_TO_TARGET
+export VALIDATE_DISTROS=1
+./mongodl --list
+./mongodl --edition enterprise --version 7.0.6 --component archive --no-download
+./mongodl --edition enterprise --version 3.6 --component archive --test
+./mongodl --edition enterprise --version 4.0 --component archive --test
+./mongodl --edition enterprise --version 4.2 --component archive --test
+./mongodl --edition enterprise --version 4.4 --component archive --test
+./mongodl --edition enterprise --version 5.0 --component archive --test
+./mongodl --edition enterprise --version 6.0 --component crypt_shared --test
+./mongodl --edition enterprise --version 8.0 --component archive --test
+./mongodl --edition enterprise --version rapid --component archive --test
+./mongodl --edition enterprise --version latest --component archive --out ${DOWNLOAD_DIR}
+./mongodl --edition enterprise --version v6.0-perf --component cryptd --test
+./mongodl --edition enterprise --version v8.0-perf --component cryptd --test
+
+popd
+make -C ${DRIVERS_TOOLS} test