feat(ingest): detect old or missing docker compose (datahub-project#6466

)

docs/quickstart.md

@@ -5,9 +5,9 @@
 To deploy a new instance of DataHub, perform the following steps.
 
 
-1. Install Docker for your platform.
+1. Install Docker and Docker Compose v2 for your platform.
 - On Windows or Mac, install [Docker Desktop](https://www.docker.com/products/docker-desktop/).
-- On Linux, install [Docker for Linux](https://docs.docker.com/desktop/install/linux-install/).
+- On Linux, install [Docker for Linux](https://docs.docker.com/desktop/install/linux-install/) and [Docker Compose](https://docs.docker.com/compose/install/linux/).
 
 :::note
 

metadata-ingestion/src/datahub/cli/docker_check.py

@@ -46,12 +46,16 @@
 def get_client_with_error() -> Iterator[
     Tuple[docker.DockerClient, Optional[Exception]]
 ]:
+    # This method is structured somewhat strangely because we
+    # need to make sure that we only yield once.
+
     docker_cli = None
     try:
         docker_cli = docker.from_env()
     except docker.errors.DockerException as error:
         try:
-            # newer docker versions create the socket in a user directory, try that before giving up
+            # Docker Desktop 4.13.0 broke the docker.sock symlink.
+            # See https://github.com/docker/docker-py/issues/3059.
             maybe_sock_path = os.path.expanduser("~/.docker/run/docker.sock")
             if os.path.exists(maybe_sock_path):
                 docker_cli = docker.DockerClient(base_url=f"unix://{maybe_sock_path}")
@@ -62,9 +66,14 @@ def get_client_with_error() -> Iterator[
 
     if docker_cli is not None:
         try:
-            yield docker_cli, None
-        finally:
-            docker_cli.close()
+            docker_cli.ping()
+        except docker.errors.DockerException as error:
+            yield None, error
+        else:
+            try:
+                yield docker_cli, None
+            finally:
+                docker_cli.close()
 
 
 def memory_in_gb(mem_bytes: int) -> float:

metadata-ingestion/src/datahub/cli/docker_cli.py

@@ -11,13 +11,15 @@
 import time
 from enum import Enum
 from pathlib import Path
-from typing import Dict, List, NoReturn, Optional
+from typing import Dict, List, NoReturn, Optional, Union
 
 import click
+import click_spinner
 import pydantic
 import requests
 from expandvars import expandvars
 from requests_file import FileAdapter
+from typing_extensions import Literal
 
 from datahub.cli.cli_utils import DATAHUB_ROOT_FOLDER
 from datahub.cli.docker_check import (
@@ -221,6 +223,44 @@ def _get_default_quickstart_compose_file() -> Optional[str]:
     return None
 
 
+def _docker_compose_v2() -> Union[List[str], Literal[False]]:
+    try:
+        # Check for the docker compose v2 plugin.
+        assert (
+            subprocess.check_output(
+                ["docker", "compose", "version", "--short"], stderr=subprocess.STDOUT
+            )
+            .decode()
+            .startswith("2.")
+        )
+        return ["docker", "compose"]
+    except (OSError, subprocess.CalledProcessError, AssertionError):
+        # We'll check for docker-compose as well.
+        try:
+            compose_version = subprocess.check_output(
+                ["docker-compose", "version", "--short"], stderr=subprocess.STDOUT
+            ).decode()
+            if compose_version.startswith("2."):
+                # This will happen if docker compose v2 is installed in standalone mode
+                # instead of as a plugin.
+                return ["docker-compose"]
+
+            click.secho(
+                "You have docker-compose v1 installed, but we require Docker Compose v2. "
+                "Please upgrade to Docker Compose v2. "
+                "See https://docs.docker.com/compose/compose-v2/ for more information.",
+                fg="red",
+            )
+            return False
+        except (OSError, subprocess.CalledProcessError):
+            # docker-compose v1 is not installed either.
+            click.secho(
+                "You don't have Docker Compose installed. Please install Docker Compose. See https://docs.docker.com/compose/install/.",
+                fg="red",
+            )
+            return False
+
+
 def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None:
     default_quickstart_compose_file = _get_default_quickstart_compose_file()
     compose_files_for_stopping = (
@@ -232,9 +272,11 @@ def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None:
     )
     if compose_files_for_stopping:
         # docker-compose stop
+        compose = _docker_compose_v2()
+        if not compose:
+            return
         base_command: List[str] = [
-            "docker",
-            "compose",
+            *compose,
             *itertools.chain.from_iterable(
                 ("-f", f"{path}") for path in compose_files_for_stopping
             ),
@@ -695,9 +737,11 @@ def quickstart(
         elastic_port=elastic_port,
     )
 
+    compose = _docker_compose_v2()
+    if not compose:
+        return
     base_command: List[str] = [
-        "docker",
-        "compose",
+        *compose,
         *itertools.chain.from_iterable(
             ("-f", f"{path}") for path in quickstart_compose_file
         ),
@@ -709,11 +753,15 @@ def quickstart(
     try:
         if pull_images:
             click.echo("Pulling docker images...")
-            subprocess.run(
-                [*base_command, "pull", "-q"],
-                check=True,
-                env=_docker_subprocess_env(),
-            )
+            # FIXME: Remove args once https://github.com/python/typeshed/pull/9220 is merged.
+            with click_spinner.spinner(
+                beep=False, disable=False, force=False, stream=sys.stdout
+            ):
+                subprocess.run(
+                    [*base_command, "pull", "-q"],
+                    check=True,
+                    env=_docker_subprocess_env(),
+                )
             click.secho("Finished pulling docker images!")
     except subprocess.CalledProcessError:
         click.secho(

metadata-ingestion/tests/integration/sql_server/test_sql_server.py

@@ -22,8 +22,7 @@ def mssql_runner(docker_compose_runner, pytestconfig):
         time.sleep(5)
 
         # Run the setup.sql file to populate the database.
-        docker = "docker"
-        command = f"{docker} exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql"
+        command = "docker exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql"
         ret = subprocess.run(
             command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
         )