diff --git a/docs/quickstart.md b/docs/quickstart.md index 905da3001bbed..9fc647b36aa35 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -5,9 +5,9 @@ To deploy a new instance of DataHub, perform the following steps. -1. Install Docker for your platform. +1. Install Docker and Docker Compose v2 for your platform. - On Windows or Mac, install [Docker Desktop](https://www.docker.com/products/docker-desktop/). -- On Linux, install [Docker for Linux](https://docs.docker.com/desktop/install/linux-install/). +- On Linux, install [Docker for Linux](https://docs.docker.com/desktop/install/linux-install/) and [Docker Compose](https://docs.docker.com/compose/install/linux/). :::note diff --git a/metadata-ingestion/src/datahub/cli/docker_check.py b/metadata-ingestion/src/datahub/cli/docker_check.py index f5c01d10d18f1..bc2ab4d78f3a3 100644 --- a/metadata-ingestion/src/datahub/cli/docker_check.py +++ b/metadata-ingestion/src/datahub/cli/docker_check.py @@ -46,12 +46,16 @@ def get_client_with_error() -> Iterator[ Tuple[docker.DockerClient, Optional[Exception]] ]: + # This method is structured somewhat strangely because we + # need to make sure that we only yield once. + docker_cli = None try: docker_cli = docker.from_env() except docker.errors.DockerException as error: try: - # newer docker versions create the socket in a user directory, try that before giving up + # Docker Desktop 4.13.0 broke the docker.sock symlink. + # See https://github.com/docker/docker-py/issues/3059. maybe_sock_path = os.path.expanduser("~/.docker/run/docker.sock") if os.path.exists(maybe_sock_path): docker_cli = docker.DockerClient(base_url=f"unix://{maybe_sock_path}") @@ -62,9 +66,14 @@ def get_client_with_error() -> Iterator[ if docker_cli is not None: try: - yield docker_cli, None - finally: - docker_cli.close() + docker_cli.ping() + except docker.errors.DockerException as error: + yield None, error + else: + try: + yield docker_cli, None + finally: + docker_cli.close() def memory_in_gb(mem_bytes: int) -> float: diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index e89a864febf0a..8a939fc419572 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -11,13 +11,15 @@ import time from enum import Enum from pathlib import Path -from typing import Dict, List, NoReturn, Optional +from typing import Dict, List, NoReturn, Optional, Union import click +import click_spinner import pydantic import requests from expandvars import expandvars from requests_file import FileAdapter +from typing_extensions import Literal from datahub.cli.cli_utils import DATAHUB_ROOT_FOLDER from datahub.cli.docker_check import ( @@ -221,6 +223,44 @@ def _get_default_quickstart_compose_file() -> Optional[str]: return None +def _docker_compose_v2() -> Union[List[str], Literal[False]]: + try: + # Check for the docker compose v2 plugin. + assert ( + subprocess.check_output( + ["docker", "compose", "version", "--short"], stderr=subprocess.STDOUT + ) + .decode() + .startswith("2.") + ) + return ["docker", "compose"] + except (OSError, subprocess.CalledProcessError, AssertionError): + # We'll check for docker-compose as well. + try: + compose_version = subprocess.check_output( + ["docker-compose", "version", "--short"], stderr=subprocess.STDOUT + ).decode() + if compose_version.startswith("2."): + # This will happen if docker compose v2 is installed in standalone mode + # instead of as a plugin. + return ["docker-compose"] + + click.secho( + "You have docker-compose v1 installed, but we require Docker Compose v2. " + "Please upgrade to Docker Compose v2. " + "See https://docs.docker.com/compose/compose-v2/ for more information.", + fg="red", + ) + return False + except (OSError, subprocess.CalledProcessError): + # docker-compose v1 is not installed either. + click.secho( + "You don't have Docker Compose installed. Please install Docker Compose. See https://docs.docker.com/compose/install/.", + fg="red", + ) + return False + + def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None: default_quickstart_compose_file = _get_default_quickstart_compose_file() compose_files_for_stopping = ( @@ -232,9 +272,11 @@ def _attempt_stop(quickstart_compose_file: List[pathlib.Path]) -> None: ) if compose_files_for_stopping: # docker-compose stop + compose = _docker_compose_v2() + if not compose: + return base_command: List[str] = [ - "docker", - "compose", + *compose, *itertools.chain.from_iterable( ("-f", f"{path}") for path in compose_files_for_stopping ), @@ -695,9 +737,11 @@ def quickstart( elastic_port=elastic_port, ) + compose = _docker_compose_v2() + if not compose: + return base_command: List[str] = [ - "docker", - "compose", + *compose, *itertools.chain.from_iterable( ("-f", f"{path}") for path in quickstart_compose_file ), @@ -709,11 +753,15 @@ def quickstart( try: if pull_images: click.echo("Pulling docker images...") - subprocess.run( - [*base_command, "pull", "-q"], - check=True, - env=_docker_subprocess_env(), - ) + # FIXME: Remove args once https://github.com/python/typeshed/pull/9220 is merged. + with click_spinner.spinner( + beep=False, disable=False, force=False, stream=sys.stdout + ): + subprocess.run( + [*base_command, "pull", "-q"], + check=True, + env=_docker_subprocess_env(), + ) click.secho("Finished pulling docker images!") except subprocess.CalledProcessError: click.secho( diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py index c1004f2a18b2e..3e7b75edd4878 100644 --- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py +++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py @@ -22,8 +22,7 @@ def mssql_runner(docker_compose_runner, pytestconfig): time.sleep(5) # Run the setup.sql file to populate the database. - docker = "docker" - command = f"{docker} exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql" + command = "docker exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql" ret = subprocess.run( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE )