diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py b/.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py index b1af97029078e..03a58aee22b2f 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py @@ -162,9 +162,13 @@ def build_steps(self) -> List[BuildkiteTopLevelStep]: if v not in unsupported_python_versions ] - pytest_python_versions = sorted( - list(set(default_python_versions) - set(unsupported_python_versions)) - ) + pytest_python_versions = [ + AvailablePythonVersion(v) + for v in sorted( + set(e.value for e in default_python_versions) + - set(e.value for e in unsupported_python_versions) + ) + ] # Use highest supported python version if no defaults_match if len(pytest_python_versions) == 0: pytest_python_versions = [supported_python_versions[-1]] diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py b/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py index 3d684cba0a119..68b27a661340e 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py @@ -5,7 +5,7 @@ from dagster_buildkite.utils import is_release_branch, safe_getenv -class AvailablePythonVersion(str, Enum): +class AvailablePythonVersion(Enum): # Ordering is important here, because some steps will take the highest/lowest available version. V3_9 = "3.9" V3_10 = "3.10" @@ -60,6 +60,6 @@ def from_major_minor(cls, major: int, minor: int) -> "AvailablePythonVersion": @classmethod def to_tox_factor(cls, version: "AvailablePythonVersion") -> str: - ver_parts = version.split(".") + ver_parts = version.value.split(".") major, minor = ver_parts[0], ver_parts[1] return f"py{major}{minor}" diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py index 74c27cd8e0497..98ea3ec22004e 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py @@ -6,7 +6,11 @@ GCP_CREDS_LOCAL_FILE, LATEST_DAGSTER_RELEASE, ) -from dagster_buildkite.package_spec import PackageSpec, UnsupportedVersionsFunction +from dagster_buildkite.package_spec import ( + PackageSpec, + PytestExtraCommandsFunction, + UnsupportedVersionsFunction, +) from dagster_buildkite.python_version import AvailablePythonVersion from dagster_buildkite.step_builder import BuildkiteQueue from dagster_buildkite.steps.test_project import test_project_depends_fn @@ -60,12 +64,12 @@ def build_backcompat_suite_steps() -> List[BuildkiteTopLevelStep]: ) -def backcompat_extra_cmds(_, factor: str) -> List[str]: +def backcompat_extra_cmds(_, factor: Optional[str]) -> List[str]: tox_factor_map = { "user-code-latest-release": LATEST_DAGSTER_RELEASE, "user-code-earliest-release": EARLIEST_TESTED_RELEASE, } - + assert factor webserver_version = DAGSTER_CURRENT_BRANCH webserver_library_version = _get_library_version(webserver_version) user_code_version = tox_factor_map[factor] @@ -163,7 +167,7 @@ def build_auto_materialize_perf_suite_steps(): def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _): return [ - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', "pushd integration_tests/test_suites/daemon-test-suite/monitoring_daemon_tests/", "docker-compose up -d --remove-orphans", @@ -182,7 +186,7 @@ def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _): # ######################## -def build_k8s_suite_steps(): +def build_k8s_suite_steps() -> List[BuildkiteTopLevelStep]: pytest_tox_factors = ["-default", "-subchart"] directory = os.path.join("integration_tests", "test_suites", "k8s-test-suite") return build_integration_suite_steps( @@ -201,7 +205,7 @@ def build_k8s_suite_steps(): def build_integration_suite_steps( directory: str, pytest_tox_factors: Optional[List[str]], - pytest_extra_cmds: Optional[Callable] = None, + pytest_extra_cmds: Optional[PytestExtraCommandsFunction] = None, queue=None, always_run_if: Optional[Callable[[], bool]] = None, unsupported_python_versions: Optional[ @@ -229,19 +233,19 @@ def build_integration_suite_steps( ).build_steps() -def k8s_integration_suite_pytest_extra_cmds(version: str, _) -> List[str]: +def k8s_integration_suite_pytest_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return [ - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', "aws ecr get-login --no-include-email --region us-west-2 | sh", ] -def celery_k8s_integration_suite_pytest_extra_cmds(version: str, _) -> List[str]: +def celery_k8s_integration_suite_pytest_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: cmds = [ 'export AIRFLOW_HOME="/airflow"', "mkdir -p $${AIRFLOW_HOME}", - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', "aws ecr get-login --no-include-email --region us-west-2 | sh", ] diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index 022d178363460..ddc6291ff301f 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -92,7 +92,7 @@ def _get_uncustomized_pkg_roots(root: str, custom_pkg_roots: List[str]) -> List[ # ######################## -def airflow_extra_cmds(version: str, _) -> List[str]: +def airflow_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return [ 'export AIRFLOW_HOME="/airflow"', "mkdir -p $${AIRFLOW_HOME}", @@ -160,9 +160,9 @@ def dagster_graphql_extra_cmds(_, tox_factor: Optional[str]) -> List[str]: ] -def celery_extra_cmds(version: str, _) -> List[str]: +def celery_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return [ - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', "pushd python_modules/libraries/dagster-celery", # Run the rabbitmq db. We are in docker running docker @@ -178,7 +178,7 @@ def celery_extra_cmds(version: str, _) -> List[str]: ] -def celery_docker_extra_cmds(version: str, _) -> List[str]: +def celery_docker_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return celery_extra_cmds(version, _) + [ "pushd python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/", "docker-compose up -d --remove-orphans", @@ -192,9 +192,9 @@ def celery_docker_extra_cmds(version: str, _) -> List[str]: ] -def docker_extra_cmds(version: str, _) -> List[str]: +def docker_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return [ - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', "pushd python_modules/libraries/dagster-docker/dagster_docker_tests/", "docker-compose up -d --remove-orphans", @@ -230,9 +230,9 @@ def docker_extra_cmds(version: str, _) -> List[str]: ] -def k8s_extra_cmds(version: str, _) -> List[str]: +def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: return [ - "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version, + "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value, 'export DAGSTER_DOCKER_REPOSITORY="$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com"', ] diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/test_project.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/test_project.py index fe97134dc4106..bbdcd26bf43a5 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/test_project.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/test_project.py @@ -49,16 +49,16 @@ def build_test_project_steps() -> List[GroupStep]: " $${GOOGLE_APPLICATION_CREDENTIALS}", "export" " BASE_IMAGE=$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/test-project-base:py" - + version + + version.value + "-" + TEST_PROJECT_BASE_IMAGE_VERSION, # build and tag test image "export" " TEST_PROJECT_IMAGE=$${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/test-project:$${BUILDKITE_BUILD_ID}-" - + version, + + version.value, "git config --global --add safe.directory /workdir", "./python_modules/dagster-test/dagster_test/test_project/build.sh " - + version + + version.value + " $${TEST_PROJECT_IMAGE}", # # push the built image @@ -67,7 +67,7 @@ def build_test_project_steps() -> List[GroupStep]: ) .on_python_image( # py version can be bumped when rebuilt - f"buildkite-build-test-project-image:py{AvailablePythonVersion.V3_11}-{BUILDKITE_BUILD_TEST_PROJECT_IMAGE_IMAGE_VERSION}", + f"buildkite-build-test-project-image:py{AvailablePythonVersion.V3_11.value}-{BUILDKITE_BUILD_TEST_PROJECT_IMAGE_IMAGE_VERSION}", [ "AIRFLOW_HOME", "AWS_ACCOUNT_ID",