From 15980708998bf1cd431a0272eb25fd4359f764cb Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 25 Mar 2024 18:46:57 -0700 Subject: [PATCH] fix(ci): simplify python release process (#10133) --- .github/workflows/documentation.yml | 6 +----- docs-website/build.gradle | 11 +++-------- .../airflow-plugin/build.gradle | 2 +- .../airflow-plugin/scripts/release.sh | 2 +- .../dagster-plugin/build.gradle | 2 +- .../dagster-plugin/scripts/release.sh | 4 ++-- metadata-ingestion/scripts/release.sh | 8 +++++--- vercel.json | 2 +- 8 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c8552a8b85e90f..ab7890ad5b44f9 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -47,11 +47,7 @@ jobs: run: ./metadata-ingestion/scripts/install_deps.sh - name: Build Docs run: | - ./gradlew --info docs-website:build \ - -x :metadata-ingestion-modules:airflow-plugin:build \ - -x :metadata-ingestion-modules:airflow-plugin:check \ - -x :metadata-ingestion-modules:dagster-plugin:build \ - -x :metadata-ingestion-modules:dagster-plugin:check \ + ./gradlew --info docs-website:build - name: Deploy if: github.event_name == 'push' && github.repository == 'datahub-project/datahub' diff --git a/docs-website/build.gradle b/docs-website/build.gradle index d1fdc6dc83365f..f3bedd2516319a 100644 --- a/docs-website/build.gradle +++ b/docs-website/build.gradle @@ -79,19 +79,14 @@ task yarnInstall(type: YarnTask) { ) outputs.dir('node_modules') } -task airflowPluginBuild(dependsOn: [':metadata-ingestion-modules:airflow-plugin:buildWheel']) { -} - -// The Dagster plugin build and airflow plugin build can't be built at the same time; otherwise, it will raise -// fatal: Unable to create '/home/runner/work/datahub/datahub/.git/index.lock': File exists.. on CI -task dagsterPluginBuild(dependsOn: [':metadata-ingestion-modules:dagster-plugin:buildWheel', airflowPluginBuild]) { -} task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall, generateGraphQLSchema, generateJsonSchema, ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen', ':metadata-ingestion:buildWheel', - airflowPluginBuild, dagsterPluginBuild] ) { + ':metadata-ingestion-modules:airflow-plugin:buildWheel', + ':metadata-ingestion-modules:dagster-plugin:buildWheel', + ]) { inputs.files(projectMdFiles) outputs.cacheIf { true } args = ['run', 'generate'] diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 350c43b2568084..fd7ff8a776a902 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -106,7 +106,7 @@ task cleanPythonCache(type: Exec) { commandLine 'bash', '-c', "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete" } -task buildWheel(type: Exec, dependsOn: [install, cleanPythonCache]) { +task buildWheel(type: Exec, dependsOn: [environmentSetup, cleanPythonCache]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' diff --git a/metadata-ingestion-modules/airflow-plugin/scripts/release.sh b/metadata-ingestion-modules/airflow-plugin/scripts/release.sh index 5667e761ea5581..e9c2896c3aba2a 100755 --- a/metadata-ingestion-modules/airflow-plugin/scripts/release.sh +++ b/metadata-ingestion-modules/airflow-plugin/scripts/release.sh @@ -23,4 +23,4 @@ python -m build if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then python -m twine upload 'dist/*' fi -git restore src/${MODULE}/__init__.py +mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle index 163e0e6738b4d3..f6822a0f2fcb67 100644 --- a/metadata-ingestion-modules/dagster-plugin/build.gradle +++ b/metadata-ingestion-modules/dagster-plugin/build.gradle @@ -109,7 +109,7 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" } -task buildWheel(type: Exec, dependsOn: [install]) { +task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' } diff --git a/metadata-ingestion-modules/dagster-plugin/scripts/release.sh b/metadata-ingestion-modules/dagster-plugin/scripts/release.sh index 30219956534d99..3f55f5cefe0da9 100755 --- a/metadata-ingestion-modules/dagster-plugin/scripts/release.sh +++ b/metadata-ingestion-modules/dagster-plugin/scripts/release.sh @@ -13,7 +13,7 @@ MODULE=datahub_dagster_plugin python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"' if [[ ${RELEASE_VERSION:-} ]]; then # Replace version with RELEASE_VERSION env variable - sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/${MODULE}/__init__.py + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py else vim src/${MODULE}/__init__.py fi @@ -23,4 +23,4 @@ python -m build if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then python -m twine upload 'dist/*' fi -git restore src/${MODULE}/__init__.py +mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py diff --git a/metadata-ingestion/scripts/release.sh b/metadata-ingestion/scripts/release.sh index 955eb562089f79..8786eb21bff5cb 100755 --- a/metadata-ingestion/scripts/release.sh +++ b/metadata-ingestion/scripts/release.sh @@ -7,13 +7,15 @@ elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then ../gradlew install fi +MODULE=datahub + # Check packaging constraint. python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"' if [[ ${RELEASE_VERSION:-} ]]; then # Replace version with RELEASE_VERSION env variable - sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py + sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py else - vim src/datahub/__init__.py + vim src/${MODULE}/__init__.py fi rm -rf build dist || true @@ -21,4 +23,4 @@ python -m build if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then python -m twine upload 'dist/*' fi -git restore src/datahub/__init__.py +mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py diff --git a/vercel.json b/vercel.json index 077f69ff249ec9..a1815cab8ae88c 100644 --- a/vercel.json +++ b/vercel.json @@ -1,5 +1,5 @@ { - "buildCommand": "./gradlew -PuseSystemNode=true :docs-website:build -x :metadata-ingestion-modules:dagster-plugin:build -x :metadata-ingestion-modules:dagster-plugin:check", + "buildCommand": "./gradlew -PuseSystemNode=true :docs-website:build", "github": { "silent": true, "autoJobCancelation": true