diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb9226bdd9a84..977c51601e6ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,6 +165,7 @@ jobs: docs-filter: ${{ steps.selective-checks.outputs.docs-filter }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} + suspended-providers-folders: ${{ steps.selective-checks.outputs.suspended-providers-folders }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} @@ -884,6 +885,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -924,6 +926,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -976,6 +979,7 @@ jobs: PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" BACKEND: "mysql" PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" @@ -1019,6 +1023,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1061,6 +1066,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" @@ -1097,6 +1103,7 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" BACKEND: "postgres" @@ -1156,6 +1163,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "${{needs.build-info.output.parallel-test-types}}" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" @@ -1197,6 +1205,7 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PARALLEL_TEST_TYPES: "Quarantined" + SUSPENDED_PROVIDERS_FOLDERS: "${{ needs.build-info.outputs.suspended-providers-folders }}" PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07fa5cfb3890f..e0b0637b52ee5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -321,7 +321,7 @@ repos: name: Checks setup extra packages description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file language: python - files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$ + files: ^setup\.py$|^docs/apache-airflow/extra-packages-ref\.rst$|^airflow/providers/.*/provider\.yaml$ pass_filenames: false entry: ./scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py additional_dependencies: ['rich>=12.4.4'] @@ -350,7 +350,7 @@ repos: name: Update extras in documentation entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py language: python - files: ^setup\.py$|^CONTRIBUTING\.rst$|^INSTALL$ + files: ^setup\.py$|^CONTRIBUTING\.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$ pass_filenames: false additional_dependencies: ['rich>=12.4.4'] - id: check-extras-order @@ -698,7 +698,7 @@ repos: files: ^dev/breeze/.*$ pass_filenames: false require_serial: true - additional_dependencies: ['click', 'rich>=12.4.4'] + additional_dependencies: ['click', 'rich>=12.4.4', 'pyyaml'] - id: check-system-tests-present name: Check if system tests have required segments of code entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py @@ -844,10 +844,15 @@ repos: name: Update output of breeze commands in BREEZE.rst entry: ./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py language: python - files: ^BREEZE\.rst$|^dev/breeze/.*$|^\.pre-commit-config\.yaml$ + files: > + (?x) + ^BREEZE\.rst$|^dev/breeze/.*$| + ^\.pre-commit-config\.yaml$| + ^scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py$| + ^generated/provider_dependencies.json$ require_serial: true pass_filenames: false - additional_dependencies: ['rich>=12.4.4', 'rich-click>=1.5', 'inputimeout'] + additional_dependencies: ['rich>=12.4.4', 'rich-click>=1.5', 'inputimeout', 'pyyaml'] - id: check-example-dags-urls name: Check that example dags url include provider versions entry: ./scripts/ci/pre_commit/pre_commit_update_example_dags_paths.py @@ -894,7 +899,7 @@ repos: entry: ./scripts/ci/pre_commit/pre_commit_mypy.py files: ^dev/.*\.py$ require_serial: true - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] + additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml'] - id: run-mypy name: Run mypy for core language: python @@ -902,7 +907,7 @@ repos: files: \.py$ exclude: ^.*/_vendor/|^airflow/migrations|^airflow/providers|^dev|^docs|^provider_packages|^tests/providers|^tests/system/providers require_serial: true - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] + additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml'] - id: run-mypy name: Run mypy for providers language: python @@ -910,7 +915,7 @@ repos: files: ^airflow/providers/.*\.py$|^tests/providers/\*\.py$|^tests/system/providers/\*\.py$ exclude: ^.*/_vendor/ require_serial: true - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] + additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml'] - id: run-mypy name: Run mypy for /docs/ folder language: python @@ -918,7 +923,7 @@ repos: files: ^docs/.*\.py$ exclude: ^docs/rtd-deprecation require_serial: true - additional_dependencies: ['rich>=12.4.4', 'inputimeout'] + additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml'] - id: check-provider-yaml-valid name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py diff --git a/Dockerfile.ci b/Dockerfile.ci index b2b83d50f1e90..cae8826589640 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -890,6 +890,17 @@ EXTRA_PYTEST_ARGS=( "-rfEX" ) +if [[ ${SUSPENDED_PROVIDERS_FOLDERS=} != "" ]]; then + for provider in ${SUSPENDED_PROVIDERS_FOLDERS=}; do + echo "Skipping tests for suspended provider: ${provider}" + EXTRA_PYTEST_ARGS+=( + "--ignore=tests/providers/${provider}" + "--ignore=tests/system/providers/${provider}" + "--ignore=tests/integration/providers/${provider}" + ) + done +fi + if [[ "${TEST_TYPE}" == "Helm" ]]; then _cpus="$(grep -c 'cpu[0-9]' /proc/stat)" echo "Running tests with ${_cpus} CPUs in parallel" diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index ab86fd1643f14..fcb8851830d01 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -21,6 +21,10 @@ "type": "string" } }, + "suspended": { + "description": "If set to true, the provider is suspended and it's not a candidate for release nor contributes dependencies to constraint calculations/CI image. Tests are excluded.", + "type:": "boolean" + }, "dependencies": { "description": "Dependencies that should be added to the provider", "type": "array", @@ -326,6 +330,7 @@ "name", "package-name", "description", + "suspended", "dependencies", "versions" ] diff --git a/airflow/providers/SUSPENDING_AND_RESUMING_PROVIDERS.rst b/airflow/providers/SUSPENDING_AND_RESUMING_PROVIDERS.rst new file mode 100644 index 0000000000000..a6b5a01247ea6 --- /dev/null +++ b/airflow/providers/SUSPENDING_AND_RESUMING_PROVIDERS.rst @@ -0,0 +1,150 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Suspending providers +==================== + +As of April 2023, we have the possibility to suspend individual providers, so that they are not holding +back dependencies for Airflow and other providers. The process of suspending providers is described +in `description of the process `_ + +Technically, suspending a provider is done by setting ``suspended : true``, in the provider.yaml of the +provider. This should be followed by committing the change and either automatically or manually running +pre-commit checks that will either update derived configuration files or ask you to update them manually. +Note that you might need to run pre-commit several times until all the static checks pass, +because modification from one pre-commit might impact other pre-commits. + +If you have pre-commit installed, pre-commit will be run automatically on commit. If you want to run it +manually after commit, you can run it via ``breeze static-checks --last-commit`` some of the tests might fail +because suspension of the provider might cause changes in the dependencies, so if you see errors about +missing dependencies imports, non-usable classes etc., you will need to build the CI image locally +via ``breeze build-image --python 3.7 --upgrade-to-newer-dependencies`` after the first pre-commit run +and then run the static checks again. + +If you want to be absolutely sure to run all static checks you can always do this via +``pre-commit run --all-files`` or ``breeze static-checks --all-files``. + +Some of the manual modifications you will have to do (in both cases ``pre-commit`` will guide you on what +to do. + +* You will have to run ``breeze setup regenerate-command-images`` to regenerate breeze help files +* you will need to update ``extra-packages-ref.rst`` and in some cases - when mentioned there explicitly - + ``setup.py`` to remove the provider from list of dependencies. + +What happens under-the-hood as the result, is that ``generated/providers.json`` file is updated with +the information about available providers and their dependencies and it is used by our tooling to +exclude suspended providers from all relevant parts of the build and CI system (such as building CI image +with dependencies, building documentation, running tests, etc.) + + +Additional changes needed for cross-dependent providers +======================================================= + +Those steps above are usually enough for most providers that are "standalone" and not imported or used by +other providers (in most cases we will not suspend such providers). However some extra steps might be needed +for providers that are used by other providers, or that are part of the default PROD Dockerfile: + +* Most of the tests for the suspended provider, will be automatically excluded by pytest collection. However, + in case a provider is dependent on by another provider, the relevant tests might fail to be collected or + run by ``pytest``. In such cases you should skip the whole test module failing to be collected by + adding ``pytest.importorskip`` at the top of the test module. + For example if your tests fail because they need to import ``apache.airflow.providers.google`` + and you have suspended it, you should add this line at the top of the test module that fails. + +Example failing collection after ``google`` provider has been suspended: + + .. code-block:: txt + + _____ ERROR collecting tests/providers/apache/beam/operators/test_beam.py ______ + ImportError while importing test module '/opt/airflow/tests/providers/apache/beam/operators/test_beam.py'. + Hint: make sure your test modules/packages have valid Python names. + Traceback: + /usr/local/lib/python3.7/importlib/__init__.py:127: in import_module + return _bootstrap._gcd_import(name[level:], package, level) + tests/providers/apache/beam/operators/test_beam.py:25: in + from airflow.providers.apache.beam.operators.beam import ( + airflow/providers/apache/beam/operators/beam.py:35: in + from airflow.providers.google.cloud.hooks.dataflow import ( + airflow/providers/google/cloud/hooks/dataflow.py:32: in + from google.cloud.dataflow_v1beta3 import GetJobRequest, Job, JobState, JobsV1Beta3AsyncClient, JobView + E ModuleNotFoundError: No module named 'google.cloud.dataflow_v1beta3' + _ ERROR collecting tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py _ + + +The fix is to add this line at the top of the ``tests/providers/apache/beam/operators/test_beam.py`` module: + + .. code-block:: python + + pytest.importorskip("apache.airflow.providers.google") + + +* Some of the other providers might also just import unconditionally the suspended provider and they will + fail during provider verification step in CI. In this case you should turn the provider imports + into conditional imports. For example when import fails after ``amazon`` provider has been suspended: + + .. code-block:: txt + + Traceback (most recent call last): + File "/opt/airflow/scripts/in_container/verify_providers.py", line 266, in import_all_classes + _module = importlib.import_module(modinfo.name) + File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module + return _bootstrap._gcd_import(name, package, level) + File "", line 1006, in _gcd_import + File "", line 983, in _find_and_load + File "", line 967, in _find_and_load_unlocked + File "", line 677, in _load_unlocked + File "", line 728, in exec_module + File "", line 219, in _call_with_frames_removed + File "/usr/local/lib/python3.7/site-packages/airflow/providers/mysql/transfers/s3_to_mysql.py", line 23, in + from airflow.providers.amazon.aws.hooks.s3 import S3Hook + ModuleNotFoundError: No module named 'airflow.providers.amazon' + +or: + + .. code-block:: txt + + Error: The ``airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs`` object in transfers list in + airflow/providers/microsoft/azure/provider.yaml does not exist or is not a module: + No module named 'gcloud.aio.storage' + +The fix for that is to turn the feature into an optional provider feature (in the place where the excluded +``airflow.providers`` import happens: + + .. code-block:: python + + try: + from airflow.providers.amazon.aws.hooks.s3 import S3Hook + except ImportError as e: + from airflow.exceptions import AirflowOptionalProviderFeatureException + + raise AirflowOptionalProviderFeatureException(e) + + +* In case we suspend an important provider, which is part of the default Dockerfile you might want to + update the tests for PROD docker image in ``docker_tests/test_prod_image.py``. + +* Some of the suspended providers might also fail ``breeze`` unit tests that expect a fixed set of providers. + Those tests should be adjusted (but this is not very likely to happen, because the tests are using only + the most common providers that we will not be likely to suspend). + + +Resuming providers +================== + +Resuming providers is done by reverting the original change that suspended it. In case there are changes +needed to fix problems in the reverted provider, our CI will detect them and you will have to fix them +as part of the PR reverting the suspension. diff --git a/airflow/providers/airbyte/provider.yaml b/airflow/providers/airbyte/provider.yaml index d4f70f3b0be00..c0a9428f90115 100644 --- a/airflow/providers/airbyte/provider.yaml +++ b/airflow/providers/airbyte/provider.yaml @@ -21,6 +21,7 @@ name: Airbyte description: | `Airbyte `__ +suspended: false versions: - 3.2.1 - 3.2.0 diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index 51beba76ab498..bc979f78d382f 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -21,6 +21,7 @@ name: Alibaba description: | Alibaba Cloud integration (including `Alibaba Cloud `__). +suspended: false versions: - 2.3.0 - 2.2.0 diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 9040fc7c4a9b7..e770e141f3b54 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -21,6 +21,7 @@ name: Amazon description: | Amazon integration (including `Amazon Web Services (AWS) `__). +suspended: false versions: - 7.4.0 - 7.3.0 diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index 7802afe2e8c5d..cb1d6e7440291 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -21,6 +21,7 @@ name: Apache Beam description: | `Apache Beam `__. +suspended: false versions: - 4.3.0 - 4.2.0 diff --git a/airflow/providers/apache/cassandra/provider.yaml b/airflow/providers/apache/cassandra/provider.yaml index 2055bd6c6041a..4e52df6c7694a 100644 --- a/airflow/providers/apache/cassandra/provider.yaml +++ b/airflow/providers/apache/cassandra/provider.yaml @@ -21,6 +21,7 @@ name: Apache Cassandra description: | `Apache Cassandra `__. +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/apache/drill/provider.yaml b/airflow/providers/apache/drill/provider.yaml index 3f7ab23961999..0829eff6965eb 100644 --- a/airflow/providers/apache/drill/provider.yaml +++ b/airflow/providers/apache/drill/provider.yaml @@ -21,6 +21,7 @@ name: Apache Drill description: | `Apache Drill `__. +suspended: false versions: - 2.3.2 - 2.3.1 diff --git a/airflow/providers/apache/druid/provider.yaml b/airflow/providers/apache/druid/provider.yaml index d391f1a371978..29c3e6d895379 100644 --- a/airflow/providers/apache/druid/provider.yaml +++ b/airflow/providers/apache/druid/provider.yaml @@ -21,6 +21,7 @@ name: Apache Druid description: | `Apache Druid `__. +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/apache/flink/provider.yaml b/airflow/providers/apache/flink/provider.yaml index 918e30af17197..d1af83fef87ec 100644 --- a/airflow/providers/apache/flink/provider.yaml +++ b/airflow/providers/apache/flink/provider.yaml @@ -21,6 +21,7 @@ name: Apache Flink description: | `Apache Flink `__ +suspended: false versions: - 1.0.1 - 1.0.0 diff --git a/airflow/providers/apache/hdfs/provider.yaml b/airflow/providers/apache/hdfs/provider.yaml index 1291e9dfec0b0..86cb3fa717895 100644 --- a/airflow/providers/apache/hdfs/provider.yaml +++ b/airflow/providers/apache/hdfs/provider.yaml @@ -22,6 +22,7 @@ description: | `Hadoop Distributed File System (HDFS) `__ and `WebHDFS `__. +suspended: false versions: - 3.2.1 - 3.2.0 diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 4c591336a84ca..48d506ca89d4c 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -21,6 +21,7 @@ name: Apache Hive description: | `Apache Hive `__ +suspended: false versions: - 6.0.0 - 5.1.3 diff --git a/airflow/providers/apache/impala/provider.yaml b/airflow/providers/apache/impala/provider.yaml index 0d8501a652732..8edb2b569d41d 100644 --- a/airflow/providers/apache/impala/provider.yaml +++ b/airflow/providers/apache/impala/provider.yaml @@ -21,6 +21,7 @@ name: Apache Impala description: | `Apache Impala `__. +suspended: false versions: - 1.0.0 diff --git a/airflow/providers/apache/kylin/provider.yaml b/airflow/providers/apache/kylin/provider.yaml index 73f69523e11e8..b4658c02bbece 100644 --- a/airflow/providers/apache/kylin/provider.yaml +++ b/airflow/providers/apache/kylin/provider.yaml @@ -21,6 +21,7 @@ name: Apache Kylin description: | `Apache Kylin `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/apache/livy/provider.yaml b/airflow/providers/apache/livy/provider.yaml index 30855d9253bb9..e84341acf5134 100644 --- a/airflow/providers/apache/livy/provider.yaml +++ b/airflow/providers/apache/livy/provider.yaml @@ -21,6 +21,7 @@ name: Apache Livy description: | `Apache Livy `__ +suspended: false versions: - 3.4.0 - 3.3.0 diff --git a/airflow/providers/apache/pig/provider.yaml b/airflow/providers/apache/pig/provider.yaml index df1c55a8a7b61..86b502571f16e 100644 --- a/airflow/providers/apache/pig/provider.yaml +++ b/airflow/providers/apache/pig/provider.yaml @@ -21,6 +21,7 @@ name: Apache Pig description: | `Apache Pig `__ +suspended: false versions: - 4.0.0 - 3.0.0 diff --git a/airflow/providers/apache/pinot/provider.yaml b/airflow/providers/apache/pinot/provider.yaml index b283ea9ca2b2f..f743ee99f46b7 100644 --- a/airflow/providers/apache/pinot/provider.yaml +++ b/airflow/providers/apache/pinot/provider.yaml @@ -21,6 +21,7 @@ name: Apache Pinot description: | `Apache Pinot `__ +suspended: false versions: - 4.0.1 - 4.0.0 diff --git a/airflow/providers/apache/spark/provider.yaml b/airflow/providers/apache/spark/provider.yaml index 8eb23ac0d418f..0f3082a27af9b 100644 --- a/airflow/providers/apache/spark/provider.yaml +++ b/airflow/providers/apache/spark/provider.yaml @@ -21,6 +21,7 @@ name: Apache Spark description: | `Apache Spark `__ +suspended: false versions: - 4.0.1 - 4.0.0 diff --git a/airflow/providers/apache/sqoop/provider.yaml b/airflow/providers/apache/sqoop/provider.yaml index 0da5dfddd7bfa..783d2e5001563 100644 --- a/airflow/providers/apache/sqoop/provider.yaml +++ b/airflow/providers/apache/sqoop/provider.yaml @@ -21,6 +21,7 @@ name: Apache Sqoop description: | `Apache Sqoop `__ +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/arangodb/provider.yaml b/airflow/providers/arangodb/provider.yaml index f70fce64076fc..b984df05bdbbc 100644 --- a/airflow/providers/arangodb/provider.yaml +++ b/airflow/providers/arangodb/provider.yaml @@ -25,6 +25,7 @@ dependencies: - apache-airflow>=2.3.0 - python-arango>=7.3.2 +suspended: false versions: - 2.1.1 - 2.1.0 diff --git a/airflow/providers/asana/provider.yaml b/airflow/providers/asana/provider.yaml index 2585758a6374b..e4da44bde3977 100644 --- a/airflow/providers/asana/provider.yaml +++ b/airflow/providers/asana/provider.yaml @@ -21,6 +21,7 @@ name: Asana description: | `Asana `__ +suspended: false versions: - 2.1.0 - 2.0.1 diff --git a/airflow/providers/atlassian/jira/provider.yaml b/airflow/providers/atlassian/jira/provider.yaml index 3a4ff601f8670..d6a1f38a12310 100644 --- a/airflow/providers/atlassian/jira/provider.yaml +++ b/airflow/providers/atlassian/jira/provider.yaml @@ -21,6 +21,7 @@ name: Atlassian Jira description: | `Atlassian Jira `__ +suspended: false versions: - 2.0.1 - 2.0.0 diff --git a/airflow/providers/celery/provider.yaml b/airflow/providers/celery/provider.yaml index 8adc4c1c6c12c..a5f3cfdecb838 100644 --- a/airflow/providers/celery/provider.yaml +++ b/airflow/providers/celery/provider.yaml @@ -21,6 +21,7 @@ name: Celery description: | `Celery `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/cloudant/provider.yaml b/airflow/providers/cloudant/provider.yaml index 8495f93af0b22..64a9069d64d94 100644 --- a/airflow/providers/cloudant/provider.yaml +++ b/airflow/providers/cloudant/provider.yaml @@ -21,6 +21,7 @@ name: IBM Cloudant description: | `IBM Cloudant `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/cncf/kubernetes/provider.yaml b/airflow/providers/cncf/kubernetes/provider.yaml index 75f2d9d2da656..77fc7893fce4c 100644 --- a/airflow/providers/cncf/kubernetes/provider.yaml +++ b/airflow/providers/cncf/kubernetes/provider.yaml @@ -21,6 +21,7 @@ name: Kubernetes description: | `Kubernetes `__ +suspended: false versions: - 5.3.0 - 5.2.2 diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index efbbbad55dc21..2aedf9752e9e4 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -21,6 +21,7 @@ name: Common SQL description: | `Common SQL Provider `__ +suspended: false versions: - 1.4.0 - 1.3.4 diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index b3a9631ac7faf..c48a47f2da3fd 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -21,6 +21,7 @@ name: Databricks description: | `Databricks `__ +suspended: false versions: - 4.0.1 - 4.0.0 diff --git a/airflow/providers/datadog/provider.yaml b/airflow/providers/datadog/provider.yaml index f43ad466b3c5a..2d2c77348aa45 100644 --- a/airflow/providers/datadog/provider.yaml +++ b/airflow/providers/datadog/provider.yaml @@ -21,6 +21,7 @@ name: Datadog description: | `Datadog `__ +suspended: false versions: - 3.2.0 - 3.1.0 diff --git a/airflow/providers/dbt/cloud/provider.yaml b/airflow/providers/dbt/cloud/provider.yaml index 4d0836bbf5735..93df19e267add 100644 --- a/airflow/providers/dbt/cloud/provider.yaml +++ b/airflow/providers/dbt/cloud/provider.yaml @@ -21,6 +21,7 @@ name: dbt Cloud description: | `dbt Cloud `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/dingding/provider.yaml b/airflow/providers/dingding/provider.yaml index 37ddc69b8706e..821a94a484254 100644 --- a/airflow/providers/dingding/provider.yaml +++ b/airflow/providers/dingding/provider.yaml @@ -21,6 +21,7 @@ name: Dingding description: | `Dingding `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/discord/provider.yaml b/airflow/providers/discord/provider.yaml index 3dc1a30ac9165..ef8c89f0bb1c6 100644 --- a/airflow/providers/discord/provider.yaml +++ b/airflow/providers/discord/provider.yaml @@ -21,6 +21,7 @@ name: Discord description: | `Discord `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/docker/provider.yaml b/airflow/providers/docker/provider.yaml index 0387c86d5dbe6..99e793cc0caa9 100644 --- a/airflow/providers/docker/provider.yaml +++ b/airflow/providers/docker/provider.yaml @@ -21,6 +21,7 @@ name: Docker description: | `Docker `__ +suspended: false versions: - 3.5.1 - 3.5.0 diff --git a/airflow/providers/elasticsearch/provider.yaml b/airflow/providers/elasticsearch/provider.yaml index b862da59ddd81..941390d1c3db5 100644 --- a/airflow/providers/elasticsearch/provider.yaml +++ b/airflow/providers/elasticsearch/provider.yaml @@ -21,6 +21,7 @@ name: Elasticsearch description: | `Elasticsearch `__ +suspended: false versions: - 4.4.0 - 4.3.3 diff --git a/airflow/providers/exasol/provider.yaml b/airflow/providers/exasol/provider.yaml index 4d5215bdc1a58..70af2fc0896a0 100644 --- a/airflow/providers/exasol/provider.yaml +++ b/airflow/providers/exasol/provider.yaml @@ -21,6 +21,7 @@ name: Exasol description: | `Exasol `__ +suspended: false versions: - 4.1.3 - 4.1.2 diff --git a/airflow/providers/facebook/provider.yaml b/airflow/providers/facebook/provider.yaml index 2e05cca631562..ec5f0be209573 100644 --- a/airflow/providers/facebook/provider.yaml +++ b/airflow/providers/facebook/provider.yaml @@ -21,6 +21,7 @@ name: Facebook description: | `Facebook Ads `__ +suspended: false versions: - 3.1.0 - 3.0.1 diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index 8214a4b3e52ab..c86c9c5fb5c95 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -21,6 +21,7 @@ name: File Transfer Protocol (FTP) description: | `File Transfer Protocol (FTP) `__ +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index 92ef8c1c3ffef..f545d26967efb 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -28,6 +28,7 @@ dependencies: # See https://github.com/PyGithub/PyGithub/issues/2436. - PyGithub!=1.58 +suspended: false versions: - 2.2.1 - 2.2.0 diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index 9895225de2b0f..63d797d995755 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -28,6 +28,7 @@ description: | - `Google Marketing Platform `__ - `Google Workspace `__ (formerly Google Suite) +suspended: false versions: - 8.12.0 - 8.11.0 diff --git a/airflow/providers/grpc/provider.yaml b/airflow/providers/grpc/provider.yaml index 86cd9be1ecacd..a52769e4c5d37 100644 --- a/airflow/providers/grpc/provider.yaml +++ b/airflow/providers/grpc/provider.yaml @@ -21,6 +21,7 @@ name: gRPC description: | `gRPC `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/hashicorp/provider.yaml b/airflow/providers/hashicorp/provider.yaml index 6dc5d4fb54bad..dd6ecb70c03e5 100644 --- a/airflow/providers/hashicorp/provider.yaml +++ b/airflow/providers/hashicorp/provider.yaml @@ -21,6 +21,7 @@ name: Hashicorp description: | Hashicorp including `Hashicorp Vault `__ +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index 1d343ce7d536c..3c3b615b63fb2 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -21,6 +21,7 @@ name: Hypertext Transfer Protocol (HTTP) description: | `Hypertext Transfer Protocol (HTTP) `__ +suspended: false versions: - 4.3.0 - 4.2.0 diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index fc844bedb68d0..2e38b04c4ab9a 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -22,6 +22,7 @@ name: Internet Message Access Protocol (IMAP) description: | `Internet Message Access Protocol (IMAP) `__ +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/influxdb/provider.yaml b/airflow/providers/influxdb/provider.yaml index 0518b1e5a507b..9a0ee1cc05d8e 100644 --- a/airflow/providers/influxdb/provider.yaml +++ b/airflow/providers/influxdb/provider.yaml @@ -28,6 +28,7 @@ dependencies: - influxdb-client>=1.19.0 - requests>=2.26.0 +suspended: false versions: - 2.1.0 - 2.0.0 diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index e229585814ae7..3768925e0f50f 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -21,6 +21,7 @@ name: Java Database Connectivity (JDBC) description: | `Java Database Connectivity (JDBC) `__ +suspended: false versions: - 3.3.0 - 3.2.1 diff --git a/airflow/providers/jenkins/provider.yaml b/airflow/providers/jenkins/provider.yaml index 962373593760d..ecf57526447c8 100644 --- a/airflow/providers/jenkins/provider.yaml +++ b/airflow/providers/jenkins/provider.yaml @@ -21,6 +21,7 @@ name: Jenkins description: | `Jenkins `__ +suspended: false versions: - 3.2.1 - 3.2.0 diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 1373d3a65a594..e8122a5321f8b 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -20,6 +20,7 @@ package-name: apache-airflow-providers-microsoft-azure name: Microsoft Azure description: | `Microsoft Azure `__ +suspended: false versions: - 5.3.0 - 5.2.1 diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml index 1ef6f29e5dfd4..f2add9f5fc908 100644 --- a/airflow/providers/microsoft/mssql/provider.yaml +++ b/airflow/providers/microsoft/mssql/provider.yaml @@ -21,6 +21,7 @@ name: Microsoft SQL Server (MSSQL) description: | `Microsoft SQL Server (MSSQL) `__ +suspended: false versions: - 3.3.2 - 3.3.1 diff --git a/airflow/providers/microsoft/psrp/provider.yaml b/airflow/providers/microsoft/psrp/provider.yaml index fb4fde018a307..0081a58384c90 100644 --- a/airflow/providers/microsoft/psrp/provider.yaml +++ b/airflow/providers/microsoft/psrp/provider.yaml @@ -23,6 +23,7 @@ description: | `PowerShell Remoting Protocol (PSRP) `__. +suspended: false versions: - 2.2.0 - 2.1.0 diff --git a/airflow/providers/microsoft/winrm/provider.yaml b/airflow/providers/microsoft/winrm/provider.yaml index 82f9375e3c0f1..e9cd31c015558 100644 --- a/airflow/providers/microsoft/winrm/provider.yaml +++ b/airflow/providers/microsoft/winrm/provider.yaml @@ -21,6 +21,7 @@ name: Windows Remote Management (WinRM) description: | `Windows Remote Management (WinRM) `__ +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/mongo/provider.yaml b/airflow/providers/mongo/provider.yaml index d01a95fe8ecc6..006158e394978 100644 --- a/airflow/providers/mongo/provider.yaml +++ b/airflow/providers/mongo/provider.yaml @@ -21,6 +21,7 @@ name: MongoDB description: | `MongoDB `__ +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/mysql/provider.yaml b/airflow/providers/mysql/provider.yaml index 7ea9067a058a4..ca2a951939509 100644 --- a/airflow/providers/mysql/provider.yaml +++ b/airflow/providers/mysql/provider.yaml @@ -21,6 +21,7 @@ name: MySQL description: | `MySQL `__ +suspended: false versions: - 4.0.2 - 4.0.1 diff --git a/airflow/providers/neo4j/provider.yaml b/airflow/providers/neo4j/provider.yaml index fa2726b908ddf..f736d05299ae1 100644 --- a/airflow/providers/neo4j/provider.yaml +++ b/airflow/providers/neo4j/provider.yaml @@ -21,6 +21,7 @@ name: Neo4j description: | `Neo4j `__ +suspended: false versions: - 3.2.1 - 3.2.0 diff --git a/airflow/providers/odbc/provider.yaml b/airflow/providers/odbc/provider.yaml index 76d1d5e548352..a5e0575002a01 100644 --- a/airflow/providers/odbc/provider.yaml +++ b/airflow/providers/odbc/provider.yaml @@ -21,6 +21,7 @@ name: ODBC description: | `ODBC `__ +suspended: false versions: - 3.2.1 - 3.2.0 diff --git a/airflow/providers/openfaas/provider.yaml b/airflow/providers/openfaas/provider.yaml index 90fed6940ffd3..1a8ea4d2b78af 100644 --- a/airflow/providers/openfaas/provider.yaml +++ b/airflow/providers/openfaas/provider.yaml @@ -21,6 +21,7 @@ name: OpenFaaS description: | `OpenFaaS `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/opsgenie/provider.yaml b/airflow/providers/opsgenie/provider.yaml index 58aba33976f60..98fdff5e337be 100644 --- a/airflow/providers/opsgenie/provider.yaml +++ b/airflow/providers/opsgenie/provider.yaml @@ -21,6 +21,7 @@ name: Opsgenie description: | `Opsgenie `__ +suspended: false versions: - 5.0.0 - 4.0.0 diff --git a/airflow/providers/oracle/provider.yaml b/airflow/providers/oracle/provider.yaml index 9c6fd77cf5174..1dde4a4d4080c 100644 --- a/airflow/providers/oracle/provider.yaml +++ b/airflow/providers/oracle/provider.yaml @@ -21,6 +21,7 @@ name: Oracle description: | `Oracle `__ +suspended: false versions: - 3.6.0 - 3.5.1 diff --git a/airflow/providers/pagerduty/provider.yaml b/airflow/providers/pagerduty/provider.yaml index 80965bdb69885..2e773b8a25184 100644 --- a/airflow/providers/pagerduty/provider.yaml +++ b/airflow/providers/pagerduty/provider.yaml @@ -21,6 +21,7 @@ name: Pagerduty description: | `Pagerduty `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index afcfe412765f1..855ef932c8483 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -21,6 +21,7 @@ name: Papermill description: | `Papermill `__ +suspended: false versions: - 3.1.1 - 3.1.0 diff --git a/airflow/providers/plexus/provider.yaml b/airflow/providers/plexus/provider.yaml index 0e908c502caec..baa61ce013b45 100644 --- a/airflow/providers/plexus/provider.yaml +++ b/airflow/providers/plexus/provider.yaml @@ -21,6 +21,7 @@ name: Plexus description: | `Plexus `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/postgres/provider.yaml b/airflow/providers/postgres/provider.yaml index 9dfe43e4133b9..e14f1e12de71e 100644 --- a/airflow/providers/postgres/provider.yaml +++ b/airflow/providers/postgres/provider.yaml @@ -21,6 +21,7 @@ name: PostgreSQL description: | `PostgreSQL `__ +suspended: false versions: - 5.4.0 - 5.3.1 diff --git a/airflow/providers/presto/provider.yaml b/airflow/providers/presto/provider.yaml index 452f232bcb4f5..08c68f1ed4d3a 100644 --- a/airflow/providers/presto/provider.yaml +++ b/airflow/providers/presto/provider.yaml @@ -21,6 +21,7 @@ name: Presto description: | `Presto `__ +suspended: false versions: - 4.2.2 - 4.2.1 diff --git a/airflow/providers/qubole/provider.yaml b/airflow/providers/qubole/provider.yaml index bd869384798c7..3de981a22dce8 100644 --- a/airflow/providers/qubole/provider.yaml +++ b/airflow/providers/qubole/provider.yaml @@ -21,6 +21,7 @@ name: Qubole description: | `Qubole `__ +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/redis/provider.yaml b/airflow/providers/redis/provider.yaml index 0cbcab3a88e66..47335f2a2e2a5 100644 --- a/airflow/providers/redis/provider.yaml +++ b/airflow/providers/redis/provider.yaml @@ -21,6 +21,7 @@ name: Redis description: | `Redis `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/salesforce/provider.yaml b/airflow/providers/salesforce/provider.yaml index d1f18cddcc251..140c1cdfd5cf4 100644 --- a/airflow/providers/salesforce/provider.yaml +++ b/airflow/providers/salesforce/provider.yaml @@ -21,6 +21,7 @@ name: Salesforce description: | `Salesforce `__ +suspended: false versions: - 5.3.0 - 5.2.0 diff --git a/airflow/providers/samba/provider.yaml b/airflow/providers/samba/provider.yaml index 9b210018515a6..80d299699cd70 100644 --- a/airflow/providers/samba/provider.yaml +++ b/airflow/providers/samba/provider.yaml @@ -21,6 +21,7 @@ name: Samba description: | `Samba `__ +suspended: false versions: - 4.1.0 - 4.0.0 diff --git a/airflow/providers/segment/provider.yaml b/airflow/providers/segment/provider.yaml index b46b96eecab56..a51118a28e084 100644 --- a/airflow/providers/segment/provider.yaml +++ b/airflow/providers/segment/provider.yaml @@ -21,6 +21,7 @@ name: Segment description: | `Segment `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/sendgrid/provider.yaml b/airflow/providers/sendgrid/provider.yaml index 210af20925f31..890b2b9310844 100644 --- a/airflow/providers/sendgrid/provider.yaml +++ b/airflow/providers/sendgrid/provider.yaml @@ -25,6 +25,7 @@ dependencies: - apache-airflow>=2.3.0 - sendgrid>=6.0.0 +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/sftp/provider.yaml b/airflow/providers/sftp/provider.yaml index bcae88c76441f..3a1168ef104e5 100644 --- a/airflow/providers/sftp/provider.yaml +++ b/airflow/providers/sftp/provider.yaml @@ -21,6 +21,7 @@ name: SFTP description: | `SSH File Transfer Protocol (SFTP) `__ +suspended: false versions: - 4.2.4 - 4.2.3 diff --git a/airflow/providers/singularity/provider.yaml b/airflow/providers/singularity/provider.yaml index ff0ab0945d249..99a9468df1072 100644 --- a/airflow/providers/singularity/provider.yaml +++ b/airflow/providers/singularity/provider.yaml @@ -21,6 +21,7 @@ name: Singularity description: | `Singularity `__ +suspended: false versions: - 3.1.0 - 3.0.0 diff --git a/airflow/providers/slack/provider.yaml b/airflow/providers/slack/provider.yaml index 920033d705b53..7e40553dff8a6 100644 --- a/airflow/providers/slack/provider.yaml +++ b/airflow/providers/slack/provider.yaml @@ -21,6 +21,7 @@ name: Slack description: | `Slack `__ +suspended: false versions: - 7.2.0 - 7.1.1 diff --git a/airflow/providers/smtp/provider.yaml b/airflow/providers/smtp/provider.yaml index b7b96eba23aa9..0c707cc967182 100644 --- a/airflow/providers/smtp/provider.yaml +++ b/airflow/providers/smtp/provider.yaml @@ -22,6 +22,7 @@ name: Simple Mail Transfer Protocol (SMTP) description: | `Simple Mail Transfer Protocol (SMTP) `__ +suspended: false versions: - 1.0.0 diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index 09551be81af66..d0bd2d20763d9 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -21,6 +21,7 @@ name: Snowflake description: | `Snowflake `__ +suspended: false versions: - 4.0.4 - 4.0.3 diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index 2c144320556c0..31b06edbba15c 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -21,6 +21,7 @@ name: SQLite description: | `SQLite `__ +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/ssh/provider.yaml b/airflow/providers/ssh/provider.yaml index eaaa6fa38f640..09990bd6a3d9e 100644 --- a/airflow/providers/ssh/provider.yaml +++ b/airflow/providers/ssh/provider.yaml @@ -21,6 +21,7 @@ name: SSH description: | `Secure Shell (SSH) `__ +suspended: false versions: - 3.6.0 - 3.5.0 diff --git a/airflow/providers/tableau/provider.yaml b/airflow/providers/tableau/provider.yaml index 07b65084a52a7..e2719fc1c306a 100644 --- a/airflow/providers/tableau/provider.yaml +++ b/airflow/providers/tableau/provider.yaml @@ -21,6 +21,7 @@ name: Tableau description: | `Tableau `__ +suspended: false versions: - 4.1.0 - 4.0.0 diff --git a/airflow/providers/tabular/provider.yaml b/airflow/providers/tabular/provider.yaml index 28c265fc98158..4519359c04066 100644 --- a/airflow/providers/tabular/provider.yaml +++ b/airflow/providers/tabular/provider.yaml @@ -21,6 +21,7 @@ name: Tabular description: | `Tabular `__ +suspended: false versions: - 1.1.0 - 1.0.1 diff --git a/airflow/providers/telegram/provider.yaml b/airflow/providers/telegram/provider.yaml index 922da6caf651e..1365ede2d93aa 100644 --- a/airflow/providers/telegram/provider.yaml +++ b/airflow/providers/telegram/provider.yaml @@ -21,6 +21,7 @@ name: Telegram description: | `Telegram `__ +suspended: false versions: - 4.0.0 - 3.1.1 diff --git a/airflow/providers/trino/provider.yaml b/airflow/providers/trino/provider.yaml index b2c91f295c9c7..0412897ffa173 100644 --- a/airflow/providers/trino/provider.yaml +++ b/airflow/providers/trino/provider.yaml @@ -21,6 +21,7 @@ name: Trino description: | `Trino `__ +suspended: false versions: - 4.3.2 - 4.3.1 diff --git a/airflow/providers/vertica/provider.yaml b/airflow/providers/vertica/provider.yaml index e6de112965118..fb7853ce41841 100644 --- a/airflow/providers/vertica/provider.yaml +++ b/airflow/providers/vertica/provider.yaml @@ -21,6 +21,7 @@ name: Vertica description: | `Vertica `__ +suspended: false versions: - 3.3.1 - 3.3.0 diff --git a/airflow/providers/yandex/provider.yaml b/airflow/providers/yandex/provider.yaml index 5a23ce60af16e..6aa8eec2099eb 100644 --- a/airflow/providers/yandex/provider.yaml +++ b/airflow/providers/yandex/provider.yaml @@ -20,7 +20,7 @@ package-name: apache-airflow-providers-yandex name: Yandex description: | Yandex including `Yandex.Cloud `__ - +suspended: false versions: - 3.3.0 - 3.2.0 diff --git a/airflow/providers/zendesk/provider.yaml b/airflow/providers/zendesk/provider.yaml index 95d53cc7593bc..2fcf9f618fdf6 100644 --- a/airflow/providers/zendesk/provider.yaml +++ b/airflow/providers/zendesk/provider.yaml @@ -21,6 +21,7 @@ name: Zendesk description: | `Zendesk `__ +suspended: false versions: - 4.2.0 - 4.1.0 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index d1ca6e04af600..46ec131831c60 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -95,6 +95,7 @@ run_compile_www_assets, ) from airflow_breeze.utils.shared_options import get_forced_answer +from airflow_breeze.utils.suspended_providers import get_suspended_provider_ids option_debug_release_management = click.option( "--debug", @@ -276,8 +277,16 @@ def prepare_provider_packages( perform_environment_checks() cleanup_python_generated_files() packages_list = list(packages) + + suspended_provider_ids = get_suspended_provider_ids() if package_list_file: - packages_list.extend([package.strip() for package in package_list_file.readlines()]) + packages_list.extend( + [ + package.strip() + for package in package_list_file.readlines() + if package.strip() not in suspended_provider_ids + ] + ) shell_params = ShellParams( mount_sources=MOUNT_ALL, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 43661474fb2b9..ccc14c6bcd02e 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -617,6 +617,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "SKIP_ENVIRONMENT_INITIALIZATION", "false") set_value_to_default_if_not_set(env, "SKIP_PROVIDER_TESTS", "false") set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false") + set_value_to_default_if_not_set(env, "SUSPENDED_PROVIDERS_FOLDERS", "") set_value_to_default_if_not_set(env, "TEST_TYPE", "") set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60") set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 36d4c5afa524a..1146c1ecc2204 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -262,6 +262,9 @@ def find_airflow_sources_root_to_operate_on() -> Path: AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve() +TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "providers" +SYSTEM_TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "system" / "providers" +AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / ".build" WWW_CACHE_DIR = BUILD_CACHE_DIR / "www" WWW_ASSET_COMPILE_LOCK = WWW_CACHE_DIR / ".asset_compile.lock" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 61c9dc6f04261..6ce85c0f0aac6 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -24,7 +24,13 @@ from airflow_breeze.utils.exclude_from_matrix import excluded_combos from airflow_breeze.utils.github_actions import get_ga_output from airflow_breeze.utils.kubernetes_utils import get_kubernetes_python_combos -from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT +from airflow_breeze.utils.path_utils import ( + AIRFLOW_PROVIDERS_ROOT, + AIRFLOW_SOURCES_ROOT, + SYSTEM_TESTS_PROVIDERS_ROOT, + TESTS_PROVIDERS_ROOT, +) +from airflow_breeze.utils.suspended_providers import get_suspended_providers_folders if sys.version_info >= (3, 8): from functools import cached_property @@ -188,10 +194,6 @@ def __hash__(self): } ) -TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "providers" -SYSTEM_TESTS_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "tests" / "system" / "providers" -AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" - def find_provider_affected(changed_file: str) -> str | None: file_path = AIRFLOW_SOURCES_ROOT / changed_file @@ -230,13 +232,39 @@ def add_dependent_providers( def find_all_providers_affected(changed_files: tuple[str, ...]) -> set[str]: all_providers: set[str] = set() + dependencies = json.loads((AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json").read_text()) + all_providers_affected = False + suspended_providers: set[str] = set() for changed_file in changed_files: provider = find_provider_affected(changed_file) if provider == "Providers": - return set() - if provider is not None: - all_providers.add(provider) - dependencies = json.loads((AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json").read_text()) + all_providers_affected = True + elif provider is not None: + if provider not in dependencies: + suspended_providers.add(provider) + else: + all_providers.add(provider) + if all_providers_affected: + return set() + if suspended_providers: + # We check for suspended providers only after we have checked if all providers are affected. + # No matter if we found that we are modifying a suspended provider individually, if all providers are + # affected, then it means that we are ok to proceed because likely we are running some kind of + # global refactoring that affects multiple providers including the suspended one. This is a + # potential escape hatch if someone would like to modify suspended provider, + # but it can be found at the review time and is anyway harmless as the provider will not be + # released nor tested nor used in CI anyway. + get_console().print("[error]You are modifying suspended providers.\n") + get_console().print( + "[info]Some providers modified by this change have been suspended, " + "and before attempting such changes you should fix the reason for suspension." + ) + get_console().print( + "[info]When fixing it, you should set suspended = false in provider.yaml " + "to make changes to the provider." + ) + get_console().print(f"Suspended providers: {suspended_providers}") + sys.exit(1) for provider in list(all_providers): add_dependent_providers(all_providers, provider, dependencies) return all_providers @@ -612,3 +640,7 @@ def cache_directive(self) -> str: @cached_property def debug_resources(self) -> bool: return DEBUG_CI_RESOURCES_LABEL in self._pr_labels + + @cached_property + def suspended_providers_folders(self) -> str: + return " ".join(get_suspended_providers_folders()) diff --git a/dev/breeze/src/airflow_breeze/utils/suspended_providers.py b/dev/breeze/src/airflow_breeze/utils/suspended_providers.py new file mode 100644 index 0000000000000..df849129c210d --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/suspended_providers.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import yaml + +from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_ROOT, AIRFLOW_SOURCES_ROOT + + +def get_suspended_providers_folders() -> list[str]: + """ + Returns a list of suspended providers folders that should be + skipped when running tests (without any prefix - for example apache/beam, yandex, google etc.). + """ + suspended_providers = [] + for provider_path in AIRFLOW_PROVIDERS_ROOT.glob("**/provider.yaml"): + provider_yaml = yaml.safe_load(provider_path.read_text()) + if provider_yaml.get("suspended"): + suspended_providers.append( + provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) + .as_posix() + .replace("airflow/providers/", "") + ) + return suspended_providers + + +def get_suspended_provider_ids() -> list[str]: + """ + Yields the ids of suspended providers. + """ + suspended_provider_ids = [] + for provider_path in AIRFLOW_PROVIDERS_ROOT.glob("**/provider.yaml"): + provider_yaml = yaml.safe_load(provider_path.read_text()) + if provider_yaml.get("suspended"): + suspended_provider_ids.append( + provider_yaml["package-name"][len("apache-airflow-providers-") :].replace("-", ".") + ) + return suspended_provider_ids diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py index 90baffe2ba26f..f03f432a01c6a 100644 --- a/docs/exts/docs_build/docs_builder.py +++ b/docs/exts/docs_build/docs_builder.py @@ -298,7 +298,7 @@ def publish(self, override_versioned: bool): def get_available_providers_packages(): """Get list of all available providers packages to build.""" - return [provider["package-name"] for provider in ALL_PROVIDER_YAMLS] + return [provider["package-name"] for provider in ALL_PROVIDER_YAMLS if not provider.get("suspended")] def get_available_packages(): diff --git a/docs/publish_docs.py b/docs/publish_docs.py index 9a17aa29ef48e..eeda3f80e45c9 100755 --- a/docs/publish_docs.py +++ b/docs/publish_docs.py @@ -53,7 +53,9 @@ def get_available_packages(): """Get list of all available packages to build.""" - provider_package_names = [provider["package-name"] for provider in ALL_PROVIDER_YAMLS] + provider_package_names = [ + provider["package-name"] for provider in ALL_PROVIDER_YAMLS if not provider.get("suspended") + ] return [ "apache-airflow", "docker-stack", diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index 2a0435a831639..8a9e0ca0a1d1a 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -35,8 +35,8 @@ .breeze-build-docs-r1 { fill: #c5c8c6;font-weight: bold } .breeze-build-docs-r2 { fill: #c5c8c6 } .breeze-build-docs-r3 { fill: #d0b344;font-weight: bold } -.breeze-build-docs-r4 { fill: #68a0b3;font-weight: bold } -.breeze-build-docs-r5 { fill: #868887 } +.breeze-build-docs-r4 { fill: #868887 } +.breeze-build-docs-r5 { fill: #68a0b3;font-weight: bold } .breeze-build-docs-r6 { fill: #98a84b;font-weight: bold } .breeze-build-docs-r7 { fill: #8d7b39 } @@ -235,65 +235,65 @@ -Usage: breeze build-docs [OPTIONS] +Usage: breeze build-docs [OPTIONS] Build documentation in the container. -╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--docs-only-dOnly build documentation.│ -│--spellcheck-only-sOnly run spell checking.│ -│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts    │ -│before the build - useful for a clean build.                                               │ -│--for-productionBuilds documentation for official release i.e. all links point to stable version. Implies  │ -│--clean-build│ -│--package-filterList of packages to consider.                                                              │ -│(apache-airflow | apache-airflow-providers-airbyte | apache-airflow-providers-alibaba |    │ -│apache-airflow-providers-amazon | apache-airflow-providers-apache-beam |                   │ -│apache-airflow-providers-apache-cassandra | apache-airflow-providers-apache-drill |        │ -│apache-airflow-providers-apache-druid | apache-airflow-providers-apache-flink |            │ -│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |              │ -│apache-airflow-providers-apache-impala | apache-airflow-providers-apache-kylin |           │ -│apache-airflow-providers-apache-livy | apache-airflow-providers-apache-pig |               │ -│apache-airflow-providers-apache-pinot | apache-airflow-providers-apache-spark |            │ -│apache-airflow-providers-apache-sqoop | apache-airflow-providers-arangodb |                │ -│apache-airflow-providers-asana | apache-airflow-providers-atlassian-jira |                 │ -│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                      │ -│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |           │ -│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                   │ -│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                   │ -│apache-airflow-providers-discord | apache-airflow-providers-docker |                       │ -│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                 │ -│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                         │ -│apache-airflow-providers-github | apache-airflow-providers-google |                        │ -│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                       │ -│apache-airflow-providers-http | apache-airflow-providers-imap |                            │ -│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                        │ -│apache-airflow-providers-jenkins | apache-airflow-providers-microsoft-azure |              │ -│apache-airflow-providers-microsoft-mssql | apache-airflow-providers-microsoft-psrp |       │ -│apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |                │ -│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                          │ -│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                        │ -│apache-airflow-providers-opsgenie | apache-airflow-providers-oracle |                      │ -│apache-airflow-providers-pagerduty | apache-airflow-providers-papermill |                  │ -│apache-airflow-providers-plexus | apache-airflow-providers-postgres |                      │ -│apache-airflow-providers-presto | apache-airflow-providers-qubole |                        │ -│apache-airflow-providers-redis | apache-airflow-providers-salesforce |                     │ -│apache-airflow-providers-samba | apache-airflow-providers-segment |                        │ -│apache-airflow-providers-sendgrid | apache-airflow-providers-sftp |                        │ -│apache-airflow-providers-singularity | apache-airflow-providers-slack |                    │ -│apache-airflow-providers-smtp | apache-airflow-providers-snowflake |                       │ -│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                           │ -│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                      │ -│apache-airflow-providers-telegram | apache-airflow-providers-trino |                       │ -│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                       │ -│apache-airflow-providers-zendesk | docker-stack | helm-chart)                              │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Doc flags ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--docs-only-dOnly build documentation.│ +│--spellcheck-only-sOnly run spell checking.│ +│--clean-buildClean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts    │ +│before the build - useful for a clean build.                                               │ +│--for-productionBuilds documentation for official release i.e. all links point to stable version. Implies  │ +│--clean-build│ +│--package-filterList of packages to consider.                                                              │ +│(apache-airflow | apache-airflow-providers-airbyte | apache-airflow-providers-alibaba |    │ +│apache-airflow-providers-amazon | apache-airflow-providers-apache-beam |                   │ +│apache-airflow-providers-apache-cassandra | apache-airflow-providers-apache-drill |        │ +│apache-airflow-providers-apache-druid | apache-airflow-providers-apache-flink |            │ +│apache-airflow-providers-apache-hdfs | apache-airflow-providers-apache-hive |              │ +│apache-airflow-providers-apache-impala | apache-airflow-providers-apache-kylin |           │ +│apache-airflow-providers-apache-livy | apache-airflow-providers-apache-pig |               │ +│apache-airflow-providers-apache-pinot | apache-airflow-providers-apache-spark |            │ +│apache-airflow-providers-apache-sqoop | apache-airflow-providers-arangodb |                │ +│apache-airflow-providers-asana | apache-airflow-providers-atlassian-jira |                 │ +│apache-airflow-providers-celery | apache-airflow-providers-cloudant |                      │ +│apache-airflow-providers-cncf-kubernetes | apache-airflow-providers-common-sql |           │ +│apache-airflow-providers-databricks | apache-airflow-providers-datadog |                   │ +│apache-airflow-providers-dbt-cloud | apache-airflow-providers-dingding |                   │ +│apache-airflow-providers-discord | apache-airflow-providers-docker |                       │ +│apache-airflow-providers-elasticsearch | apache-airflow-providers-exasol |                 │ +│apache-airflow-providers-facebook | apache-airflow-providers-ftp |                         │ +│apache-airflow-providers-github | apache-airflow-providers-google |                        │ +│apache-airflow-providers-grpc | apache-airflow-providers-hashicorp |                       │ +│apache-airflow-providers-http | apache-airflow-providers-imap |                            │ +│apache-airflow-providers-influxdb | apache-airflow-providers-jdbc |                        │ +│apache-airflow-providers-jenkins | apache-airflow-providers-microsoft-azure |              │ +│apache-airflow-providers-microsoft-mssql | apache-airflow-providers-microsoft-psrp |       │ +│apache-airflow-providers-microsoft-winrm | apache-airflow-providers-mongo |                │ +│apache-airflow-providers-mysql | apache-airflow-providers-neo4j |                          │ +│apache-airflow-providers-odbc | apache-airflow-providers-openfaas |                        │ +│apache-airflow-providers-opsgenie | apache-airflow-providers-oracle |                      │ +│apache-airflow-providers-pagerduty | apache-airflow-providers-papermill |                  │ +│apache-airflow-providers-plexus | apache-airflow-providers-postgres |                      │ +│apache-airflow-providers-presto | apache-airflow-providers-qubole |                        │ +│apache-airflow-providers-redis | apache-airflow-providers-salesforce |                     │ +│apache-airflow-providers-samba | apache-airflow-providers-segment |                        │ +│apache-airflow-providers-sendgrid | apache-airflow-providers-sftp |                        │ +│apache-airflow-providers-singularity | apache-airflow-providers-slack |                    │ +│apache-airflow-providers-smtp | apache-airflow-providers-snowflake |                       │ +│apache-airflow-providers-sqlite | apache-airflow-providers-ssh |                           │ +│apache-airflow-providers-tableau | apache-airflow-providers-tabular |                      │ +│apache-airflow-providers-telegram | apache-airflow-providers-trino |                       │ +│apache-airflow-providers-vertica | apache-airflow-providers-yandex |                       │ +│apache-airflow-providers-zendesk | docker-stack | helm-chart)                              │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index c6d4cabe9eea7..71f1cdbf178f6 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -35,8 +35,8 @@ .breeze-release-management-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-r2 { fill: #c5c8c6 } .breeze-release-management-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-r5 { fill: #868887 } +.breeze-release-management-r4 { fill: #868887 } +.breeze-release-management-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-r6 { fill: #98a84b;font-weight: bold } @@ -105,22 +105,22 @@ -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare and manage Airflow releases -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│generate-constraints              Generates pinned constraint files with all extras from setup.py in parallel.    │ -│generate-issue-content-providers  Generates content for issue to test the release.                                │ -│prepare-airflow-package           Prepare sdist/whl package of Airflow.                                           │ -│prepare-provider-documentation    Prepare CHANGELOG, README and COMMITS information for providers.                │ -│prepare-provider-packages         Prepare sdist/whl packages of Airflow Providers.                                │ -│release-prod-images               Release production images to DockerHub (needs DockerHub permissions).           │ -│verify-provider-packages          Verifies if all provider code is following expectations for providers.          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-constraints              Generates pinned constraint files with all extras from setup.py in parallel.    │ +│generate-issue-content-providers  Generates content for issue to test the release.                                │ +│prepare-airflow-package           Prepare sdist/whl package of Airflow.                                           │ +│prepare-provider-documentation    Prepare CHANGELOG, README and COMMITS information for providers.                │ +│prepare-provider-packages         Prepare sdist/whl packages of Airflow Providers.                                │ +│release-prod-images               Release production images to DockerHub (needs DockerHub permissions).           │ +│verify-provider-packages          Verifies if all provider code is following expectations for providers.          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index ace464046a763..e7ceb94b0e0a0 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -35,8 +35,8 @@ .breeze-release-management-generate-issue-content-providers-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-generate-issue-content-providers-r2 { fill: #c5c8c6 } .breeze-release-management-generate-issue-content-providers-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-generate-issue-content-providers-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-generate-issue-content-providers-r5 { fill: #868887 } +.breeze-release-management-generate-issue-content-providers-r4 { fill: #868887 } +.breeze-release-management-generate-issue-content-providers-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-generate-issue-content-providers-r6 { fill: #8d7b39 } .breeze-release-management-generate-issue-content-providers-r7 { fill: #98a84b;font-weight: bold } @@ -160,7 +160,7 @@ -Usage: breeze release-management generate-issue-content-providers [OPTIONS] [airbyte | alibaba | amazon | apache.beam +Usage: breeze release-management generate-issue-content-providers [OPTIONS] [airbyte | alibaba | amazon | apache.beam                                                                   | apache.cassandra | apache.drill | apache.druid |                                                                   apache.flink | apache.hdfs | apache.hive |                                                                   apache.impala | apache.kylin | apache.livy | @@ -181,19 +181,19 @@ Generates content for issue to test the release. -╭─ Generate issue content flags ───────────────────────────────────────────────────────────────────────────────────────╮ -│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ -│variable set. Can be generated with:                                                    │ -│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status     │ -│(TEXT)                                                                                  │ -│--suffixSuffix to add to the version prepared(TEXT)│ -│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ -│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ -│--disable-progressDisable progress bar│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Generate issue content flags ───────────────────────────────────────────────────────────────────────────────────────╮ +│--github-tokenGitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env     │ +│variable set. Can be generated with:                                                    │ +│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status     │ +│(TEXT)                                                                                  │ +│--suffixSuffix to add to the version prepared(TEXT)│ +│--only-available-in-distOnly consider package ids with packages prepared in the dist folder│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue.(TEXT)│ +│--disable-progressDisable progress bar│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index 807577a6ef118..c834b84177d88 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -35,8 +35,8 @@ .breeze-release-management-prepare-provider-documentation-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r2 { fill: #c5c8c6 } .breeze-release-management-prepare-provider-documentation-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-prepare-provider-documentation-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-prepare-provider-documentation-r5 { fill: #868887 } +.breeze-release-management-prepare-provider-documentation-r4 { fill: #868887 } +.breeze-release-management-prepare-provider-documentation-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r6 { fill: #98a84b;font-weight: bold } .breeze-release-management-prepare-provider-documentation-r7 { fill: #8d7b39 } @@ -154,7 +154,7 @@ -Usage: breeze release-management prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +Usage: breeze release-management prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam |                                                                 apache.cassandra | apache.drill | apache.druid |                                                                 apache.flink | apache.hdfs | apache.hive |                                                                 apache.impala | apache.kylin | apache.livy | @@ -173,19 +173,19 @@                                                                 snowflake | sqlite | ssh | tableau | tabular |                                                                 telegram | trino | vertica | yandex | zendesk]... -Prepare CHANGELOG, README and COMMITS information for providers. +Prepare CHANGELOG, README and COMMITS information for providers. -╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--base-branch(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--base-branch(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index d75838e2621eb..0dd411719e858 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -35,8 +35,8 @@ .breeze-release-management-prepare-provider-packages-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-prepare-provider-packages-r2 { fill: #c5c8c6 } .breeze-release-management-prepare-provider-packages-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-prepare-provider-packages-r4 { fill: #68a0b3;font-weight: bold } -.breeze-release-management-prepare-provider-packages-r5 { fill: #868887 } +.breeze-release-management-prepare-provider-packages-r4 { fill: #868887 } +.breeze-release-management-prepare-provider-packages-r5 { fill: #68a0b3;font-weight: bold } .breeze-release-management-prepare-provider-packages-r6 { fill: #8d7b39 } .breeze-release-management-prepare-provider-packages-r7 { fill: #98a84b;font-weight: bold } @@ -154,7 +154,7 @@ -Usage: breeze release-management prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | +Usage: breeze release-management prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam |                                                            apache.cassandra | apache.drill | apache.druid |                                                            apache.flink | apache.hdfs | apache.hive | apache.impala |                                                            apache.kylin | apache.livy | apache.pig | apache.pinot | @@ -174,18 +174,18 @@ Prepare sdist/whl packages of Airflow Providers. -╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Package flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--package-formatFormat of packages.(wheel | sdist | both)[default: wheel]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--package-list-fileRead list of packages from text file (one package per line).(FILENAME)│ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 57e363b10daa6..3f465fe8ff7ec 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -63,6 +63,7 @@ SKIP_CONSTRAINTS SKIP_ENVIRONMENT_INITIALIZATION SKIP_PROVIDER_TESTS SKIP_SSH_SETUP +SUSPENDED_PROVIDERS_FOLDERS TEST_TIMEOUT TEST_TYPE UPGRADE_BOTO diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 28915019da3e6..b60b25850f1c1 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -76,6 +76,7 @@ services: - SKIP_ENVIRONMENT_INITIALIZATION=${SKIP_ENVIRONMENT_INITIALIZATION} - SKIP_PROVIDER_TESTS=${SKIP_PROVIDER_TESTS} - SKIP_SSH_SETUP=${SKIP_SSH_SETUP} + - SUSPENDED_PROVIDERS_FOLDERS=${SUSPENDED_PROVIDERS_FOLDERS} - TEST_TYPE=${TEST_TYPE} - TEST_TIMEOUT=${TEST_TIMEOUT} - UPGRADE_BOTO=${UPGRADE_BOTO} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index b6be4cb05c58b..482f8a0b9e7d0 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -62,6 +62,7 @@ SKIP_CONSTRAINTS="false" SKIP_SSH_SETUP="true" SKIP_ENVIRONMENT_INITIALIZATION="false" SKIP_PROVIDER_TESTS="false" +SUSPENDED_PROVIDERS_FOLDERS="" TEST_TYPE= UPGRADE_BOTO="false" UPGRADE_TO_NEWER_DEPENDENCIES="false" diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 5c9de455b97f0..2c6ac05bfe98a 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -45,6 +45,9 @@ get_ci_image_for_pre_commits, run_command, ) + from airflow_breeze.utils.suspended_providers import get_suspended_providers_folders + + suspended_providers_folders = get_suspended_providers_folders() files_to_test = filter_out_providers_on_non_main_branch(sys.argv[1:]) if files_to_test == ["--namespace-packages"]: @@ -61,6 +64,8 @@ "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", + f"SUSPENDED_PROVIDERS_FOLDERS={' '.join(suspended_providers_folders)}", + "-e", "BACKEND=sqlite", "--pull", "never", diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py index b3782955a64d5..a66bb7f05ba97 100755 --- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py @@ -47,6 +47,9 @@ CROSS_PROVIDERS_DEPS = "cross-providers-deps" DEPS = "deps" +SUSPENDED = "suspended" + +suspended_paths: list[str] = [] ALL_DEPENDENCIES: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list)) @@ -97,7 +100,11 @@ def find_all_providers_and_provider_files(): provider_name = str(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR)).replace( os.sep, "." ) - ALL_PROVIDERS[provider_name] = yaml.safe_load(provider_file.read_text()) + provider_info = yaml.safe_load(provider_file.read_text()) + if not provider_info["suspended"]: + ALL_PROVIDERS[provider_name] = provider_info + else: + suspended_paths.append(provider_file.parent.relative_to(AIRFLOW_PROVIDERS_DIR).as_posix()) path = Path(root, filename) if path.is_file() and path.name.endswith(".py"): ALL_PROVIDER_FILES.append(Path(root, filename)) @@ -120,6 +127,9 @@ def get_provider_id_from_import(import_name: str, file_path: Path) -> str | None relative_provider_import = import_name[len(AIRFLOW_PROVIDERS_IMPORT_PREFIX) :] provider_id = get_provider_id_from_relative_import_or_file(relative_provider_import) if provider_id is None: + relative_path_from_import = relative_provider_import.replace(".", os.sep) + if any(relative_path_from_import.startswith(suspended_path) for suspended_path in suspended_paths): + return None warnings.append(f"We could not determine provider id from import {import_name} in {file_path}") return provider_id @@ -146,7 +156,10 @@ def get_provider_id_from_file_name(file_path: Path) -> str | None: return None provider_id = get_provider_id_from_relative_import_or_file(str(relative_path)) if provider_id is None and file_path.name not in ["__init__.py", "get_provider_info.py"]: - warnings.append(f"We had a problem to classify the file {file_path} to a provider") + if any(relative_path.as_posix().startswith(suspended_path) for suspended_path in suspended_paths): + return None + else: + warnings.append(f"We had a problem to classify the file {file_path} to a provider") return provider_id @@ -168,13 +181,11 @@ def check_if_different_provider_used(file_path: Path) -> None: num_files = len(ALL_PROVIDER_FILES) num_providers = len(ALL_PROVIDERS) console.print(f"Found {len(ALL_PROVIDERS)} providers with {len(ALL_PROVIDER_FILES)} Python files.") - for file in ALL_PROVIDER_FILES: check_if_different_provider_used(file) - for provider, provider_yaml_content in ALL_PROVIDERS.items(): - ALL_DEPENDENCIES[provider][DEPS].extend(provider_yaml_content["dependencies"]) - + if not provider_yaml_content.get("suspended"): + ALL_DEPENDENCIES[provider][DEPS].extend(provider_yaml_content["dependencies"]) if warnings: console.print("[yellow]Warnings!\n") for warning in warnings: diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 93f133036c21d..12f70257e4b43 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -317,6 +317,17 @@ EXTRA_PYTEST_ARGS=( "-rfEX" ) +if [[ ${SUSPENDED_PROVIDERS_FOLDERS=} != "" ]]; then + for provider in ${SUSPENDED_PROVIDERS_FOLDERS=}; do + echo "Skipping tests for suspended provider: ${provider}" + EXTRA_PYTEST_ARGS+=( + "--ignore=tests/providers/${provider}" + "--ignore=tests/system/providers/${provider}" + "--ignore=tests/integration/providers/${provider}" + ) + done +fi + if [[ "${TEST_TYPE}" == "Helm" ]]; then _cpus="$(grep -c 'cpu[0-9]' /proc/stat)" echo "Running tests with ${_cpus} CPUs in parallel" diff --git a/scripts/in_container/remove_arm_packages.py b/scripts/in_container/remove_arm_packages.py index 28bfad48a1f69..0966a94ed72a0 100644 --- a/scripts/in_container/remove_arm_packages.py +++ b/scripts/in_container/remove_arm_packages.py @@ -25,7 +25,7 @@ from rich.console import Console AIRFLOW_SOURCES_ROOT = Path(__file__).parents[2].resolve() - +AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" console = Console(width=400, color_system="standard") diff --git a/scripts/in_container/run_mypy.sh b/scripts/in_container/run_mypy.sh index 3dac50faf7810..cee381d8cb2cc 100755 --- a/scripts/in_container/run_mypy.sh +++ b/scripts/in_container/run_mypy.sh @@ -20,4 +20,16 @@ . "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" export PYTHONPATH=${AIRFLOW_SOURCES} -mypy "${@}" +ADDITIONAL_MYPY_OPTIONS=() + +if [[ ${SUSPENDED_PROVIDERS_FOLDERS=} != "" ]]; +then + for folder in ${SUSPENDED_PROVIDERS_FOLDERS=} + do + ADDITIONAL_MYPY_OPTIONS+=( + "--exclude" "airflow/providers/${folder}/*" + "--exclude" "tests/providers/${folder}/*" + ) + done +fi +mypy "${ADDITIONAL_MYPY_OPTIONS[@]}" "${@}" diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index b6fc7c41c0f3b..9f821ce74dd58 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -60,6 +60,10 @@ console = Console(width=400, color_system="standard") +suspended_providers: set[str] = set() +suspended_logos: set[str] = set() +suspended_integrations: set[str] = set() + def _filepath_to_module(filepath: pathlib.Path) -> str: p = filepath.resolve().relative_to(ROOT_DIR).as_posix() @@ -85,7 +89,14 @@ def _load_package_data(package_paths: Iterable[str]): jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: raise Exception(f"Unable to parse: {rel_path}.") - result[rel_path] = provider + if not provider.get("suspended"): + result[rel_path] = provider + else: + suspended_providers.add(provider["package-name"]) + for integration in provider["integrations"]: + suspended_integrations.add(integration["integration-name"]) + if "logo" in integration: + suspended_logos.add(integration["logo"]) return result @@ -113,7 +124,7 @@ def check_integration_duplicates(yaml_files: dict[str, dict]): sys.exit(3) -def assert_sets_equal(set1, set2): +def assert_sets_equal(set1, set2, allow_extra_in_set2=False): try: difference1 = set1.difference(set2) except TypeError as e: @@ -128,21 +139,19 @@ def assert_sets_equal(set1, set2): except AttributeError as e: raise AssertionError(f"second argument does not support set difference: {e}") - if not (difference1 or difference2): - return - - lines = [] - if difference1: - lines.append(" -- Items in the left set but not the right:") - for item in sorted(difference1): - lines.append(f" {item!r}") - if difference2: - lines.append(" -- Items in the right set but not the left:") - for item in sorted(difference2): - lines.append(f" {item!r}") + if difference1 or (difference2 and not allow_extra_in_set2): + lines = [] + if difference1: + lines.append(" -- Items in the left set but not the right:") + for item in sorted(difference1): + lines.append(f" {item!r}") + if difference2 and not allow_extra_in_set2: + lines.append(" -- Items in the right set but not the left:") + for item in sorted(difference2): + lines.append(f" {item!r}") - standard_msg = "\n".join(lines) - raise AssertionError(standard_msg) + standard_msg = "\n".join(lines) + raise AssertionError(standard_msg) class ObjectType(Enum): @@ -166,7 +175,7 @@ def check_if_object_exist(object_name: str, resource_type: str, yaml_file_path: except Exception as e: errors.append( f"The `{object_name}` object in {resource_type} list in {yaml_file_path} does not exist " - f"or is not a class: {e}" + f"or is not a {object_type.value}: {e}" ) else: errors.append( @@ -357,7 +366,7 @@ def check_invalid_integration(yaml_files: dict[str, dict]): ): resource_data = provider_data.get("transfers", []) current_names = {r[key] for r in resource_data} - invalid_names = current_names - all_integration_names + invalid_names = current_names - all_integration_names - suspended_integrations if invalid_names: errors.append( f"Incorrect content of key 'transfers/{key}' in file: {yaml_file_path}. " @@ -384,19 +393,26 @@ def check_doc_files(yaml_files: dict[str, dict]): current_doc_urls.extend( op["how-to-guide"] for op in provider["transfers"] if "how-to-guide" in op ) - + console.print("[yellow]Suspended providers:[/]") + console.print(suspended_providers) expected_doc_urls = { f"/docs/{f.relative_to(DOCS_DIR).as_posix()}" for f in DOCS_DIR.glob("apache-airflow-providers-*/operators/**/*.rst") - if f.name != "index.rst" and "_partials" not in f.parts + if f.name != "index.rst" + and "_partials" not in f.parts + and not any(f.relative_to(DOCS_DIR).as_posix().startswith(s) for s in suspended_providers) } | { f"/docs/{f.relative_to(DOCS_DIR).as_posix()}" for f in DOCS_DIR.glob("apache-airflow-providers-*/operators.rst") + if not any(f.relative_to(DOCS_DIR).as_posix().startswith(s) for s in suspended_providers) } + console.print("[yellow]Suspended logos:[/]") + console.print(suspended_logos) expected_logo_urls = { f"/{f.relative_to(DOCS_DIR).as_posix()}" for f in DOCS_DIR.glob("integration-logos/**/*") if f.is_file() + and not any(f"/{f.relative_to(DOCS_DIR).as_posix()}".startswith(s) for s in suspended_logos) } try: @@ -433,7 +449,8 @@ def check_providers_are_mentioned_in_issue_template(yaml_files: dict[str, dict]) f" -- Checking providers: present in code (left), " f"mentioned in {PROVIDER_ISSUE_TEMPLATE_PATH} (right)" ) - assert_sets_equal(set(short_provider_names), set(all_mentioned_providers)) + # in case of suspended providers, we still want to have them in the issue template + assert_sets_equal(set(short_provider_names), set(all_mentioned_providers), allow_extra_in_set2=True) except AssertionError as ex: print(ex) sys.exit(1) diff --git a/setup.py b/setup.py index 9989003707298..1404aba0626f6 100644 --- a/setup.py +++ b/setup.py @@ -369,6 +369,7 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve "ipdb", "jira", "jsondiff", + "jsonpath_ng>=1.5.3", "mongomock", "moto[cloudformation, glue]>=4.0", "paramiko", @@ -410,6 +411,8 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve def get_provider_dependencies(provider_name: str) -> list[str]: + if provider_name not in PROVIDER_DEPENDENCIES: + return [] return PROVIDER_DEPENDENCIES[provider_name][DEPS] @@ -544,7 +547,7 @@ def add_extras_for_all_deprecated_aliases() -> None: for alias, extra in EXTRAS_DEPRECATED_ALIASES.items(): dependencies = EXTRAS_DEPENDENCIES.get(extra) if extra != "" else [] if dependencies is None: - raise Exception(f"The extra {extra} is missing for deprecated alias {alias}") + continue EXTRAS_DEPENDENCIES[alias] = dependencies @@ -599,6 +602,8 @@ def add_all_deprecated_provider_packages() -> None: def get_all_db_dependencies() -> list[str]: _all_db_reqs: set[str] = set() for provider in ALL_DB_PROVIDERS: + if provider not in PROVIDER_DEPENDENCIES: + continue for req in PROVIDER_DEPENDENCIES[provider][DEPS]: _all_db_reqs.add(req) return list(_all_db_reqs) diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py index 9fc3e18251145..da07bb3d27109 100644 --- a/tests/always/test_example_dags.py +++ b/tests/always/test_example_dags.py @@ -18,38 +18,64 @@ import os from glob import glob +from pathlib import Path import pytest from airflow.models import DagBag +from airflow.utils import yaml from tests.test_utils.asserts import assert_queries_count -ROOT_FOLDER = os.path.realpath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) -) +AIRFLOW_SOURCES_ROOT = Path(__file__).resolve().parents[3] +AIRFLOW_PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" NO_DB_QUERY_EXCEPTION = ["/airflow/example_dags/example_subdag_operator.py"] -def example_dags(): +def get_suspended_providers_folders() -> list[str]: + """ + Returns a list of suspended providers folders that should be + skipped when running tests (without any prefix - for example apache/beam, yandex, google etc.). + """ + suspended_providers = [] + for provider_path in AIRFLOW_PROVIDERS_ROOT.glob("**/provider.yaml"): + provider_yaml = yaml.safe_load(provider_path.read_text()) + if provider_yaml.get("suspended"): + suspended_providers.append( + provider_path.parent.relative_to(AIRFLOW_SOURCES_ROOT) + .as_posix() + .replace("airflow/providers/", "") + ) + return suspended_providers + + +def example_not_suspended_dags(): example_dirs = ["airflow/**/example_dags/example_*.py", "tests/system/providers/**/example_*.py"] + suspended_providers_folders = get_suspended_providers_folders() + possible_prefixes = ["airflow/providers/", "tests/system/providers/"] + suspended_providers_folders = [ + f"{prefix}{provider}" for prefix in possible_prefixes for provider in suspended_providers_folders + ] for example_dir in example_dirs: - yield from glob(f"{ROOT_FOLDER}/{example_dir}", recursive=True) + candidates = glob(f"{AIRFLOW_SOURCES_ROOT.as_posix()}/{example_dir}", recursive=True) + for candidate in candidates: + if not any(candidate.startswith(s) for s in suspended_providers_folders): + yield candidate def example_dags_except_db_exception(): return [ dag_file - for dag_file in example_dags() + for dag_file in example_not_suspended_dags() if any(not dag_file.endswith(e) for e in NO_DB_QUERY_EXCEPTION) ] def relative_path(path): - return os.path.relpath(path, ROOT_FOLDER) + return os.path.relpath(path, AIRFLOW_SOURCES_ROOT.as_posix()) -@pytest.mark.parametrize("example", example_dags(), ids=relative_path) +@pytest.mark.parametrize("example", example_not_suspended_dags(), ids=relative_path) def test_should_be_importable(example): dagbag = DagBag( dag_folder=example,