diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 55479d1aa18cb..69c70392a5a25 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -102,14 +102,7 @@ labelPRBasedOnFilePath: - providers/cloudant/** provider:cncf-kubernetes: - - airflow/example_dags/example_kubernetes_executor.py - - airflow/example_dags/example_local_kubernetes_executor.py - - providers/src/airflow/providers/cncf/kubernetes/**/* - - providers/celery/src/airflow/providers/celery/executors/celery_kubernetes_executor.py - - docs/apache-airflow-providers-cncf-kubernetes/**/* - - kubernetes_tests/**/* - - providers/tests/cncf/kubernetes/**/* - - providers/tests/system/cncf/kubernetes/**/* + - providers/cncf/kubernetes/** provider:cohere: - providers/cohere/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8e1b71934903e..e58166f9d383d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -665,7 +665,7 @@ repos: ^providers/fab/docs/auth-manager/webserver-authentication.rst$| ^providers/google/docs/operators/cloud/kubernetes_engine.rst$| ^docs/apache-airflow-providers-microsoft-azure/connections/azure_cosmos.rst$| - ^docs/apache-airflow-providers-cncf-kubernetes/operators.rst$| + ^providers/cncf/kubernetes/docs/operators.rst$| ^docs/conf.py$| ^docs/exts/removemarktransform.py$| ^newsfragments/41761.significant.rst$| diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index 92a0895cc131d..ac6d070a4d4e3 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -601,7 +601,7 @@ def _rebuild_k8s_image( COPY --chown=airflow:0 airflow/example_dags/ /opt/airflow/dags/ -COPY --chown=airflow:0 providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ +COPY --chown=airflow:0 providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/ /opt/airflow/pod_templates/ ENV GUNICORN_CMD_ARGS='--preload' AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL=0 """ diff --git a/docs/.gitignore b/docs/.gitignore index 679773f57728c..4f44a2d9d9799 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -19,6 +19,7 @@ apache-airflow-providers-asana apache-airflow-providers-atlassian-jira apache-airflow-providers-celery apache-airflow-providers-cloudant +apache-airflow-providers-cncf-kubernetes apache-airflow-providers-cohere apache-airflow-providers-common-compat apache-airflow-providers-common-io diff --git a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst b/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst deleted file mode 100644 index 6ad86cec6753c..0000000000000 --- a/docs/apache-airflow-providers-cncf-kubernetes/changelog.rst +++ /dev/null @@ -1,25 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE - OVERWRITTEN WHEN PREPARING PACKAGES. - - .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE - `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY - -.. include:: ../../providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst diff --git a/docs/apache-airflow/tutorial/taskflow.rst b/docs/apache-airflow/tutorial/taskflow.rst index 8c77c5ddc6718..455465f545c47 100644 --- a/docs/apache-airflow/tutorial/taskflow.rst +++ b/docs/apache-airflow/tutorial/taskflow.rst @@ -338,7 +338,7 @@ Below is an example of using the ``@task.kubernetes`` decorator to run a Python .. _taskflow/kubernetes_example: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py :language: python :dedent: 4 :start-after: [START howto_operator_kubernetes] diff --git a/providers/cncf/kubernetes/README.rst b/providers/cncf/kubernetes/README.rst new file mode 100644 index 0000000000000..8cdf33e1696e7 --- /dev/null +++ b/providers/cncf/kubernetes/README.rst @@ -0,0 +1,67 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +Package ``apache-airflow-providers-cncf-kubernetes`` + +Release: ``10.1.0`` + + +`Kubernetes `__ + + +Provider package +---------------- + +This is a provider package for ``cncf.kubernetes`` provider. All classes for this provider package +are in ``airflow.providers.cncf.kubernetes`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-cncf-kubernetes`` + +The package supports the following python versions: 3.9,3.10,3.11,3.12 + +Requirements +------------ + +====================== ===================== +PIP package Version required +====================== ===================== +``aiofiles`` ``>=23.2.0`` +``apache-airflow`` ``>=2.9.0`` +``asgiref`` ``>=3.5.2`` +``cryptography`` ``>=41.0.0`` +``kubernetes`` ``>=29.0.0,<=31.0.0`` +``kubernetes_asyncio`` ``>=29.0.0,<=31.0.0`` +``google-re2`` ``>=1.0`` +====================== ===================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/src/airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt b/providers/cncf/kubernetes/docs/.latest-doc-only-change.txt similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/.latest-doc-only-change.txt rename to providers/cncf/kubernetes/docs/.latest-doc-only-change.txt diff --git a/providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst b/providers/cncf/kubernetes/docs/changelog.rst similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/CHANGELOG.rst rename to providers/cncf/kubernetes/docs/changelog.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/cli-ref.rst b/providers/cncf/kubernetes/docs/cli-ref.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/cli-ref.rst rename to providers/cncf/kubernetes/docs/cli-ref.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/commits.rst b/providers/cncf/kubernetes/docs/commits.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/commits.rst rename to providers/cncf/kubernetes/docs/commits.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst b/providers/cncf/kubernetes/docs/configurations-ref.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/configurations-ref.rst rename to providers/cncf/kubernetes/docs/configurations-ref.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/connections/kubernetes.rst b/providers/cncf/kubernetes/docs/connections/kubernetes.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/connections/kubernetes.rst rename to providers/cncf/kubernetes/docs/connections/kubernetes.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png b/providers/cncf/kubernetes/docs/img/arch-diag-kubernetes.png similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png rename to providers/cncf/kubernetes/docs/img/arch-diag-kubernetes.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png b/providers/cncf/kubernetes/docs/img/arch-diag-kubernetes2.png similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png rename to providers/cncf/kubernetes/docs/img/arch-diag-kubernetes2.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png b/providers/cncf/kubernetes/docs/img/k8s-failed-pod.png similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png rename to providers/cncf/kubernetes/docs/img/k8s-failed-pod.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png b/providers/cncf/kubernetes/docs/img/k8s-happy-path.png similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png rename to providers/cncf/kubernetes/docs/img/k8s-happy-path.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst b/providers/cncf/kubernetes/docs/index.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/index.rst rename to providers/cncf/kubernetes/docs/index.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst b/providers/cncf/kubernetes/docs/installing-providers-from-sources.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/installing-providers-from-sources.rst rename to providers/cncf/kubernetes/docs/installing-providers-from-sources.rst diff --git a/docs/integration-logos/kubernetes/Kubernetes.png b/providers/cncf/kubernetes/docs/integration-logos/Kubernetes.png similarity index 100% rename from docs/integration-logos/kubernetes/Kubernetes.png rename to providers/cncf/kubernetes/docs/integration-logos/Kubernetes.png diff --git a/docs/integration-logos/kubernetes/Spark-On-Kubernetes.png b/providers/cncf/kubernetes/docs/integration-logos/Spark-On-Kubernetes.png similarity index 100% rename from docs/integration-logos/kubernetes/Spark-On-Kubernetes.png rename to providers/cncf/kubernetes/docs/integration-logos/Spark-On-Kubernetes.png diff --git a/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst b/providers/cncf/kubernetes/docs/kubernetes_executor.rst similarity index 96% rename from docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst rename to providers/cncf/kubernetes/docs/kubernetes_executor.rst index a85a793712885..91f1747e97ab2 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst +++ b/providers/cncf/kubernetes/docs/kubernetes_executor.rst @@ -108,21 +108,21 @@ With these requirements in mind, here are some examples of basic ``pod_template_ Storing DAGs in the image: -.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +.. literalinclude:: /../../providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml :language: yaml :start-after: [START template_with_dags_in_image] :end-before: [END template_with_dags_in_image] Storing DAGs in a ``persistentVolume``: -.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +.. literalinclude:: /../../providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml :language: yaml :start-after: [START template_with_dags_in_volume] :end-before: [END template_with_dags_in_volume] Pulling DAGs from ``git``: -.. literalinclude:: /../../providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml +.. literalinclude:: /../../providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml :language: yaml :start-after: [START git_sync_template] :end-before: [END git_sync_template] diff --git a/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst b/providers/cncf/kubernetes/docs/local_kubernetes_executor.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst rename to providers/cncf/kubernetes/docs/local_kubernetes_executor.rst diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst b/providers/cncf/kubernetes/docs/operators.rst similarity index 95% rename from docs/apache-airflow-providers-cncf-kubernetes/operators.rst rename to providers/cncf/kubernetes/docs/operators.rst index b9876ea60d0cb..160f9cfccffd9 100644 --- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst +++ b/providers/cncf/kubernetes/docs/operators.rst @@ -102,7 +102,7 @@ Using this method will ensure correctness and type safety. While we have removed almost all Kubernetes convenience classes, we have kept the :class:`~airflow.providers.cncf.kubernetes.secret.Secret` class to simplify the process of generating secret volumes/env variables. -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_cluster_resources] :end-before: [END howto_operator_k8s_cluster_resources] @@ -135,21 +135,21 @@ Create the Secret using ``kubectl``: Then use it in your pod like so: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_private_image] :end-before: [END howto_operator_k8s_private_image] Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_k8s_private_image_async] :end-before: [END howto_operator_k8s_private_image_async] Example to fetch and display container log periodically -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_async_log] :end-before: [END howto_operator_async_log] @@ -168,7 +168,7 @@ alongside the Pod. The Pod must write the XCom value into this location at the ` See the following example on how this occurs: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py :language: python :start-after: [START howto_operator_k8s_write_xcom] :end-before: [END howto_operator_k8s_write_xcom] @@ -177,7 +177,7 @@ See the following example on how this occurs: Also for this action you can use operator in the deferrable mode: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_async.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py :language: python :start-after: [START howto_operator_k8s_write_xcom_async] :end-before: [END howto_operator_k8s_write_xcom_async] @@ -621,7 +621,7 @@ request that dynamically launches this Job. Users can specify a kubeconfig file using the ``config_file`` parameter, otherwise the operator will default to ``~/.kube/config``. It also allows users to supply a template YAML file using the ``job_template_file`` parameter. -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_job] @@ -629,7 +629,7 @@ to ``~/.kube/config``. It also allows users to supply a template YAML file using The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesJobOperator` also supports deferrable mode: -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_job_deferrable] @@ -656,7 +656,7 @@ KubernetesDeleteJobOperator The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesDeleteJobOperator` allows you to delete Jobs on a Kubernetes cluster. -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_delete_k8s_job] @@ -671,7 +671,7 @@ KubernetesPatchJobOperator The :class:`~airflow.providers.cncf.kubernetes.operators.job.KubernetesPatchJobOperator` allows you to update Jobs on a Kubernetes cluster. -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_job.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py :language: python :dedent: 4 :start-after: [START howto_operator_update_job] @@ -686,7 +686,7 @@ KubernetesInstallKueueOperator The :class:`~airflow.providers.cncf.kubernetes.operators.kueue.KubernetesInstallKueueOperator` allows you to install the Kueue component in a Kubernetes cluster -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_kueue.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_kueue_install] @@ -709,7 +709,7 @@ KubernetesStartKueueJobOperator The :class:`~airflow.providers.cncf.kubernetes.operators.kueue.KubernetesStartKueueJobOperator` allows you to start a Kueue job in a Kubernetes cluster -.. exampleinclude:: /../../providers/tests/system/cncf/kubernetes/example_kubernetes_kueue.py +.. exampleinclude:: /../../providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py :language: python :dedent: 4 :start-after: [START howto_operator_k8s_install_kueue] diff --git a/docs/apache-airflow-providers-cncf-kubernetes/redirects.txt b/providers/cncf/kubernetes/docs/redirects.txt similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/redirects.txt rename to providers/cncf/kubernetes/docs/redirects.txt diff --git a/docs/apache-airflow-providers-cncf-kubernetes/security.rst b/providers/cncf/kubernetes/docs/security.rst similarity index 100% rename from docs/apache-airflow-providers-cncf-kubernetes/security.rst rename to providers/cncf/kubernetes/docs/security.rst diff --git a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml b/providers/cncf/kubernetes/provider.yaml similarity index 88% rename from providers/src/airflow/providers/cncf/kubernetes/provider.yaml rename to providers/cncf/kubernetes/provider.yaml index 9d38a70aa14b9..c9d213373fce7 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/provider.yaml +++ b/providers/cncf/kubernetes/provider.yaml @@ -95,38 +95,16 @@ versions: - 1.0.1 - 1.0.0 -dependencies: - - aiofiles>=23.2.0 - - apache-airflow>=2.9.0 - - asgiref>=3.5.2 - - cryptography>=41.0.0 - # The Kubernetes API is known to introduce problems when upgraded to a MAJOR version. Airflow Core - # Uses Kubernetes for Kubernetes executor, and we also know that Kubernetes Python client follows SemVer - # (https://github.com/kubernetes-client/python#compatibility). This is a crucial component of Airflow - # So we should limit it to the next MAJOR version and only deliberately bump the version when we - # tested it, and we know it can be bumped. Bumping this version should also be connected with - # limiting minimum airflow version supported in cncf.kubernetes provider, due to the - # potential breaking changes in Airflow Core as well (kubernetes is added as extra, so Airflow - # core is not hard-limited via install-requirements, only by extra). - - kubernetes>=29.0.0,<=31.0.0 - # The Kubernetes_asyncio package is used for providing Asynchronous (AsyncIO) client library for - # standard Kubernetes API. The version is limited by minimum 18.20.1 because of introducing the ability to - # load kubernetes config file from dictionary in that release and is limited to the next MAJOR version - # (started from current 24.2.2 version) to prevent introducing some problems that could be due to some - # major changes in the package. - - kubernetes_asyncio>=29.0.0,<=31.0.0 - - google-re2>=1.0 - integrations: - integration-name: Kubernetes external-doc-url: https://kubernetes.io/ how-to-guide: - /docs/apache-airflow-providers-cncf-kubernetes/operators.rst - logo: /integration-logos/kubernetes/Kubernetes.png + logo: /docs/integration-logos/Kubernetes.png tags: [software] - integration-name: Spark on Kubernetes external-doc-url: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator - logo: /integration-logos/kubernetes/Spark-On-Kubernetes.png + logo: /docs/integration-logos/Spark-On-Kubernetes.png tags: [software] operators: diff --git a/providers/cncf/kubernetes/pyproject.toml b/providers/cncf/kubernetes/pyproject.toml new file mode 100644 index 0000000000000..32728f7c81e9a --- /dev/null +++ b/providers/cncf/kubernetes/pyproject.toml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.10.1"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-cncf-kubernetes" +version = "10.1.0" +description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow" +readme = "README.rst" +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "cncf.kubernetes", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +requires-python = "~=3.9" + +# The dependencies should be modified in place in the generated file +# Any change in the dependencies is preserved when the file is regenerated +dependencies = [ + "aiofiles>=23.2.0", + "apache-airflow>=2.9.0", + "asgiref>=3.5.2", + "cryptography>=41.0.0", + # The Kubernetes API is known to introduce problems when upgraded to a MAJOR version. Airflow Core + # Uses Kubernetes for Kubernetes executor, and we also know that Kubernetes Python client follows SemVer + # (https://github.com/kubernetes-client/python#compatibility). This is a crucial component of Airflow + # So we should limit it to the next MAJOR version and only deliberately bump the version when we + # tested it, and we know it can be bumped. Bumping this version should also be connected with + # limiting minimum airflow version supported in cncf.kubernetes provider, due to the + # potential breaking changes in Airflow Core as well (kubernetes is added as extra, so Airflow + # core is not hard-limited via install-requirements, only by extra). + "kubernetes>=29.0.0,<=31.0.0", + # The Kubernetes_asyncio package is used for providing Asynchronous (AsyncIO) client library for + # standard Kubernetes API. The version is limited by minimum 18.20.1 because of introducing the ability to + # load kubernetes config file from dictionary in that release and is limited to the next MAJOR version + # (started from current 24.2.2 version) to prevent introducing some problems that could be due to some + # major changes in the package. + "kubernetes_asyncio>=29.0.0,<=31.0.0", + "google-re2>=1.0", +] + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Twitter" = "https://x.com/ApacheAirflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.cncf.kubernetes.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.cncf.kubernetes" + +[tool.pytest.ini_options] +ignore = "tests/system/" diff --git a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/LICENSE b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/src/airflow/providers/cncf/kubernetes/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/backcompat/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py index 9bc821a816d99..2d6f94d6ea018 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py @@ -18,9 +18,8 @@ from __future__ import annotations -from kubernetes.client import ApiClient, models as k8s - from airflow.exceptions import AirflowException +from kubernetes.client import ApiClient, models as k8s def _convert_kube_model_object(obj, new_class): diff --git a/providers/src/airflow/providers/cncf/kubernetes/callbacks.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/callbacks.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/callbacks.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/callbacks.py index d87e8065dbd1a..723de2f94f990 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/callbacks.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/callbacks.py @@ -19,9 +19,10 @@ from enum import Enum from typing import TYPE_CHECKING, Union -import kubernetes.client as k8s import kubernetes_asyncio.client as async_k8s +import kubernetes.client as k8s + if TYPE_CHECKING: from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.utils.context import Context diff --git a/providers/src/airflow/providers/cncf/kubernetes/cli/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/cli/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/cli/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/cli/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py index ee629cc16e9a5..d138fd9c77c00 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py @@ -22,10 +22,6 @@ import sys from datetime import datetime, timedelta -from kubernetes import client -from kubernetes.client.api_client import ApiClient -from kubernetes.client.rest import ApiException - from airflow.models import DagRun, TaskInstance from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig @@ -36,6 +32,9 @@ from airflow.utils import cli as cli_utils, yaml from airflow.utils.cli import get_dag from airflow.utils.providers_configuration_loader import providers_configuration_loaded +from kubernetes import client +from kubernetes.client.api_client import ApiClient +from kubernetes.client.rest import ApiException @cli_utils.action_cli diff --git a/providers/src/airflow/providers/cncf/kubernetes/decorators/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/decorators/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/decorators/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/decorators/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py index cce1ffa8d6c19..c35b492dcabe0 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/decorators/kubernetes.py @@ -26,13 +26,13 @@ from typing import TYPE_CHECKING, Callable import dill -from kubernetes.client import models as k8s from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.python_kubernetes_script import ( write_python_script, ) +from kubernetes.client import models as k8s if TYPE_CHECKING: from airflow.utils.context import Context @@ -119,7 +119,7 @@ def execute(self, context: Context): } write_python_script(jinja_context=jinja_context, filename=script_filename) - self.env_vars = [ + self.env_vars: list[k8s.V1EnvVar] = [ *self.env_vars, k8s.V1EnvVar(name=_PYTHON_SCRIPT_ENV, value=_read_file_contents(script_filename)), k8s.V1EnvVar(name=_PYTHON_INPUT_ENV, value=_read_file_contents(input_filename)), diff --git a/providers/src/airflow/providers/cncf/kubernetes/exceptions.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/exceptions.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/exceptions.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/exceptions.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/executors/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index 0f98f2f7dcb46..845c91e0db37d 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -37,9 +37,10 @@ from typing import TYPE_CHECKING, Any from deprecated import deprecated -from kubernetes.dynamic import DynamicClient from sqlalchemy import select +from kubernetes.dynamic import DynamicClient + try: from airflow.cli.cli_config import ARG_LOGICAL_DATE except ImportError: # 2.x compatibility. @@ -75,8 +76,6 @@ if TYPE_CHECKING: import argparse - from kubernetes import client - from kubernetes.client import models as k8s from sqlalchemy.orm import Session from airflow.executors.base_executor import CommandType @@ -89,6 +88,8 @@ from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ( AirflowKubernetesScheduler, ) + from kubernetes import client + from kubernetes.client import models as k8s # CLI Args ARG_NAMESPACE = Arg( diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py index 15fa954439a9d..1b7917502f0b6 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py @@ -23,8 +23,6 @@ from queue import Empty, Queue from typing import TYPE_CHECKING, Any -from kubernetes import client, watch -from kubernetes.client.rest import ApiException from urllib3.exceptions import ReadTimeoutError from airflow.exceptions import AirflowException @@ -45,15 +43,16 @@ from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.singleton import Singleton from airflow.utils.state import TaskInstanceState +from kubernetes import client, watch +from kubernetes.client.rest import ApiException if TYPE_CHECKING: - from kubernetes.client import Configuration, models as k8s - from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import ( KubernetesJobType, KubernetesResultsType, KubernetesWatchType, ) + from kubernetes.client import Configuration, models as k8s class ResourceVersion(metaclass=Singleton): diff --git a/providers/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py diff --git a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py new file mode 100644 index 0000000000000..53e3d1f831c22 --- /dev/null +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/get_provider_info.py @@ -0,0 +1,361 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-cncf-kubernetes", + "name": "Kubernetes", + "description": "`Kubernetes `__\n", + "state": "ready", + "source-date-epoch": 1734537609, + "versions": [ + "10.1.0", + "10.0.1", + "10.0.0", + "9.0.1", + "9.0.0", + "8.4.2", + "8.4.1", + "8.4.0", + "8.3.4", + "8.3.3", + "8.3.2", + "8.3.1", + "8.3.0", + "8.2.0", + "8.1.1", + "8.1.0", + "8.0.1", + "8.0.0", + "7.14.0", + "7.13.0", + "7.12.0", + "7.11.0", + "7.10.0", + "7.9.0", + "7.8.0", + "7.7.0", + "7.6.0", + "7.5.1", + "7.5.0", + "7.4.2", + "7.4.1", + "7.4.0", + "7.3.0", + "7.2.0", + "7.1.0", + "7.0.0", + "6.1.0", + "6.0.0", + "5.3.0", + "5.2.2", + "5.2.1", + "5.2.0", + "5.1.1", + "5.1.0", + "5.0.0", + "4.4.0", + "4.3.0", + "4.2.0", + "4.1.0", + "4.0.2", + "4.0.1", + "4.0.0", + "3.1.2", + "3.1.1", + "3.1.0", + "3.0.2", + "3.0.1", + "3.0.0", + "2.2.0", + "2.1.0", + "2.0.3", + "2.0.2", + "2.0.1", + "2.0.0", + "1.2.0", + "1.1.0", + "1.0.2", + "1.0.1", + "1.0.0", + ], + "integrations": [ + { + "integration-name": "Kubernetes", + "external-doc-url": "https://kubernetes.io/", + "how-to-guide": ["/docs/apache-airflow-providers-cncf-kubernetes/operators.rst"], + "logo": "/docs/integration-logos/Kubernetes.png", + "tags": ["software"], + }, + { + "integration-name": "Spark on Kubernetes", + "external-doc-url": "https://github.com/GoogleCloudPlatform/spark-on-k8s-operator", + "logo": "/docs/integration-logos/Spark-On-Kubernetes.png", + "tags": ["software"], + }, + ], + "operators": [ + { + "integration-name": "Kubernetes", + "python-modules": [ + "airflow.providers.cncf.kubernetes.operators.custom_object_launcher", + "airflow.providers.cncf.kubernetes.operators.kueue", + "airflow.providers.cncf.kubernetes.operators.pod", + "airflow.providers.cncf.kubernetes.operators.spark_kubernetes", + "airflow.providers.cncf.kubernetes.operators.resource", + "airflow.providers.cncf.kubernetes.operators.job", + ], + } + ], + "sensors": [ + { + "integration-name": "Kubernetes", + "python-modules": ["airflow.providers.cncf.kubernetes.sensors.spark_kubernetes"], + } + ], + "hooks": [ + { + "integration-name": "Kubernetes", + "python-modules": ["airflow.providers.cncf.kubernetes.hooks.kubernetes"], + } + ], + "triggers": [ + { + "integration-name": "Kubernetes", + "python-modules": [ + "airflow.providers.cncf.kubernetes.triggers.pod", + "airflow.providers.cncf.kubernetes.triggers.job", + ], + } + ], + "connection-types": [ + { + "hook-class-name": "airflow.providers.cncf.kubernetes.hooks.kubernetes.KubernetesHook", + "connection-type": "kubernetes", + } + ], + "task-decorators": [ + { + "class-name": "airflow.providers.cncf.kubernetes.decorators.kubernetes.kubernetes_task", + "name": "kubernetes", + } + ], + "config": { + "local_kubernetes_executor": { + "description": "This section only applies if you are using the ``LocalKubernetesExecutor`` in\n``[core]`` section above\n", + "options": { + "kubernetes_queue": { + "description": "Define when to send a task to ``KubernetesExecutor`` when using ``LocalKubernetesExecutor``.\nWhen the queue of a task is the value of ``kubernetes_queue`` (default ``kubernetes``),\nthe task is executed via ``KubernetesExecutor``,\notherwise via ``LocalExecutor``\n", + "version_added": None, + "type": "string", + "example": None, + "default": "kubernetes", + } + }, + }, + "kubernetes_executor": { + "description": None, + "options": { + "api_client_retry_configuration": { + "description": "Kwargs to override the default urllib3 Retry used in the kubernetes API client\n", + "version_added": None, + "type": "string", + "example": '{ "total": 3, "backoff_factor": 0.5 }', + "default": "", + }, + "logs_task_metadata": { + "description": "Flag to control the information added to kubernetes executor logs for better traceability\n", + "version_added": None, + "type": "boolean", + "example": None, + "default": "False", + }, + "pod_template_file": { + "description": "Path to the YAML pod file that forms the basis for KubernetesExecutor workers.\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + "see_also": ":ref:`concepts:pod_template_file`", + }, + "worker_container_repository": { + "description": "The repository of the Kubernetes Image for the Worker to Run\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + }, + "worker_container_tag": { + "description": "The tag of the Kubernetes Image for the Worker to Run\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + }, + "namespace": { + "description": "The Kubernetes namespace where airflow workers should be created. Defaults to ``default``\n", + "version_added": None, + "type": "string", + "example": None, + "default": "default", + }, + "delete_worker_pods": { + "description": "If True, all worker pods will be deleted upon termination\n", + "version_added": None, + "type": "string", + "example": None, + "default": "True", + }, + "delete_worker_pods_on_failure": { + "description": "If False (and delete_worker_pods is True),\nfailed worker pods will not be deleted so users can investigate them.\nThis only prevents removal of worker pods where the worker itself failed,\nnot when the task it ran failed.\n", + "version_added": None, + "type": "string", + "example": None, + "default": "False", + }, + "worker_pod_pending_fatal_container_state_reasons": { + "description": "If the worker pods are in a pending state due to a fatal container\nstate reasons, then fail the task and delete the worker pod\nif delete_worker_pods is True and delete_worker_pods_on_failure is True.\n", + "version_added": "8.1.0", + "type": "string", + "example": None, + "default": "CreateContainerConfigError,ErrImagePull,CreateContainerError,ImageInspectError, InvalidImageName", + }, + "worker_pods_creation_batch_size": { + "description": 'Number of Kubernetes Worker Pod creation calls per scheduler loop.\nNote that the current default of "1" will only launch a single pod\nper-heartbeat. It is HIGHLY recommended that users increase this\nnumber to match the tolerance of their kubernetes cluster for\nbetter performance.\n', + "version_added": None, + "type": "string", + "example": None, + "default": "1", + }, + "multi_namespace_mode": { + "description": "Allows users to launch pods in multiple namespaces.\nWill require creating a cluster-role for the scheduler,\nor use multi_namespace_mode_namespace_list configuration.\n", + "version_added": None, + "type": "boolean", + "example": None, + "default": "False", + }, + "multi_namespace_mode_namespace_list": { + "description": "If multi_namespace_mode is True while scheduler does not have a cluster-role,\ngive the list of namespaces where the scheduler will schedule jobs\nScheduler needs to have the necessary permissions in these namespaces.\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + }, + "in_cluster": { + "description": "Use the service account kubernetes gives to pods to connect to kubernetes cluster.\nIt's intended for clients that expect to be running inside a pod running on kubernetes.\nIt will raise an exception if called from a process not running in a kubernetes environment.\n", + "version_added": None, + "type": "string", + "example": None, + "default": "True", + }, + "cluster_context": { + "description": "When running with in_cluster=False change the default cluster_context or config_file\noptions to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has.\n", + "version_added": None, + "type": "string", + "example": None, + "default": None, + }, + "config_file": { + "description": "Path to the kubernetes configfile to be used when ``in_cluster`` is set to False\n", + "version_added": None, + "type": "string", + "example": None, + "default": None, + }, + "kube_client_request_args": { + "description": "Keyword parameters to pass while calling a kubernetes client core_v1_api methods\nfrom Kubernetes Executor provided as a single line formatted JSON dictionary string.\nList of supported params are similar for all core_v1_apis, hence a single config\nvariable for all apis. See:\nhttps://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + }, + "delete_option_kwargs": { + "description": "Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client\n``core_v1_api`` method when using the Kubernetes Executor.\nThis should be an object and can contain any of the options listed in the ``v1DeleteOptions``\nclass defined here:\nhttps://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19\n", + "version_added": None, + "type": "string", + "example": '{"grace_period_seconds": 10}', + "default": "", + }, + "enable_tcp_keepalive": { + "description": "Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely\nwhen idle connection is time-outed on services like cloud load balancers or firewalls.\n", + "version_added": None, + "type": "boolean", + "example": None, + "default": "True", + }, + "tcp_keep_idle": { + "description": "When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has\nbeen idle for `tcp_keep_idle` seconds.\n", + "version_added": None, + "type": "integer", + "example": None, + "default": "120", + }, + "tcp_keep_intvl": { + "description": "When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond\nto a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds.\n", + "version_added": None, + "type": "integer", + "example": None, + "default": "30", + }, + "tcp_keep_cnt": { + "description": "When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond\nto a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before\na connection is considered to be broken.\n", + "version_added": None, + "type": "integer", + "example": None, + "default": "6", + }, + "verify_ssl": { + "description": "Set this to false to skip verifying SSL certificate of Kubernetes python client.\n", + "version_added": None, + "type": "boolean", + "example": None, + "default": "True", + }, + "ssl_ca_cert": { + "description": "Path to a CA certificate to be used by the Kubernetes client to verify the server's SSL certificate.\n", + "version_added": None, + "type": "string", + "example": None, + "default": "", + }, + "task_publish_max_retries": { + "description": "The Maximum number of retries for queuing the task to the kubernetes scheduler when\nfailing due to Kube API exceeded quota errors before giving up and marking task as failed.\n-1 for unlimited times.\n", + "version_added": None, + "type": "integer", + "example": None, + "default": "0", + }, + }, + }, + }, + "executors": ["airflow.providers.cncf.kubernetes.kubernetes_executor.KubernetesExecutor"], + "dependencies": [ + "aiofiles>=23.2.0", + "apache-airflow>=2.9.0", + "asgiref>=3.5.2", + "cryptography>=41.0.0", + "kubernetes>=29.0.0,<=31.0.0", + "kubernetes_asyncio>=29.0.0,<=31.0.0", + "google-re2>=1.0", + ], + } diff --git a/providers/src/airflow/providers/cncf/kubernetes/hooks/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/hooks/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py index 6a377be3eb27c..114a933e50e35 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/hooks/kubernetes.py @@ -29,9 +29,6 @@ import requests import tenacity from asgiref.sync import sync_to_async -from kubernetes import client, config, utils, watch -from kubernetes.client.models import V1Deployment -from kubernetes.config import ConfigException from kubernetes_asyncio import client as async_client, config as async_config from urllib3.exceptions import HTTPError @@ -46,6 +43,9 @@ container_is_running, ) from airflow.utils import yaml +from kubernetes import client, config, utils, watch +from kubernetes.client.models import V1Deployment +from kubernetes.config import ConfigException if TYPE_CHECKING: from kubernetes.client import V1JobList diff --git a/providers/src/airflow/providers/cncf/kubernetes/k8s_model.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/k8s_model.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/k8s_model.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/k8s_model.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/kube_client.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kube_client.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/kube_client.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kube_client.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/kube_config.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kube_config.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/kube_config.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kube_config.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml diff --git a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py index a353b7a5c8026..2732907886970 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py @@ -23,11 +23,11 @@ from typing import TYPE_CHECKING import pendulum -from kubernetes.client.rest import ApiException from slugify import slugify from airflow.configuration import conf from airflow.providers.cncf.kubernetes.backcompat import get_logical_date_key +from kubernetes.client.rest import ApiException if TYPE_CHECKING: from airflow.models.taskinstancekey import TaskInstanceKey diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/operators/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py index c6d1d14cc624e..33c204b8d3f19 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py @@ -24,8 +24,6 @@ from functools import cached_property import tenacity -from kubernetes.client import CoreV1Api, CustomObjectsApi, models as k8s -from kubernetes.client.rest import ApiException from airflow.exceptions import AirflowException from airflow.providers.cncf.kubernetes.resource_convert.configmap import ( @@ -39,6 +37,8 @@ ) from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager from airflow.utils.log.logging_mixin import LoggingMixin +from kubernetes.client import CoreV1Api, CustomObjectsApi, models as k8s +from kubernetes.client.rest import ApiException def should_retry_start_spark_job(exception: BaseException) -> bool: diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/job.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/job.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/operators/job.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/job.py index 36fa4b92a91be..eb6dfae43162a 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/job.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/job.py @@ -26,10 +26,6 @@ from functools import cached_property from typing import TYPE_CHECKING -from kubernetes.client import BatchV1Api, models as k8s -from kubernetes.client.api_client import ApiClient -from kubernetes.client.rest import ApiException - from airflow.configuration import conf from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -44,6 +40,9 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import EMPTY_XCOM_RESULT, PodNotFoundException from airflow.utils import yaml from airflow.utils.context import Context +from kubernetes.client import BatchV1Api, models as k8s +from kubernetes.client.api_client import ApiClient +from kubernetes.client.rest import ApiException if TYPE_CHECKING: from airflow.utils.context import Context @@ -167,6 +166,7 @@ def execute(self, context: Context): ti.xcom_push(key="job_name", value=self.job.metadata.name) ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace) + self.pod: k8s.V1Pod | None if self.pod is None: self.pod = self.get_or_create_pod( # must set `self.pod` for `on_kill` pod_request_obj=self.pod_request_obj, diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/kueue.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/kueue.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/operators/kueue.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/kueue.py index 64965b34a9526..aa6ebb32ad5c5 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/kueue.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/kueue.py @@ -22,12 +22,11 @@ from collections.abc import Sequence from functools import cached_property -from kubernetes.utils import FailToCreateError - from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook from airflow.providers.cncf.kubernetes.operators.job import KubernetesJobOperator +from kubernetes.utils import FailToCreateError class KubernetesInstallKueueOperator(BaseOperator): @@ -95,6 +94,7 @@ def __init__(self, queue_name: str, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.queue_name = queue_name + self.suspend: bool if self.suspend is False: raise AirflowException( "The `suspend` parameter can't be False. If you want to use Kueue for running Job" diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/operators/pod.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py index 5d1bea11922e8..1d77f459c9863 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py @@ -32,13 +32,10 @@ from functools import cached_property from typing import TYPE_CHECKING, Any, Callable, Literal -import kubernetes import tenacity -from kubernetes.client import CoreV1Api, V1Pod, models as k8s -from kubernetes.client.exceptions import ApiException -from kubernetes.stream import stream from urllib3.exceptions import HTTPError +import kubernetes from airflow.configuration import conf from airflow.exceptions import ( AirflowException, @@ -84,6 +81,9 @@ from airflow.utils import yaml from airflow.utils.helpers import prune_dict, validate_key from airflow.version import version as airflow_version +from kubernetes.client import CoreV1Api, V1Pod, models as k8s +from kubernetes.client.exceptions import ApiException +from kubernetes.stream import stream if TYPE_CHECKING: import jinja2 diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/resource.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/resource.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/operators/resource.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/resource.py index aef972faf26e8..ca2767d48e520 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/resource.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/resource.py @@ -25,7 +25,6 @@ import tenacity import yaml -from kubernetes.utils import create_from_yaml from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -33,6 +32,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import should_retry_creation from airflow.providers.cncf.kubernetes.utils.delete_from import delete_from_yaml from airflow.providers.cncf.kubernetes.utils.k8s_resource_iterator import k8s_resource_iterator +from kubernetes.utils import create_from_yaml if TYPE_CHECKING: from kubernetes.client import ApiClient, CustomObjectsApi diff --git a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py index 583388c6d14d6..d90085e68c44b 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py @@ -21,8 +21,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast -from kubernetes.client import CoreV1Api, CustomObjectsApi, models as k8s - from airflow.exceptions import AirflowException from airflow.providers.cncf.kubernetes import pod_generator from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook, _load_body_to_dict @@ -32,6 +30,7 @@ from airflow.providers.cncf.kubernetes.pod_generator import MAX_LABEL_LEN, PodGenerator from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager from airflow.utils.helpers import prune_dict +from kubernetes.client import CoreV1Api, CustomObjectsApi, models as k8s if TYPE_CHECKING: import jinja2 @@ -117,6 +116,10 @@ def __init__( self.success_run_history_limit = success_run_history_limit self.random_name_suffix = random_name_suffix + # fix mypy typing + self.base_container_name: str + self.container_logs: list[str] + if self.base_container_name != self.BASE_CONTAINER_NAME: self.log.warning( "base_container_name is not supported and will be overridden to %s", self.BASE_CONTAINER_NAME diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/pod_generator.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator.py index b90fa715333bf..5fab194963e4c 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/pod_generator.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator.py @@ -34,8 +34,6 @@ import re2 from dateutil import parser -from kubernetes.client import models as k8s -from kubernetes.client.api_client import ApiClient from airflow.exceptions import ( AirflowConfigException, @@ -49,6 +47,8 @@ from airflow.utils import yaml from airflow.utils.hashlib_wrapper import md5 from airflow.version import version as airflow_version +from kubernetes.client import models as k8s +from kubernetes.client.api_client import ApiClient if TYPE_CHECKING: import datetime diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py index 9a978cbd08df9..b21fa757c874d 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py @@ -29,6 +29,7 @@ import uuid import re2 + from kubernetes.client import models as k8s MAX_POD_ID_LEN = 253 diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml diff --git a/providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml diff --git a/providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 diff --git a/providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/python_kubernetes_script.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/configmap.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py index db8c5301cb051..d950acf5ac42b 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py @@ -16,9 +16,8 @@ # under the License. from __future__ import annotations -from kubernetes.client import models as k8s - from airflow.exceptions import AirflowException +from kubernetes.client import models as k8s def convert_env_vars(env_vars) -> list[k8s.V1EnvVar]: diff --git a/providers/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/secret.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/secret.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/secret.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/secret.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/secret.py index 692777de447cd..2fc6c87757313 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/secret.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/secret.py @@ -21,10 +21,9 @@ import copy import uuid -from kubernetes.client import models as k8s - from airflow.exceptions import AirflowConfigException from airflow.providers.cncf.kubernetes.k8s_model import K8SModel +from kubernetes.client import models as k8s class Secret(K8SModel): diff --git a/providers/src/airflow/providers/cncf/kubernetes/sensors/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/sensors/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/sensors/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/sensors/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py index 08968d245fde7..7a42c21d7184c 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py @@ -21,11 +21,10 @@ from functools import cached_property from typing import TYPE_CHECKING -from kubernetes import client - from airflow.exceptions import AirflowException from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook from airflow.sensors.base import BaseSensorOperator +from kubernetes import client if TYPE_CHECKING: from airflow.utils.context import Context diff --git a/providers/src/airflow/providers/cncf/kubernetes/template_rendering.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/template_rendering.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/template_rendering.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/template_rendering.py index c499dad24038c..6abeeddde7884 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/template_rendering.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/template_rendering.py @@ -20,13 +20,13 @@ from typing import TYPE_CHECKING from jinja2 import TemplateAssertionError, UndefinedError -from kubernetes.client.api_client import ApiClient from airflow.exceptions import AirflowException from airflow.providers.cncf.kubernetes.kube_config import KubeConfig from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator from airflow.utils.session import NEW_SESSION, provide_session +from kubernetes.client.api_client import ApiClient if TYPE_CHECKING: from airflow.models.taskinstance import TaskInstance diff --git a/providers/src/airflow/providers/cncf/kubernetes/triggers/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/triggers/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/triggers/job.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/job.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/triggers/job.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/job.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/triggers/pod.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/pod.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/triggers/pod.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/pod.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/utils/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/__init__.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/utils/__init__.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/__init__.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/utils/delete_from.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/delete_from.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/utils/delete_from.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/delete_from.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py index d66eb9fc80bc0..6433ae465a40b 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py @@ -19,9 +19,8 @@ from collections.abc import Iterator from typing import Callable -from kubernetes.utils import FailToCreateError - from airflow.providers.cncf.kubernetes.utils.delete_from import FailToDeleteError +from kubernetes.utils import FailToCreateError def k8s_resource_iterator(callback: Callable[[dict], None], resources: Iterator) -> None: diff --git a/providers/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py similarity index 99% rename from providers/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 199d6a6d35dd3..9ea5228dac040 100644 --- a/providers/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -31,9 +31,6 @@ import pendulum import tenacity -from kubernetes import client, watch -from kubernetes.client.rest import ApiException -from kubernetes.stream import stream as kubernetes_stream from pendulum import DateTime from pendulum.parsing.exceptions import ParserError from typing_extensions import Literal @@ -44,12 +41,16 @@ from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults from airflow.utils.log.logging_mixin import LoggingMixin from airflow.utils.timezone import utcnow +from kubernetes import client, watch +from kubernetes.client.rest import ApiException +from kubernetes.stream import stream as kubernetes_stream if TYPE_CHECKING: + from urllib3.response import HTTPResponse + from kubernetes.client.models.core_v1_event_list import CoreV1EventList from kubernetes.client.models.v1_container_status import V1ContainerStatus from kubernetes.client.models.v1_pod import V1Pod - from urllib3.response import HTTPResponse EMPTY_XCOM_RESULT = "__airflow_xcom_result_empty__" diff --git a/providers/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py diff --git a/providers/src/airflow/providers/cncf/kubernetes/version_compat.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/version_compat.py similarity index 100% rename from providers/src/airflow/providers/cncf/kubernetes/version_compat.py rename to providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/version_compat.py diff --git a/providers/cncf/kubernetes/tests/conftest.py b/providers/cncf/kubernetes/tests/conftest.py new file mode 100644 index 0000000000000..068fe6bbf5ae9 --- /dev/null +++ b/providers/cncf/kubernetes/tests/conftest.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pathlib + +import pytest + +pytest_plugins = "tests_common.pytest_plugin" + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + deprecations_ignore_path = pathlib.Path(__file__).parent.joinpath("deprecations_ignore.yml") + dep_path = [deprecations_ignore_path] if deprecations_ignore_path.exists() else [] + config.inicfg["airflow_deprecations_ignore"] = ( + config.inicfg.get("airflow_deprecations_ignore", []) + dep_path # type: ignore[assignment,operator] + ) diff --git a/providers/cncf/kubernetes/tests/provider_tests/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/cncf/kubernetes/tests/provider_tests/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/cncf/kubernetes/tests/provider_tests/cncf/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/__init__.py new file mode 100644 index 0000000000000..e8fd22856438c --- /dev/null +++ b/providers/cncf/kubernetes/tests/provider_tests/cncf/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/tests/cncf/kubernetes/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/__init__.py diff --git a/providers/tests/cncf/kubernetes/backcompat/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/backcompat/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/backcompat/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/backcompat/__init__.py diff --git a/providers/tests/cncf/kubernetes/backcompat/test_backwards_compat_converters.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/backcompat/test_backwards_compat_converters.py similarity index 100% rename from providers/tests/cncf/kubernetes/backcompat/test_backwards_compat_converters.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/backcompat/test_backwards_compat_converters.py diff --git a/providers/tests/cncf/kubernetes/cli/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/cli/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/cli/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/cli/__init__.py diff --git a/providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/cli/test_kubernetes_command.py similarity index 100% rename from providers/tests/cncf/kubernetes/cli/test_kubernetes_command.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/cli/test_kubernetes_command.py diff --git a/providers/tests/cncf/kubernetes/conftest.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/conftest.py similarity index 100% rename from providers/tests/cncf/kubernetes/conftest.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/conftest.py diff --git a/providers/tests/cncf/kubernetes/data_files/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/__init__.py diff --git a/providers/tests/cncf/kubernetes/data_files/executor/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/executor/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/executor/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/executor/__init__.py diff --git a/providers/tests/cncf/kubernetes/data_files/executor/basic_template.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/executor/basic_template.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/executor/basic_template.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/executor/basic_template.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/kube_config b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/kube_config similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/kube_config rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/kube_config diff --git a/providers/tests/cncf/kubernetes/data_files/pods/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/pods/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/__init__.py diff --git a/providers/tests/cncf/kubernetes/data_files/pods/generator_base.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/generator_base.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/pods/generator_base.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/generator_base.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/generator_base_with_secrets.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/pods/template.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/template.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/pods/template.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/pods/template.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/spark/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/__init__.py diff --git a/providers/tests/cncf/kubernetes/data_files/spark/application_template.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_template.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/application_template.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_template.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/spark/application_test.json b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test.json similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/application_test.json rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test.json diff --git a/providers/tests/cncf/kubernetes/data_files/spark/application_test.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/application_test.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test.yaml diff --git a/providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.json diff --git a/providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml similarity index 100% rename from providers/tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/data_files/spark/application_test_with_no_name_from_config.yaml diff --git a/providers/tests/cncf/kubernetes/decorators/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/decorators/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/decorators/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/decorators/__init__.py diff --git a/providers/tests/cncf/kubernetes/decorators/test_kubernetes.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/decorators/test_kubernetes.py similarity index 100% rename from providers/tests/cncf/kubernetes/decorators/test_kubernetes.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/decorators/test_kubernetes.py diff --git a/providers/tests/cncf/kubernetes/executors/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/executors/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/__init__.py diff --git a/providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_kubernetes_executor.py similarity index 100% rename from providers/tests/cncf/kubernetes/executors/test_kubernetes_executor.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_kubernetes_executor.py diff --git a/providers/tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py similarity index 100% rename from providers/tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_local_kubernetes_executor.py diff --git a/providers/tests/cncf/kubernetes/hooks/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/hooks/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/hooks/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/hooks/__init__.py diff --git a/providers/tests/cncf/kubernetes/hooks/test_kubernetes.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/hooks/test_kubernetes.py similarity index 100% rename from providers/tests/cncf/kubernetes/hooks/test_kubernetes.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/hooks/test_kubernetes.py diff --git a/providers/tests/cncf/kubernetes/log_handlers/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/log_handlers/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/log_handlers/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/log_handlers/__init__.py diff --git a/providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/log_handlers/test_log_handlers.py similarity index 100% rename from providers/tests/cncf/kubernetes/log_handlers/test_log_handlers.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/log_handlers/test_log_handlers.py diff --git a/providers/tests/cncf/kubernetes/models/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/models/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/models/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/models/__init__.py diff --git a/providers/tests/cncf/kubernetes/models/test_secret.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/models/test_secret.py similarity index 100% rename from providers/tests/cncf/kubernetes/models/test_secret.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/models/test_secret.py diff --git a/providers/tests/cncf/kubernetes/operators/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/__init__.py diff --git a/providers/tests/cncf/kubernetes/operators/test_custom_object_launcher.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_custom_object_launcher.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/test_custom_object_launcher.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_custom_object_launcher.py diff --git a/providers/tests/cncf/kubernetes/operators/test_job.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_job.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/test_job.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_job.py diff --git a/providers/tests/cncf/kubernetes/operators/test_kueue.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_kueue.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/test_kueue.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_kueue.py diff --git a/providers/tests/cncf/kubernetes/operators/test_pod.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_pod.py similarity index 99% rename from providers/tests/cncf/kubernetes/operators/test_pod.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_pod.py index 0aacc4b782fd0..309d2abe642e1 100644 --- a/providers/tests/cncf/kubernetes/operators/test_pod.py +++ b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_pod.py @@ -1543,8 +1543,7 @@ def test_get_logs_but_not_for_base_container( @patch(KUB_OP_PATH.format("find_pod")) def test_execute_sync_callbacks(self, find_pod_mock): from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode - - from providers.tests.cncf.kubernetes.test_callbacks import ( + from provider_tests.cncf.kubernetes.test_callbacks import ( MockKubernetesPodOperatorCallback, MockWrapper, ) @@ -1630,8 +1629,7 @@ def test_execute_sync_callbacks(self, find_pod_mock): @patch(KUB_OP_PATH.format("find_pod")) def test_execute_sync_multiple_callbacks(self, find_pod_mock): from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode - - from providers.tests.cncf.kubernetes.test_callbacks import ( + from provider_tests.cncf.kubernetes.test_callbacks import ( MockKubernetesPodOperatorCallback, MockWrapper, ) @@ -1716,8 +1714,7 @@ def test_execute_sync_multiple_callbacks(self, find_pod_mock): @patch(HOOK_CLASS, new=MagicMock) def test_execute_async_callbacks(self): from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode - - from providers.tests.cncf.kubernetes.test_callbacks import ( + from provider_tests.cncf.kubernetes.test_callbacks import ( MockKubernetesPodOperatorCallback, MockWrapper, ) diff --git a/providers/tests/cncf/kubernetes/operators/test_resource.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_resource.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/test_resource.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_resource.py diff --git a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_spark_kubernetes.py similarity index 100% rename from providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_spark_kubernetes.py diff --git a/providers/tests/cncf/kubernetes/resource_convert/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/resource_convert/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/__init__.py diff --git a/providers/tests/cncf/kubernetes/resource_convert/test_configmap.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_configmap.py similarity index 100% rename from providers/tests/cncf/kubernetes/resource_convert/test_configmap.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_configmap.py diff --git a/providers/tests/cncf/kubernetes/resource_convert/test_env_variable.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_env_variable.py similarity index 100% rename from providers/tests/cncf/kubernetes/resource_convert/test_env_variable.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_env_variable.py diff --git a/providers/tests/cncf/kubernetes/resource_convert/test_secret.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_secret.py similarity index 100% rename from providers/tests/cncf/kubernetes/resource_convert/test_secret.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/resource_convert/test_secret.py diff --git a/providers/tests/cncf/kubernetes/sensors/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/sensors/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/sensors/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/sensors/__init__.py diff --git a/providers/tests/cncf/kubernetes/sensors/test_spark_kubernetes.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/sensors/test_spark_kubernetes.py similarity index 100% rename from providers/tests/cncf/kubernetes/sensors/test_spark_kubernetes.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/sensors/test_spark_kubernetes.py diff --git a/providers/tests/cncf/kubernetes/test_callbacks.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_callbacks.py similarity index 100% rename from providers/tests/cncf/kubernetes/test_callbacks.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_callbacks.py diff --git a/providers/tests/cncf/kubernetes/test_client.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_client.py similarity index 100% rename from providers/tests/cncf/kubernetes/test_client.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_client.py diff --git a/providers/tests/cncf/kubernetes/test_kubernetes_helper_functions.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_kubernetes_helper_functions.py similarity index 100% rename from providers/tests/cncf/kubernetes/test_kubernetes_helper_functions.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_kubernetes_helper_functions.py diff --git a/providers/tests/cncf/kubernetes/test_pod_generator.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_pod_generator.py similarity index 100% rename from providers/tests/cncf/kubernetes/test_pod_generator.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_pod_generator.py diff --git a/providers/tests/cncf/kubernetes/test_template_rendering.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_template_rendering.py similarity index 100% rename from providers/tests/cncf/kubernetes/test_template_rendering.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_template_rendering.py diff --git a/providers/tests/cncf/kubernetes/triggers/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/triggers/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/__init__.py diff --git a/providers/tests/cncf/kubernetes/triggers/test_job.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/test_job.py similarity index 100% rename from providers/tests/cncf/kubernetes/triggers/test_job.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/test_job.py diff --git a/providers/tests/cncf/kubernetes/triggers/test_pod.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/test_pod.py similarity index 100% rename from providers/tests/cncf/kubernetes/triggers/test_pod.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/test_pod.py diff --git a/providers/tests/cncf/kubernetes/utils/__init__.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/__init__.py similarity index 100% rename from providers/tests/cncf/kubernetes/utils/__init__.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/__init__.py diff --git a/providers/tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py similarity index 100% rename from providers/tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_k8s_resource_iterator.py diff --git a/providers/tests/cncf/kubernetes/utils/test_pod_manager.py b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_pod_manager.py similarity index 99% rename from providers/tests/cncf/kubernetes/utils/test_pod_manager.py rename to providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_pod_manager.py index 24ea794c2bffd..bc11e84dfb582 100644 --- a/providers/tests/cncf/kubernetes/utils/test_pod_manager.py +++ b/providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_pod_manager.py @@ -40,8 +40,7 @@ container_is_terminated, ) from airflow.utils.timezone import utc - -from providers.tests.cncf.kubernetes.test_callbacks import MockKubernetesPodOperatorCallback, MockWrapper +from provider_tests.cncf.kubernetes.test_callbacks import MockKubernetesPodOperatorCallback, MockWrapper if TYPE_CHECKING: from pendulum import DateTime diff --git a/providers/tests/system/cncf/kubernetes/__init__.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/__init__.py similarity index 100% rename from providers/tests/system/cncf/kubernetes/__init__.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/__init__.py diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py similarity index 99% rename from providers/tests/system/cncf/kubernetes/example_kubernetes.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py index dc74895a3e0fd..08a85a4a71da6 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py @@ -24,12 +24,11 @@ import os from datetime import datetime -from kubernetes.client import models as k8s - from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret from airflow.providers.standard.operators.bash import BashOperator +from kubernetes.client import models as k8s # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py similarity index 99% rename from providers/tests/system/cncf/kubernetes/example_kubernetes_async.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py index 7eb08442be3ba..2b8b7387e09f5 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_async.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py @@ -24,12 +24,11 @@ import os from datetime import datetime -from kubernetes.client import models as k8s - from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from airflow.providers.cncf.kubernetes.secret import Secret from airflow.providers.standard.operators.bash import BashOperator +from kubernetes.client import models as k8s # [START howto_operator_k8s_cluster_resources] secret_file = Secret("volume", "/etc/sql_conn", "airflow-secrets", "sql_alchemy_conn") diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py similarity index 100% rename from providers/tests/system/cncf/kubernetes/example_kubernetes_decorator.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_job.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py similarity index 100% rename from providers/tests/system/cncf/kubernetes/example_kubernetes_job.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_kueue.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py similarity index 99% rename from providers/tests/system/cncf/kubernetes/example_kubernetes_kueue.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py index 9aac715fdbed9..063be244b7aa7 100644 --- a/providers/tests/system/cncf/kubernetes/example_kubernetes_kueue.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py @@ -24,14 +24,13 @@ import os from datetime import datetime -from kubernetes.client import models as k8s - from airflow import DAG from airflow.providers.cncf.kubernetes.operators.kueue import ( KubernetesInstallKueueOperator, KubernetesStartKueueJobOperator, ) from airflow.providers.cncf.kubernetes.operators.resource import KubernetesCreateResourceOperator +from kubernetes.client import models as k8s ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default") DAG_ID = "example_kubernetes_kueue_operators" diff --git a/providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py similarity index 100% rename from providers/tests/system/cncf/kubernetes/example_kubernetes_resource.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py diff --git a/providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py similarity index 100% rename from providers/tests/system/cncf/kubernetes/example_spark_kubernetes.py rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py diff --git a/providers/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml similarity index 100% rename from providers/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes_spark_pi.yaml diff --git a/providers/tests/system/cncf/kubernetes/spark_job_template.yaml b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/spark_job_template.yaml similarity index 100% rename from providers/tests/system/cncf/kubernetes/spark_job_template.yaml rename to providers/cncf/kubernetes/tests/system/cncf/kubernetes/spark_job_template.yaml diff --git a/pyproject.toml b/pyproject.toml index 806372d1b6d1b..67d6d1e426091 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -657,6 +657,7 @@ dev = [ "apache-airflow-providers-atlassian-jira", "apache-airflow-providers-celery", "apache-airflow-providers-cloudant", + "apache-airflow-providers-cncf-kubernetes", "apache-airflow-providers-cohere", "apache-airflow-providers-common-compat", "apache-airflow-providers-common-io", @@ -747,6 +748,7 @@ apache-airflow-providers-asana = { workspace = true } apache-airflow-providers-atlassian-jira = { workspace = true } apache-airflow-providers-celery = {workspace = true} apache-airflow-providers-cloudant = { workspace = true } +apache-airflow-providers-cncf-kubernetes = { workspace = true } apache-airflow-providers-cohere = { workspace = true } apache-airflow-providers-common-compat = { workspace = true } apache-airflow-providers-common-io = { workspace = true } @@ -835,6 +837,7 @@ members = [ "providers/atlassian/jira", "providers/celery", "providers/cloudant", + "providers/cncf/kubernetes", "providers/cohere", "providers/common/compat", "providers/common/io", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index bcf249992f8bd..540625b3f7595 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -51,6 +51,7 @@ services: - ../../../empty:/opt/airflow/providers/atlassian/jira/src - ../../../empty:/opt/airflow/providers/celery/src - ../../../empty:/opt/airflow/providers/cloudant/src + - ../../../empty:/opt/airflow/providers/cncf/kubernetes/src - ../../../empty:/opt/airflow/providers/cohere/src - ../../../empty:/opt/airflow/providers/common/compat/src - ../../../empty:/opt/airflow/providers/common/io/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index feaf8d896984b..2a989e3f2307a 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -58,6 +58,7 @@ services: - ../../../providers/atlassian/jira/tests:/opt/airflow/providers/atlassian/jira/tests - ../../../providers/celery/tests:/opt/airflow/providers/celery/tests - ../../../providers/cloudant/tests:/opt/airflow/providers/cloudant/tests + - ../../../providers/cncf/kubernetes/tests:/opt/airflow/providers/cncf/kubernetes/tests - ../../../providers/cohere/tests:/opt/airflow/providers/cohere/tests - ../../../providers/common/compat/tests:/opt/airflow/providers/common/compat/tests - ../../../providers/common/io/tests:/opt/airflow/providers/common/io/tests diff --git a/scripts/ci/kubernetes/k8s_requirements.txt b/scripts/ci/kubernetes/k8s_requirements.txt index b25efb0abfd9f..7fd367c2e014b 100644 --- a/scripts/ci/kubernetes/k8s_requirements.txt +++ b/scripts/ci/kubernetes/k8s_requirements.txt @@ -2,3 +2,4 @@ -e ./providers/standard -e ./providers -e ./task_sdk +-e ./providers/cncf/kubernetes diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py index ff3347e7fd3dd..d562405e69752 100644 --- a/tests/always/test_project_structure.py +++ b/tests/always/test_project_structure.py @@ -86,20 +86,20 @@ def test_providers_modules_should_have_tests(self): "providers/tests/celery/executors/test_celery_executor_utils.py", "providers/tests/celery/executors/test_default_celery.py", "providers/tests/cloudant/test_cloudant_fake.py", - "providers/tests/cncf/kubernetes/executors/test_kubernetes_executor_types.py", - "providers/tests/cncf/kubernetes/executors/test_kubernetes_executor_utils.py", - "providers/tests/cncf/kubernetes/operators/test_kubernetes_pod.py", - "providers/tests/cncf/kubernetes/test_k8s_model.py", - "providers/tests/cncf/kubernetes/test_kube_client.py", - "providers/tests/cncf/kubernetes/test_kube_config.py", - "providers/tests/cncf/kubernetes/test_pod_generator_deprecated.py", - "providers/tests/cncf/kubernetes/test_pod_launcher_deprecated.py", - "providers/tests/cncf/kubernetes/test_python_kubernetes_script.py", - "providers/tests/cncf/kubernetes/test_secret.py", - "providers/tests/cncf/kubernetes/triggers/test_kubernetes_pod.py", - "providers/tests/cncf/kubernetes/utils/test_delete_from.py", - "providers/tests/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py", - "providers/tests/cncf/kubernetes/utils/test_xcom_sidecar.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_kubernetes_executor_types.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/executors/test_kubernetes_executor_utils.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/operators/test_kubernetes_pod.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_k8s_model.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_kube_client.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_kube_config.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_pod_generator_deprecated.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_pod_launcher_deprecated.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_python_kubernetes_script.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/test_secret.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/triggers/test_kubernetes_pod.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_delete_from.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py", + "providers/cncf/kubernetes/tests/provider_tests/cncf/kubernetes/utils/test_xcom_sidecar.py", "providers/google/tests/provider_tests/google/cloud/fs/test_gcs.py", "providers/google/tests/provider_tests/google/cloud/links/test_automl.py", "providers/google/tests/provider_tests/google/cloud/links/test_base.py", @@ -266,7 +266,8 @@ def get_classes_from_file( continue if is_new: - results[f"{'.'.join(module.split('.')[2:])}.{current_node.name}"] = current_node + module_path = module[module.find("airflow.providers") :] + results[f"{module_path}.{current_node.name}"] = current_node else: results[f"{module}.{current_node.name}"] = current_node print(f"{results}") @@ -568,7 +569,7 @@ class TestElasticsearchProviderProjectStructure(ExampleCoverageTest): class TestCncfProviderProjectStructure(ExampleCoverageTest): - PROVIDER = "cncf" + PROVIDER = "cncf/kubernetes" CLASS_DIRS = ProjectStructureTest.CLASS_DIRS BASE_CLASSES = {"airflow.providers.cncf.kubernetes.operators.resource.KubernetesResourceBaseOperator"}