diff --git a/docs/README.rst b/docs/README.rst index dcc1b531dfeab..acbb5971bed8b 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -22,12 +22,6 @@ This directory contains documentation for the Apache Airflow project and other p For Helm Chart, see: `/chart/README.md <../chart/READMe.md>`__ -File structure -============== - -Currently in the ``/docs/`` directory there is documentation for the ``apache-airflow`` package. The ``apache-airflow-providers-*'` directories contains independent documentation for each provider package. -We are taking steps to move the documentation for ``apache-airflow`` package to the ``apache-airflow`` subdirectory to make the directory structure clearer. - Development documentation preview ================================== diff --git a/docs/backport-providers.rst b/docs/apache-airflow/backport-providers.rst similarity index 100% rename from docs/backport-providers.rst rename to docs/apache-airflow/backport-providers.rst diff --git a/docs/best-practices.rst b/docs/apache-airflow/best-practices.rst similarity index 100% rename from docs/best-practices.rst rename to docs/apache-airflow/best-practices.rst diff --git a/docs/changelog.rst b/docs/apache-airflow/changelog.rst similarity index 96% rename from docs/changelog.rst rename to docs/apache-airflow/changelog.rst index 62553fac0a5b4..7c2b6b8b56b0d 100644 --- a/docs/changelog.rst +++ b/docs/apache-airflow/changelog.rst @@ -20,4 +20,4 @@ Changelog ========= -.. include:: ../CHANGELOG.txt +.. include:: ../../CHANGELOG.txt diff --git a/docs/cli-and-env-variables-ref.rst b/docs/apache-airflow/cli-and-env-variables-ref.rst similarity index 100% rename from docs/cli-and-env-variables-ref.rst rename to docs/apache-airflow/cli-and-env-variables-ref.rst diff --git a/docs/concepts.rst b/docs/apache-airflow/concepts.rst similarity index 98% rename from docs/concepts.rst rename to docs/apache-airflow/concepts.rst index 714c76b962ff2..d7b383a7b606f 100644 --- a/docs/concepts.rst +++ b/docs/apache-airflow/concepts.rst @@ -183,7 +183,7 @@ You can also use the parameters on jinja templates by using the ``{{context.para Example DAG with decorator: -.. exampleinclude:: /../airflow/example_dags/example_dag_decorator.py +.. exampleinclude:: /../../airflow/example_dags/example_dag_decorator.py :language: python :start-after: [START dag_decorator_usage] :end-before: [END dag_decorator_usage] @@ -939,14 +939,14 @@ Note that SubDAG operators should contain a factory method that returns a DAG object. This will prevent the SubDAG from being treated like a separate DAG in the main UI. For example: -.. exampleinclude:: /../airflow/example_dags/subdags/subdag.py +.. exampleinclude:: /../../airflow/example_dags/subdags/subdag.py :language: python :start-after: [START subdag] :end-before: [END subdag] This SubDAG can then be referenced in your main DAG file: -.. exampleinclude:: /../airflow/example_dags/example_subdag_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_subdag_operator.py :language: python :start-after: [START example_subdag_operator] :end-before: [END example_subdag_operator] @@ -1013,7 +1013,7 @@ and then puts both tasks upstream of ``task3``: Here is a more complicated example DAG with multiple levels of nested TaskGroups: -.. exampleinclude:: /../airflow/example_dags/example_task_group.py +.. exampleinclude:: /../../airflow/example_dags/example_task_group.py :language: python :start-after: [START howto_task_group] :end-before: [END howto_task_group] @@ -1158,7 +1158,7 @@ right now is not between its ``execution_time`` and the next scheduled For example, consider the following DAG: -.. exampleinclude:: /../airflow/example_dags/example_latest_only_with_trigger.py +.. exampleinclude:: /../../airflow/example_dags/example_latest_only_with_trigger.py :language: python :start-after: [START example] :end-before: [END example] @@ -1240,7 +1240,7 @@ DAG level cluster policy In this example we check if each DAG has at least one tag defined. Here is what it may look like: -.. literalinclude:: /../tests/cluster_policies/__init__.py +.. literalinclude:: /../../tests/cluster_policies/__init__.py :language: python :start-after: [START example_dag_cluster_policy] :end-before: [END example_dag_cluster_policy] @@ -1252,7 +1252,7 @@ using a specific operator, or enforce a task timeout policy, making sure that no tasks run for more than 48 hours. Here's an example of what this may look like: -.. literalinclude:: /../tests/cluster_policies/__init__.py +.. literalinclude:: /../../tests/cluster_policies/__init__.py :language: python :start-after: [START example_task_cluster_policy] :end-before: [END example_task_cluster_policy] @@ -1263,7 +1263,7 @@ reviewer, rather than as technical security controls. For example, don't run tasks without airflow owners: -.. literalinclude:: /../tests/cluster_policies/__init__.py +.. literalinclude:: /../../tests/cluster_policies/__init__.py :language: python :start-after: [START example_cluster_policy_rule] :end-before: [END example_cluster_policy_rule] @@ -1276,7 +1276,7 @@ the UI (and import errors table in the database). For Example in ``airflow_local_settings.py``: -.. literalinclude:: /../tests/cluster_policies/__init__.py +.. literalinclude:: /../../tests/cluster_policies/__init__.py :language: python :start-after: [START example_list_of_cluster_policy_rules] :end-before: [END example_list_of_cluster_policy_rules] @@ -1286,7 +1286,7 @@ Task instance mutation hook Task instance mutation hook can be used for example to re-routes the task to execute in a different queue during retries: -.. literalinclude:: /../tests/cluster_policies/__init__.py +.. literalinclude:: /../../tests/cluster_policies/__init__.py :language: python :start-after: [START example_task_mutation_hook] :end-before: [END example_task_mutation_hook] diff --git a/docs/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst similarity index 100% rename from docs/configurations-ref.rst rename to docs/apache-airflow/configurations-ref.rst diff --git a/docs/dag-run.rst b/docs/apache-airflow/dag-run.rst similarity index 100% rename from docs/dag-run.rst rename to docs/apache-airflow/dag-run.rst diff --git a/docs/dag-serialization.rst b/docs/apache-airflow/dag-serialization.rst similarity index 100% rename from docs/dag-serialization.rst rename to docs/apache-airflow/dag-serialization.rst diff --git a/docs/executor/celery.rst b/docs/apache-airflow/executor/celery.rst similarity index 100% rename from docs/executor/celery.rst rename to docs/apache-airflow/executor/celery.rst diff --git a/docs/executor/celery_kubernetes.rst b/docs/apache-airflow/executor/celery_kubernetes.rst similarity index 100% rename from docs/executor/celery_kubernetes.rst rename to docs/apache-airflow/executor/celery_kubernetes.rst diff --git a/docs/executor/dask.rst b/docs/apache-airflow/executor/dask.rst similarity index 100% rename from docs/executor/dask.rst rename to docs/apache-airflow/executor/dask.rst diff --git a/docs/executor/debug.rst b/docs/apache-airflow/executor/debug.rst similarity index 100% rename from docs/executor/debug.rst rename to docs/apache-airflow/executor/debug.rst diff --git a/docs/executor/index.rst b/docs/apache-airflow/executor/index.rst similarity index 100% rename from docs/executor/index.rst rename to docs/apache-airflow/executor/index.rst diff --git a/docs/executor/kubernetes.rst b/docs/apache-airflow/executor/kubernetes.rst similarity index 95% rename from docs/executor/kubernetes.rst rename to docs/apache-airflow/executor/kubernetes.rst index c01672623f82f..9b774cf08dc95 100644 --- a/docs/executor/kubernetes.rst +++ b/docs/apache-airflow/executor/kubernetes.rst @@ -77,21 +77,21 @@ With these requirements in mind, here are some examples of basic ``pod_template_ pod_template_file using the ``dag_in_image`` setting: -.. exampleinclude:: /../airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml :language: yaml :start-after: [START template_with_dags_in_image] :end-before: [END template_with_dags_in_image] ``pod_template_file`` which stores DAGs in a ``persistentVolume``: -.. exampleinclude:: /../airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml :language: yaml :start-after: [START template_with_dags_in_volume] :end-before: [END template_with_dags_in_volume] ``pod_template_file`` which pulls DAGs from git: -.. exampleinclude:: /../airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml +.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml :language: yaml :start-after: [START git_sync_template] :end-before: [END git_sync_template] @@ -108,7 +108,7 @@ Please note that the scheduler will override the ``metadata.name`` of the V1pod To overwrite the base container of the pod launched by the KubernetesExecutor, create a V1pod with a single container, and overwrite the fields as follows: -.. exampleinclude:: /../airflow/example_dags/example_kubernetes_executor_config.py +.. exampleinclude:: /../../airflow/example_dags/example_kubernetes_executor_config.py :language: python :start-after: [START task_with_volume] :end-before: [END task_with_volume] @@ -118,7 +118,7 @@ Note that volume mounts environment variables, ports, and devices will all be ex To add a sidecar container to the launched pod, create a V1pod with an empty first container with the name ``base`` and a second container containing your desired sidecar. -.. exampleinclude:: /../airflow/example_dags/example_kubernetes_executor_config.py +.. exampleinclude:: /../../airflow/example_dags/example_kubernetes_executor_config.py :language: python :start-after: [START task_with_sidecar] :end-before: [END task_with_sidecar] @@ -128,7 +128,7 @@ This will replace the default ``pod_template_file`` named in the airflow.cfg and Here is an example of a task with both features: -.. exampleinclude:: /../airflow/example_dags/example_kubernetes_executor_config.py +.. exampleinclude:: /../../airflow/example_dags/example_kubernetes_executor_config.py :language: python :start-after: [START task_with_template] :end-before: [END task_with_template] diff --git a/docs/executor/local.rst b/docs/apache-airflow/executor/local.rst similarity index 100% rename from docs/executor/local.rst rename to docs/apache-airflow/executor/local.rst diff --git a/docs/executor/sequential.rst b/docs/apache-airflow/executor/sequential.rst similarity index 100% rename from docs/executor/sequential.rst rename to docs/apache-airflow/executor/sequential.rst diff --git a/docs/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst similarity index 100% rename from docs/extra-packages-ref.rst rename to docs/apache-airflow/extra-packages-ref.rst diff --git a/docs/faq.rst b/docs/apache-airflow/faq.rst similarity index 100% rename from docs/faq.rst rename to docs/apache-airflow/faq.rst diff --git a/docs/howto/add-dag-tags.rst b/docs/apache-airflow/howto/add-dag-tags.rst similarity index 100% rename from docs/howto/add-dag-tags.rst rename to docs/apache-airflow/howto/add-dag-tags.rst diff --git a/docs/howto/connection.rst b/docs/apache-airflow/howto/connection.rst similarity index 100% rename from docs/howto/connection.rst rename to docs/apache-airflow/howto/connection.rst diff --git a/docs/howto/custom-operator.rst b/docs/apache-airflow/howto/custom-operator.rst similarity index 100% rename from docs/howto/custom-operator.rst rename to docs/apache-airflow/howto/custom-operator.rst diff --git a/docs/howto/customize-state-colors-ui.rst b/docs/apache-airflow/howto/customize-state-colors-ui.rst similarity index 100% rename from docs/howto/customize-state-colors-ui.rst rename to docs/apache-airflow/howto/customize-state-colors-ui.rst diff --git a/docs/howto/define_extra_link.rst b/docs/apache-airflow/howto/define_extra_link.rst similarity index 100% rename from docs/howto/define_extra_link.rst rename to docs/apache-airflow/howto/define_extra_link.rst diff --git a/docs/howto/email-config.rst b/docs/apache-airflow/howto/email-config.rst similarity index 100% rename from docs/howto/email-config.rst rename to docs/apache-airflow/howto/email-config.rst diff --git a/docs/howto/index.rst b/docs/apache-airflow/howto/index.rst similarity index 100% rename from docs/howto/index.rst rename to docs/apache-airflow/howto/index.rst diff --git a/docs/howto/initialize-database.rst b/docs/apache-airflow/howto/initialize-database.rst similarity index 100% rename from docs/howto/initialize-database.rst rename to docs/apache-airflow/howto/initialize-database.rst diff --git a/docs/howto/operator/bash.rst b/docs/apache-airflow/howto/operator/bash.rst similarity index 96% rename from docs/howto/operator/bash.rst rename to docs/apache-airflow/howto/operator/bash.rst index 0df42eda5ba08..c8a923f963225 100644 --- a/docs/howto/operator/bash.rst +++ b/docs/apache-airflow/howto/operator/bash.rst @@ -25,7 +25,7 @@ BashOperator Use the :class:`~airflow.operators.bash.BashOperator` to execute commands in a `Bash `__ shell. -.. exampleinclude:: /../airflow/example_dags/example_bash_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_bash_operator.py :language: python :start-after: [START howto_operator_bash] :end-before: [END howto_operator_bash] @@ -36,7 +36,7 @@ Templating You can use :ref:`Jinja templates ` to parameterize the ``bash_command`` argument. -.. exampleinclude:: /../airflow/example_dags/example_bash_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_bash_operator.py :language: python :start-after: [START howto_operator_bash_template] :end-before: [END howto_operator_bash_template] diff --git a/docs/howto/operator/external_task_sensor.rst b/docs/apache-airflow/howto/operator/external_task_sensor.rst similarity index 94% rename from docs/howto/operator/external_task_sensor.rst rename to docs/apache-airflow/howto/operator/external_task_sensor.rst index 1064429a3a904..697e41af10d96 100644 --- a/docs/howto/operator/external_task_sensor.rst +++ b/docs/apache-airflow/howto/operator/external_task_sensor.rst @@ -44,7 +44,7 @@ wait for another task on a different DAG for a specific ``execution_date``. ExternalTaskSensor also provide options to set if the Task on a remote DAG succeeded or failed via ``allowed_states`` and ``failed_states`` parameters. -.. exampleinclude:: /../airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../../airflow/example_dags/example_external_task_marker_dag.py :language: python :start-after: [START howto_operator_external_task_sensor] :end-before: [END howto_operator_external_task_sensor] @@ -58,7 +58,7 @@ on ``child_dag`` for a specific ``execution_date`` should also be cleared, ``Ext should be used. Note that ``child_task1`` will only be cleared if "Recursive" is selected when the user clears ``parent_task``. -.. exampleinclude:: /../airflow/example_dags/example_external_task_marker_dag.py +.. exampleinclude:: /../../airflow/example_dags/example_external_task_marker_dag.py :language: python :start-after: [START howto_operator_external_task_marker] :end-before: [END howto_operator_external_task_marker] diff --git a/docs/howto/operator/index.rst b/docs/apache-airflow/howto/operator/index.rst similarity index 94% rename from docs/howto/operator/index.rst rename to docs/apache-airflow/howto/operator/index.rst index 71690a8210449..89ad5f3e5b884 100644 --- a/docs/howto/operator/index.rst +++ b/docs/apache-airflow/howto/operator/index.rst @@ -25,7 +25,7 @@ determine what actually executes when your DAG runs. .. note:: See the :ref:`Operators Concepts ` documentation and the - :doc:`Operators API Reference <../../_api/index>` for more + :doc:`Operators API Reference ` for more information. .. toctree:: diff --git a/docs/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst similarity index 92% rename from docs/howto/operator/python.rst rename to docs/apache-airflow/howto/operator/python.rst index 8b6d726bb8494..7f4d2b8163d20 100644 --- a/docs/howto/operator/python.rst +++ b/docs/apache-airflow/howto/operator/python.rst @@ -25,7 +25,7 @@ PythonOperator Use the :class:`~airflow.operators.python.PythonOperator` to execute Python callables. -.. exampleinclude:: /../airflow/example_dags/example_python_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py :language: python :start-after: [START howto_operator_python] :end-before: [END howto_operator_python] @@ -36,7 +36,7 @@ Passing in arguments Use the ``op_args`` and ``op_kwargs`` arguments to pass additional arguments to the Python callable. -.. exampleinclude:: /../airflow/example_dags/example_python_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py :language: python :start-after: [START howto_operator_python_kwargs] :end-before: [END howto_operator_python_kwargs] @@ -61,7 +61,7 @@ PythonVirtualenvOperator Use the :class:`~airflow.operators.python.PythonVirtualenvOperator` to execute Python callables inside a new Python virtual environment. -.. exampleinclude:: ../../../airflow/example_dags/example_python_operator.py +.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py :language: python :start-after: [START howto_operator_python_venv] :end-before: [END howto_operator_python_venv] diff --git a/docs/howto/run-behind-proxy.rst b/docs/apache-airflow/howto/run-behind-proxy.rst similarity index 100% rename from docs/howto/run-behind-proxy.rst rename to docs/apache-airflow/howto/run-behind-proxy.rst diff --git a/docs/howto/run-with-systemd.rst b/docs/apache-airflow/howto/run-with-systemd.rst similarity index 100% rename from docs/howto/run-with-systemd.rst rename to docs/apache-airflow/howto/run-with-systemd.rst diff --git a/docs/howto/run-with-upstart.rst b/docs/apache-airflow/howto/run-with-upstart.rst similarity index 100% rename from docs/howto/run-with-upstart.rst rename to docs/apache-airflow/howto/run-with-upstart.rst diff --git a/docs/howto/set-config.rst b/docs/apache-airflow/howto/set-config.rst similarity index 100% rename from docs/howto/set-config.rst rename to docs/apache-airflow/howto/set-config.rst diff --git a/docs/howto/use-test-config.rst b/docs/apache-airflow/howto/use-test-config.rst similarity index 100% rename from docs/howto/use-test-config.rst rename to docs/apache-airflow/howto/use-test-config.rst diff --git a/docs/howto/variable.rst b/docs/apache-airflow/howto/variable.rst similarity index 100% rename from docs/howto/variable.rst rename to docs/apache-airflow/howto/variable.rst diff --git a/docs/img/add-dag-tags.png b/docs/apache-airflow/img/add-dag-tags.png similarity index 100% rename from docs/img/add-dag-tags.png rename to docs/apache-airflow/img/add-dag-tags.png diff --git a/docs/img/add-role.png b/docs/apache-airflow/img/add-role.png similarity index 100% rename from docs/img/add-role.png rename to docs/apache-airflow/img/add-role.png diff --git a/docs/img/airflow.gif b/docs/apache-airflow/img/airflow.gif similarity index 100% rename from docs/img/airflow.gif rename to docs/apache-airflow/img/airflow.gif diff --git a/docs/img/apache.jpg b/docs/apache-airflow/img/apache.jpg similarity index 100% rename from docs/img/apache.jpg rename to docs/apache-airflow/img/apache.jpg diff --git a/docs/img/arch-diag-basic.png b/docs/apache-airflow/img/arch-diag-basic.png similarity index 100% rename from docs/img/arch-diag-basic.png rename to docs/apache-airflow/img/arch-diag-basic.png diff --git a/docs/img/arch-diag-kubernetes.png b/docs/apache-airflow/img/arch-diag-kubernetes.png similarity index 100% rename from docs/img/arch-diag-kubernetes.png rename to docs/apache-airflow/img/arch-diag-kubernetes.png diff --git a/docs/img/arch-diag-kubernetes2.png b/docs/apache-airflow/img/arch-diag-kubernetes2.png similarity index 100% rename from docs/img/arch-diag-kubernetes2.png rename to docs/apache-airflow/img/arch-diag-kubernetes2.png diff --git a/docs/img/arch-diag-logging.png b/docs/apache-airflow/img/arch-diag-logging.png similarity index 100% rename from docs/img/arch-diag-logging.png rename to docs/apache-airflow/img/arch-diag-logging.png diff --git a/docs/img/aws-web-identity-federation-gcp.png b/docs/apache-airflow/img/aws-web-identity-federation-gcp.png similarity index 100% rename from docs/img/aws-web-identity-federation-gcp.png rename to docs/apache-airflow/img/aws-web-identity-federation-gcp.png diff --git a/docs/img/branch_note.png b/docs/apache-airflow/img/branch_note.png similarity index 100% rename from docs/img/branch_note.png rename to docs/apache-airflow/img/branch_note.png diff --git a/docs/img/branch_with_trigger.png b/docs/apache-airflow/img/branch_with_trigger.png similarity index 100% rename from docs/img/branch_with_trigger.png rename to docs/apache-airflow/img/branch_with_trigger.png diff --git a/docs/img/branch_without_trigger.png b/docs/apache-airflow/img/branch_without_trigger.png similarity index 100% rename from docs/img/branch_without_trigger.png rename to docs/apache-airflow/img/branch_without_trigger.png diff --git a/docs/img/change-ui-colors/dags-page-new.png b/docs/apache-airflow/img/change-ui-colors/dags-page-new.png similarity index 100% rename from docs/img/change-ui-colors/dags-page-new.png rename to docs/apache-airflow/img/change-ui-colors/dags-page-new.png diff --git a/docs/img/change-ui-colors/dags-page-old.png b/docs/apache-airflow/img/change-ui-colors/dags-page-old.png similarity index 100% rename from docs/img/change-ui-colors/dags-page-old.png rename to docs/apache-airflow/img/change-ui-colors/dags-page-old.png diff --git a/docs/img/change-ui-colors/graph-view-new.png b/docs/apache-airflow/img/change-ui-colors/graph-view-new.png similarity index 100% rename from docs/img/change-ui-colors/graph-view-new.png rename to docs/apache-airflow/img/change-ui-colors/graph-view-new.png diff --git a/docs/img/change-ui-colors/graph-view-old.png b/docs/apache-airflow/img/change-ui-colors/graph-view-old.png similarity index 100% rename from docs/img/change-ui-colors/graph-view-old.png rename to docs/apache-airflow/img/change-ui-colors/graph-view-old.png diff --git a/docs/img/change-ui-colors/tree-view-new.png b/docs/apache-airflow/img/change-ui-colors/tree-view-new.png similarity index 100% rename from docs/img/change-ui-colors/tree-view-new.png rename to docs/apache-airflow/img/change-ui-colors/tree-view-new.png diff --git a/docs/img/change-ui-colors/tree-view-old.png b/docs/apache-airflow/img/change-ui-colors/tree-view-old.png similarity index 100% rename from docs/img/change-ui-colors/tree-view-old.png rename to docs/apache-airflow/img/change-ui-colors/tree-view-old.png diff --git a/docs/img/cli_completion.gif b/docs/apache-airflow/img/cli_completion.gif similarity index 100% rename from docs/img/cli_completion.gif rename to docs/apache-airflow/img/cli_completion.gif diff --git a/docs/img/code.png b/docs/apache-airflow/img/code.png similarity index 100% rename from docs/img/code.png rename to docs/apache-airflow/img/code.png diff --git a/docs/img/connection_create.png b/docs/apache-airflow/img/connection_create.png similarity index 100% rename from docs/img/connection_create.png rename to docs/apache-airflow/img/connection_create.png diff --git a/docs/img/connection_edit.png b/docs/apache-airflow/img/connection_edit.png similarity index 100% rename from docs/img/connection_edit.png rename to docs/apache-airflow/img/connection_edit.png diff --git a/docs/img/connections.png b/docs/apache-airflow/img/connections.png similarity index 100% rename from docs/img/connections.png rename to docs/apache-airflow/img/connections.png diff --git a/docs/img/context.png b/docs/apache-airflow/img/context.png similarity index 100% rename from docs/img/context.png rename to docs/apache-airflow/img/context.png diff --git a/docs/img/dag_serialization.png b/docs/apache-airflow/img/dag_serialization.png similarity index 100% rename from docs/img/dag_serialization.png rename to docs/apache-airflow/img/dag_serialization.png diff --git a/docs/img/dags.png b/docs/apache-airflow/img/dags.png similarity index 100% rename from docs/img/dags.png rename to docs/apache-airflow/img/dags.png diff --git a/docs/img/duration.png b/docs/apache-airflow/img/duration.png similarity index 100% rename from docs/img/duration.png rename to docs/apache-airflow/img/duration.png diff --git a/docs/img/example_passing_conf.png b/docs/apache-airflow/img/example_passing_conf.png similarity index 100% rename from docs/img/example_passing_conf.png rename to docs/apache-airflow/img/example_passing_conf.png diff --git a/docs/img/gantt.png b/docs/apache-airflow/img/gantt.png similarity index 100% rename from docs/img/gantt.png rename to docs/apache-airflow/img/gantt.png diff --git a/docs/img/graph.png b/docs/apache-airflow/img/graph.png similarity index 100% rename from docs/img/graph.png rename to docs/apache-airflow/img/graph.png diff --git a/docs/img/k8s-0-worker.jpeg b/docs/apache-airflow/img/k8s-0-worker.jpeg similarity index 100% rename from docs/img/k8s-0-worker.jpeg rename to docs/apache-airflow/img/k8s-0-worker.jpeg diff --git a/docs/img/k8s-3-worker.jpeg b/docs/apache-airflow/img/k8s-3-worker.jpeg similarity index 100% rename from docs/img/k8s-3-worker.jpeg rename to docs/apache-airflow/img/k8s-3-worker.jpeg diff --git a/docs/img/k8s-5-worker.jpeg b/docs/apache-airflow/img/k8s-5-worker.jpeg similarity index 100% rename from docs/img/k8s-5-worker.jpeg rename to docs/apache-airflow/img/k8s-5-worker.jpeg diff --git a/docs/img/k8s-failed-pod.png b/docs/apache-airflow/img/k8s-failed-pod.png similarity index 100% rename from docs/img/k8s-failed-pod.png rename to docs/apache-airflow/img/k8s-failed-pod.png diff --git a/docs/img/k8s-happy-path.png b/docs/apache-airflow/img/k8s-happy-path.png similarity index 100% rename from docs/img/k8s-happy-path.png rename to docs/apache-airflow/img/k8s-happy-path.png diff --git a/docs/img/latest_only_with_trigger.png b/docs/apache-airflow/img/latest_only_with_trigger.png similarity index 100% rename from docs/img/latest_only_with_trigger.png rename to docs/apache-airflow/img/latest_only_with_trigger.png diff --git a/docs/img/logos/airflow_64x64_emoji_transparent.png b/docs/apache-airflow/img/logos/airflow_64x64_emoji_transparent.png similarity index 100% rename from docs/img/logos/airflow_64x64_emoji_transparent.png rename to docs/apache-airflow/img/logos/airflow_64x64_emoji_transparent.png diff --git a/docs/img/logos/airflow_dark_bg.png b/docs/apache-airflow/img/logos/airflow_dark_bg.png similarity index 100% rename from docs/img/logos/airflow_dark_bg.png rename to docs/apache-airflow/img/logos/airflow_dark_bg.png diff --git a/docs/img/logos/airflow_transparent.png b/docs/apache-airflow/img/logos/airflow_transparent.png similarity index 100% rename from docs/img/logos/airflow_transparent.png rename to docs/apache-airflow/img/logos/airflow_transparent.png diff --git a/docs/img/logos/airflow_white_bg.png b/docs/apache-airflow/img/logos/airflow_white_bg.png similarity index 100% rename from docs/img/logos/airflow_white_bg.png rename to docs/apache-airflow/img/logos/airflow_white_bg.png diff --git a/docs/img/logos/wordmark_1.png b/docs/apache-airflow/img/logos/wordmark_1.png similarity index 100% rename from docs/img/logos/wordmark_1.png rename to docs/apache-airflow/img/logos/wordmark_1.png diff --git a/docs/img/logos/wordmark_1.svg b/docs/apache-airflow/img/logos/wordmark_1.svg similarity index 100% rename from docs/img/logos/wordmark_1.svg rename to docs/apache-airflow/img/logos/wordmark_1.svg diff --git a/docs/img/logos/wordmark_2.png b/docs/apache-airflow/img/logos/wordmark_2.png similarity index 100% rename from docs/img/logos/wordmark_2.png rename to docs/apache-airflow/img/logos/wordmark_2.png diff --git a/docs/img/logos/wordmark_2.svg b/docs/apache-airflow/img/logos/wordmark_2.svg similarity index 100% rename from docs/img/logos/wordmark_2.svg rename to docs/apache-airflow/img/logos/wordmark_2.svg diff --git a/docs/img/nested_branching.png b/docs/apache-airflow/img/nested_branching.png similarity index 100% rename from docs/img/nested_branching.png rename to docs/apache-airflow/img/nested_branching.png diff --git a/docs/img/new-role.png b/docs/apache-airflow/img/new-role.png similarity index 100% rename from docs/img/new-role.png rename to docs/apache-airflow/img/new-role.png diff --git a/docs/img/operator_extra_link.png b/docs/apache-airflow/img/operator_extra_link.png similarity index 100% rename from docs/img/operator_extra_link.png rename to docs/apache-airflow/img/operator_extra_link.png diff --git a/docs/img/run_task_on_celery_executor.png b/docs/apache-airflow/img/run_task_on_celery_executor.png similarity index 100% rename from docs/img/run_task_on_celery_executor.png rename to docs/apache-airflow/img/run_task_on_celery_executor.png diff --git a/docs/img/run_task_on_celery_executor.puml b/docs/apache-airflow/img/run_task_on_celery_executor.puml similarity index 100% rename from docs/img/run_task_on_celery_executor.puml rename to docs/apache-airflow/img/run_task_on_celery_executor.puml diff --git a/docs/img/scheduler_loop.jpg b/docs/apache-airflow/img/scheduler_loop.jpg similarity index 100% rename from docs/img/scheduler_loop.jpg rename to docs/apache-airflow/img/scheduler_loop.jpg diff --git a/docs/img/smart_sensor_architecture.png b/docs/apache-airflow/img/smart_sensor_architecture.png similarity index 100% rename from docs/img/smart_sensor_architecture.png rename to docs/apache-airflow/img/smart_sensor_architecture.png diff --git a/docs/img/smart_sensor_single_task_execute_flow.png b/docs/apache-airflow/img/smart_sensor_single_task_execute_flow.png similarity index 100% rename from docs/img/smart_sensor_single_task_execute_flow.png rename to docs/apache-airflow/img/smart_sensor_single_task_execute_flow.png diff --git a/docs/img/subdag_after.png b/docs/apache-airflow/img/subdag_after.png similarity index 100% rename from docs/img/subdag_after.png rename to docs/apache-airflow/img/subdag_after.png diff --git a/docs/img/subdag_before.png b/docs/apache-airflow/img/subdag_before.png similarity index 100% rename from docs/img/subdag_before.png rename to docs/apache-airflow/img/subdag_before.png diff --git a/docs/img/subdag_zoom.png b/docs/apache-airflow/img/subdag_zoom.png similarity index 100% rename from docs/img/subdag_zoom.png rename to docs/apache-airflow/img/subdag_zoom.png diff --git a/docs/img/task_group.gif b/docs/apache-airflow/img/task_group.gif similarity index 100% rename from docs/img/task_group.gif rename to docs/apache-airflow/img/task_group.gif diff --git a/docs/img/task_lifecycle_diagram.png b/docs/apache-airflow/img/task_lifecycle_diagram.png similarity index 100% rename from docs/img/task_lifecycle_diagram.png rename to docs/apache-airflow/img/task_lifecycle_diagram.png diff --git a/docs/img/task_manual_vs_scheduled.png b/docs/apache-airflow/img/task_manual_vs_scheduled.png similarity index 100% rename from docs/img/task_manual_vs_scheduled.png rename to docs/apache-airflow/img/task_manual_vs_scheduled.png diff --git a/docs/img/task_stages.png b/docs/apache-airflow/img/task_stages.png similarity index 100% rename from docs/img/task_stages.png rename to docs/apache-airflow/img/task_stages.png diff --git a/docs/img/tree.png b/docs/apache-airflow/img/tree.png similarity index 100% rename from docs/img/tree.png rename to docs/apache-airflow/img/tree.png diff --git a/docs/img/ui-timezone-chooser.png b/docs/apache-airflow/img/ui-timezone-chooser.png similarity index 100% rename from docs/img/ui-timezone-chooser.png rename to docs/apache-airflow/img/ui-timezone-chooser.png diff --git a/docs/img/usage_cli_export.png b/docs/apache-airflow/img/usage_cli_export.png similarity index 100% rename from docs/img/usage_cli_export.png rename to docs/apache-airflow/img/usage_cli_export.png diff --git a/docs/img/usage_cli_imgcat.png b/docs/apache-airflow/img/usage_cli_imgcat.png similarity index 100% rename from docs/img/usage_cli_imgcat.png rename to docs/apache-airflow/img/usage_cli_imgcat.png diff --git a/docs/img/variable_hidden.png b/docs/apache-airflow/img/variable_hidden.png similarity index 100% rename from docs/img/variable_hidden.png rename to docs/apache-airflow/img/variable_hidden.png diff --git a/docs/index.rst b/docs/apache-airflow/index.rst similarity index 97% rename from docs/index.rst rename to docs/apache-airflow/index.rst index 4ef042f580347..96c32a4d4d862 100644 --- a/docs/index.rst +++ b/docs/apache-airflow/index.rst @@ -18,7 +18,7 @@ -.. image:: ../airflow/www/static/pin_large.png +.. image:: ../../airflow/www/static/pin_large.png :width: 100 Apache Airflow Documentation @@ -112,7 +112,7 @@ Content Operators and hooks CLI Macros - Python API <_api/index> + Python API Experimental REST API Stable REST API Configurations diff --git a/docs/installation.rst b/docs/apache-airflow/installation.rst similarity index 100% rename from docs/installation.rst rename to docs/apache-airflow/installation.rst diff --git a/docs/integration.rst b/docs/apache-airflow/integration.rst similarity index 100% rename from docs/integration.rst rename to docs/apache-airflow/integration.rst diff --git a/docs/kubernetes.rst b/docs/apache-airflow/kubernetes.rst similarity index 100% rename from docs/kubernetes.rst rename to docs/apache-airflow/kubernetes.rst diff --git a/docs/license.rst b/docs/apache-airflow/license.rst similarity index 100% rename from docs/license.rst rename to docs/apache-airflow/license.rst diff --git a/docs/lineage.rst b/docs/apache-airflow/lineage.rst similarity index 100% rename from docs/lineage.rst rename to docs/apache-airflow/lineage.rst diff --git a/docs/logging-monitoring/check-health.rst b/docs/apache-airflow/logging-monitoring/check-health.rst similarity index 100% rename from docs/logging-monitoring/check-health.rst rename to docs/apache-airflow/logging-monitoring/check-health.rst diff --git a/docs/logging-monitoring/errors.rst b/docs/apache-airflow/logging-monitoring/errors.rst similarity index 100% rename from docs/logging-monitoring/errors.rst rename to docs/apache-airflow/logging-monitoring/errors.rst diff --git a/docs/logging-monitoring/index.rst b/docs/apache-airflow/logging-monitoring/index.rst similarity index 100% rename from docs/logging-monitoring/index.rst rename to docs/apache-airflow/logging-monitoring/index.rst diff --git a/docs/logging-monitoring/logging-architecture.rst b/docs/apache-airflow/logging-monitoring/logging-architecture.rst similarity index 100% rename from docs/logging-monitoring/logging-architecture.rst rename to docs/apache-airflow/logging-monitoring/logging-architecture.rst diff --git a/docs/logging-monitoring/logging-tasks.rst b/docs/apache-airflow/logging-monitoring/logging-tasks.rst similarity index 100% rename from docs/logging-monitoring/logging-tasks.rst rename to docs/apache-airflow/logging-monitoring/logging-tasks.rst diff --git a/docs/logging-monitoring/metrics.rst b/docs/apache-airflow/logging-monitoring/metrics.rst similarity index 100% rename from docs/logging-monitoring/metrics.rst rename to docs/apache-airflow/logging-monitoring/metrics.rst diff --git a/docs/logging-monitoring/tracking-user-activity.rst b/docs/apache-airflow/logging-monitoring/tracking-user-activity.rst similarity index 100% rename from docs/logging-monitoring/tracking-user-activity.rst rename to docs/apache-airflow/logging-monitoring/tracking-user-activity.rst diff --git a/docs/macros-ref.rst b/docs/apache-airflow/macros-ref.rst similarity index 100% rename from docs/macros-ref.rst rename to docs/apache-airflow/macros-ref.rst diff --git a/docs/modules_management.rst b/docs/apache-airflow/modules_management.rst similarity index 100% rename from docs/modules_management.rst rename to docs/apache-airflow/modules_management.rst diff --git a/docs/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst similarity index 100% rename from docs/operators-and-hooks-ref.rst rename to docs/apache-airflow/operators-and-hooks-ref.rst diff --git a/docs/plugins.rst b/docs/apache-airflow/plugins.rst similarity index 100% rename from docs/plugins.rst rename to docs/apache-airflow/plugins.rst diff --git a/docs/privacy_notice.rst b/docs/apache-airflow/privacy_notice.rst similarity index 100% rename from docs/privacy_notice.rst rename to docs/apache-airflow/privacy_notice.rst diff --git a/docs/production-deployment.rst b/docs/apache-airflow/production-deployment.rst similarity index 100% rename from docs/production-deployment.rst rename to docs/apache-airflow/production-deployment.rst diff --git a/docs/project.rst b/docs/apache-airflow/project.rst similarity index 100% rename from docs/project.rst rename to docs/apache-airflow/project.rst diff --git a/docs/autoapi_templates/index.rst b/docs/apache-airflow/python-api-ref.rst similarity index 94% rename from docs/autoapi_templates/index.rst rename to docs/apache-airflow/python-api-ref.rst index c0550558cadd9..8a4f1f6723883 100644 --- a/docs/autoapi_templates/index.rst +++ b/docs/apache-airflow/python-api-ref.rst @@ -62,9 +62,9 @@ All operators are in the following packages: :glob: :maxdepth: 1 - airflow/operators/index + _api/airflow/operators/index - airflow/sensors/index + _api/airflow/sensors/index .. _pythonapi:hooks: @@ -84,7 +84,7 @@ All hooks are in the following packages: :glob: :maxdepth: 1 - airflow/hooks/index + _api/airflow/hooks/index Executors --------- @@ -100,7 +100,7 @@ All executors are in the following packages: :glob: :maxdepth: 1 - airflow/executors/index + _api/airflow/executors/index Models ------ @@ -112,7 +112,7 @@ persisted in the database. :glob: :maxdepth: 1 - airflow/models/index + _api/airflow/models/index .. _pythonapi:exceptions: @@ -124,7 +124,7 @@ Exceptions :glob: :maxdepth: 1 - airflow/exceptions/index + _api/airflow/exceptions/index Secrets Backends ---------------- @@ -136,4 +136,4 @@ All secrets backends derive from :class:`~airflow.secrets.BaseSecretsBackend`. :glob: :maxdepth: 1 - airflow/secrets/index + _api/airflow/secrets/index diff --git a/docs/redirects.txt b/docs/apache-airflow/redirects.txt similarity index 99% rename from docs/redirects.txt rename to docs/apache-airflow/redirects.txt index 9c51d39b3ca77..96ba39cda39fa 100644 --- a/docs/redirects.txt +++ b/docs/apache-airflow/redirects.txt @@ -85,3 +85,4 @@ errors.rst logging-monitoring/errors.rst howto/write-logs.rst logging-monitoring/logging-tasks.rst metrics.rst logging-monitoring/metrics.rst howto/tracking-user-activity.rst logging-monitoring/tracking-user-activity.rst +_api/index.rst python-api-ref.rst diff --git a/docs/rest-api-ref.rst b/docs/apache-airflow/rest-api-ref.rst similarity index 100% rename from docs/rest-api-ref.rst rename to docs/apache-airflow/rest-api-ref.rst diff --git a/docs/scheduler.rst b/docs/apache-airflow/scheduler.rst similarity index 100% rename from docs/scheduler.rst rename to docs/apache-airflow/scheduler.rst diff --git a/docs/security/access-control.rst b/docs/apache-airflow/security/access-control.rst similarity index 99% rename from docs/security/access-control.rst rename to docs/apache-airflow/security/access-control.rst index 22df6bd6e3c63..b6269f2d7955b 100644 --- a/docs/security/access-control.rst +++ b/docs/apache-airflow/security/access-control.rst @@ -55,7 +55,7 @@ Viewer ^^^^^^ ``Viewer`` users have limited viewer permissions -.. exampleinclude:: /../airflow/www/security.py +.. exampleinclude:: /../../airflow/www/security.py :language: python :start-after: [START security_viewer_perms] :end-before: [END security_viewer_perms] @@ -66,7 +66,7 @@ User ^^^^ ``User`` users have ``Viewer`` permissions plus additional user permissions -.. exampleinclude:: /../airflow/www/security.py +.. exampleinclude:: /../../airflow/www/security.py :language: python :start-after: [START security_user_perms] :end-before: [END security_user_perms] @@ -77,7 +77,7 @@ Op ^^ ``Op`` users have ``User`` permissions plus additional op permissions -.. exampleinclude:: /../airflow/www/security.py +.. exampleinclude:: /../../airflow/www/security.py :language: python :start-after: [START security_op_perms] :end-before: [END security_op_perms] diff --git a/docs/security/api.rst b/docs/apache-airflow/security/api.rst similarity index 100% rename from docs/security/api.rst rename to docs/apache-airflow/security/api.rst diff --git a/docs/security/flower.rst b/docs/apache-airflow/security/flower.rst similarity index 100% rename from docs/security/flower.rst rename to docs/apache-airflow/security/flower.rst diff --git a/docs/security/index.rst b/docs/apache-airflow/security/index.rst similarity index 95% rename from docs/security/index.rst rename to docs/apache-airflow/security/index.rst index 31262daee8d7e..2ebc01b71a118 100644 --- a/docs/security/index.rst +++ b/docs/apache-airflow/security/index.rst @@ -27,4 +27,4 @@ Security * secrets/index -.. include:: /../.github/SECURITY.rst +.. include:: /../../.github/SECURITY.rst diff --git a/docs/security/kerberos.rst b/docs/apache-airflow/security/kerberos.rst similarity index 100% rename from docs/security/kerberos.rst rename to docs/apache-airflow/security/kerberos.rst diff --git a/docs/security/secrets/fernet.rst b/docs/apache-airflow/security/secrets/fernet.rst similarity index 100% rename from docs/security/secrets/fernet.rst rename to docs/apache-airflow/security/secrets/fernet.rst diff --git a/docs/security/secrets/index.rst b/docs/apache-airflow/security/secrets/index.rst similarity index 100% rename from docs/security/secrets/index.rst rename to docs/apache-airflow/security/secrets/index.rst diff --git a/docs/security/secrets/secrets-backend/aws-secrets-manaager-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/aws-secrets-manaager-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/aws-secrets-manaager-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/aws-secrets-manaager-backend.rst diff --git a/docs/security/secrets/secrets-backend/aws-ssm-parameter-store-secrets-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/aws-ssm-parameter-store-secrets-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/aws-ssm-parameter-store-secrets-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/aws-ssm-parameter-store-secrets-backend.rst diff --git a/docs/security/secrets/secrets-backend/azure-key-vault-secrets-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/azure-key-vault-secrets-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/azure-key-vault-secrets-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/azure-key-vault-secrets-backend.rst diff --git a/docs/security/secrets/secrets-backend/google-cloud-secret-manager-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/google-cloud-secret-manager-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/google-cloud-secret-manager-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/google-cloud-secret-manager-backend.rst diff --git a/docs/security/secrets/secrets-backend/hashicorp-vault-secrets-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/hashicorp-vault-secrets-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/hashicorp-vault-secrets-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/hashicorp-vault-secrets-backend.rst diff --git a/docs/security/secrets/secrets-backend/index.rst b/docs/apache-airflow/security/secrets/secrets-backend/index.rst similarity index 100% rename from docs/security/secrets/secrets-backend/index.rst rename to docs/apache-airflow/security/secrets/secrets-backend/index.rst diff --git a/docs/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst b/docs/apache-airflow/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst similarity index 100% rename from docs/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst rename to docs/apache-airflow/security/secrets/secrets-backend/local-filesystem-secrets-backend.rst diff --git a/docs/security/webserver.rst b/docs/apache-airflow/security/webserver.rst similarity index 100% rename from docs/security/webserver.rst rename to docs/apache-airflow/security/webserver.rst diff --git a/docs/security/workload.rst b/docs/apache-airflow/security/workload.rst similarity index 100% rename from docs/security/workload.rst rename to docs/apache-airflow/security/workload.rst diff --git a/docs/smart-sensor.rst b/docs/apache-airflow/smart-sensor.rst similarity index 100% rename from docs/smart-sensor.rst rename to docs/apache-airflow/smart-sensor.rst diff --git a/docs/stable-rest-api-ref.rst b/docs/apache-airflow/stable-rest-api-ref.rst similarity index 100% rename from docs/stable-rest-api-ref.rst rename to docs/apache-airflow/stable-rest-api-ref.rst diff --git a/docs/start.rst b/docs/apache-airflow/start.rst similarity index 100% rename from docs/start.rst rename to docs/apache-airflow/start.rst diff --git a/docs/static/exampleinclude.css b/docs/apache-airflow/static/exampleinclude.css similarity index 100% rename from docs/static/exampleinclude.css rename to docs/apache-airflow/static/exampleinclude.css diff --git a/docs/static/jira-links.js b/docs/apache-airflow/static/jira-links.js similarity index 100% rename from docs/static/jira-links.js rename to docs/apache-airflow/static/jira-links.js diff --git a/docs/templates/footer.html b/docs/apache-airflow/templates/footer.html similarity index 100% rename from docs/templates/footer.html rename to docs/apache-airflow/templates/footer.html diff --git a/docs/timezone.rst b/docs/apache-airflow/timezone.rst similarity index 100% rename from docs/timezone.rst rename to docs/apache-airflow/timezone.rst diff --git a/docs/tutorial.rst b/docs/apache-airflow/tutorial.rst similarity index 96% rename from docs/tutorial.rst rename to docs/apache-airflow/tutorial.rst index d5e85232a8948..932401425fb50 100644 --- a/docs/tutorial.rst +++ b/docs/apache-airflow/tutorial.rst @@ -30,7 +30,7 @@ Example Pipeline definition Here is an example of a basic pipeline definition. Do not worry if this looks complicated, a line by line explanation follows below. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] @@ -60,7 +60,7 @@ Importing Modules An Airflow pipeline is just a Python script that happens to define an Airflow DAG object. Let's start by importing the libraries we will need. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START import_module] :end-before: [END import_module] @@ -75,7 +75,7 @@ explicitly pass a set of arguments to each task's constructor (which would become redundant), or (better!) we can define a dictionary of default parameters that we can use when creating tasks. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START default_args] :end-before: [END default_args] @@ -96,7 +96,7 @@ that defines the ``dag_id``, which serves as a unique identifier for your DAG. We also pass the default argument dictionary that we just defined and define a ``schedule_interval`` of 1 day for the DAG. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START instantiate_dag] :end-before: [END instantiate_dag] @@ -107,7 +107,7 @@ Tasks are generated when instantiating operator objects. An object instantiated from an operator is called a task. The first argument ``task_id`` acts as a unique identifier for the task. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START basic_task] :end-before: [END basic_task] @@ -142,7 +142,7 @@ this feature exists, get you familiar with double curly brackets, and point to the most common template variable: ``{{ ds }}`` (today's "date stamp"). -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START jinja_template] :end-before: [END jinja_template] @@ -184,7 +184,7 @@ We can add documentation for DAG or each single task. DAG documentation only sup markdown so far and task documentation support plain text, markdown, reStructuredText, json, yaml. -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START documentation] :end-before: [END documentation] @@ -231,7 +231,7 @@ Recap Alright, so we have a pretty basic DAG. At this point your code should look something like this: -.. exampleinclude:: /../airflow/example_dags/tutorial.py +.. exampleinclude:: /../../airflow/example_dags/tutorial.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] diff --git a/docs/tutorial_taskflow_api.rst b/docs/apache-airflow/tutorial_taskflow_api.rst similarity index 89% rename from docs/tutorial_taskflow_api.rst rename to docs/apache-airflow/tutorial_taskflow_api.rst index 6c075e0dd591b..29422dabbc226 100644 --- a/docs/tutorial_taskflow_api.rst +++ b/docs/apache-airflow/tutorial_taskflow_api.rst @@ -34,7 +34,7 @@ Example "Taskflow API" ETL Pipeline Here is very simple ETL pipeline using the Taskflow API paradigm. A more detailed explanation is given below. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] @@ -55,7 +55,7 @@ We are creating a DAG which is the collection of our tasks with dependencies bet the tasks. This is a very simple definition, since we just want the DAG to be run when we set this up with Airflow, without any retries or complex scheduling. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START instantiate_dag] :end-before: [END instantiate_dag] @@ -65,7 +65,7 @@ Tasks In this data pipeline, tasks are created based on Python functions using the decorator as shown below. The function name acts as a unique identifier for the task. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START extract] :end-before: [END extract] @@ -79,7 +79,7 @@ Main flow of the DAG Now that we have the Extract, Transform, and Load tasks defined based on the Python functions, we can move to the main part of the DAG. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START main_flow] :end-before: [END main_flow] @@ -95,7 +95,7 @@ But how? For experienced Airflow DAG authors, this is startlingly simple! Let's contrast this with how this DAG had to be written before Airflow 2.0 below: -.. exampleinclude:: /../airflow/example_dags/tutorial_etl_dag.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_etl_dag.py :language: python :start-after: [START tutorial] :end-before: [END tutorial] @@ -106,7 +106,7 @@ it is all abstracted from the DAG developer. Let's examine this in detail by looking at the Transform task in isolation since it is in the middle of the data pipeline. In Airflow 1.x, this task is defined as shown below: -.. exampleinclude:: /../airflow/example_dags/tutorial_etl_dag.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_etl_dag.py :language: python :start-after: [START transform_function] :end-before: [END transform_function] @@ -117,7 +117,7 @@ into another Xcom variable which will then be used by the Load task. Contrasting that with Taskflow API in Airflow 2.0 as shown below. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START transform] :end-before: [END transform] @@ -130,7 +130,7 @@ Similarly, task dependencies are automatically generated within TaskFlows based functional invocation of tasks. In Airflow 1.x, tasks had to be explicitly created and dependencies specified as shown below. -.. exampleinclude:: /../airflow/example_dags/tutorial_etl_dag.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_etl_dag.py :language: python :start-after: [START main_flow] :end-before: [END main_flow] @@ -138,7 +138,7 @@ dependencies specified as shown below. In contrast, with the Taskflow API in Airflow 2.0, the invocation itself automatically generates the dependencies as shown below. -.. exampleinclude:: /../airflow/example_dags/tutorial_taskflow_api_etl.py +.. exampleinclude:: /../../airflow/example_dags/tutorial_taskflow_api_etl.py :language: python :start-after: [START main_flow] :end-before: [END main_flow] diff --git a/docs/ui.rst b/docs/apache-airflow/ui.rst similarity index 100% rename from docs/ui.rst rename to docs/apache-airflow/ui.rst diff --git a/docs/usage-cli.rst b/docs/apache-airflow/usage-cli.rst similarity index 100% rename from docs/usage-cli.rst rename to docs/apache-airflow/usage-cli.rst diff --git a/docs/build_docs.py b/docs/build_docs.py index 30b9b7f50e203..7e1367e90faf7 100755 --- a/docs/build_docs.py +++ b/docs/build_docs.py @@ -88,17 +88,7 @@ def _out_dir(self) -> str: @property def _src_dir(self) -> str: - # TODO(mik-laj): - # After migrating the content from the core to providers, we should move all documentation from . - # to /airflow/ to keep the directory structure more maintainable. - if self.package_name == 'apache-airflow': - return DOCS_DIR - elif self.package_name.startswith('apache-airflow-providers-') or ( - self.package_name == 'apache-airflow-providers' - ): - return f"{DOCS_DIR}/{self.package_name}" - else: - raise Exception(F"Unsupported package: {self.package_name}") + return f"{DOCS_DIR}/{self.package_name}" def clean_files(self) -> None: """Cleanup all artifacts generated by previous builds.""" diff --git a/docs/conf.py b/docs/conf.py index d853cacefa52b..bfd04b503f03d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -112,6 +112,7 @@ 'airflow_intersphinx', "sphinxcontrib.spelling", 'sphinx_airflow_theme', + 'redirects', ] if PACKAGE_NAME == 'apache-airflow': extensions.extend( @@ -142,21 +143,8 @@ exclude_patterns = [ # We only link to selected subpackages. '_api/airflow/index.rst', - # We have custom page - operators-and-hooks-ref.rst - '_api/airflow/providers/index.rst', - # Packages with subpackages - "_api/airflow/providers/microsoft/index.rst", - "_api/airflow/providers/apache/index.rst", - "_api/airflow/providers/cncf/index.rst", - # Templates or partials - 'autoapi_templates', - 'howto/operator/google/_partials', - 'howto/operator/microsoft/_partials', - 'apache-airflow-providers-*/', - 'apache-airflow-providers', - 'rtd-deprecation', 'README.rst', - ] + glob.glob('apache-airflow-providers-*') + ] elif PACKAGE_NAME.startswith('apache-airflow-providers-'): exclude_patterns = [ 'operators/_partials', @@ -227,7 +215,7 @@ def _get_rst_filepath_from_path(filepath: str): # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". if PACKAGE_NAME == 'apache-airflow': - html_static_path = ['static'] + html_static_path = ['apache-airflow/static'] else: html_static_path = [] # A list of JavaScript filename. The entry must be a filename string or a @@ -463,15 +451,12 @@ def _load_config(): # Relative path to output the AutoAPI files into. This can also be used to place the generated documentation # anywhere in your documentation hierarchy. -if PACKAGE_NAME == 'apache-airflow': - autoapi_root = '_api' -else: - autoapi_root = f'{PACKAGE_NAME}/_api' +autoapi_root = f'{PACKAGE_NAME}/_api' # Whether to insert the generated documentation into the TOC tree. If this is False, the default AutoAPI # index page is not generated and you will need to include the generated documentation in a # TOC tree entry yourself. -autoapi_add_toctree_entry = bool(PACKAGE_NAME == 'apache-airflow') +autoapi_add_toctree_entry = False # -- Options for ext.exampleinclude -------------------------------------------- exampleinclude_sourceroot = os.path.abspath('..') diff --git a/docs/exts/redirects.py b/docs/exts/redirects.py index beca2cb1d2b54..787412413d0c8 100644 --- a/docs/exts/redirects.py +++ b/docs/exts/redirects.py @@ -18,7 +18,6 @@ import os from sphinx.builders import html as builders -from sphinx.errors import ExtensionError from sphinx.util import logging TEMPLATE = '' @@ -30,7 +29,8 @@ def generate_redirects(app): """Generate redirects files.""" redirect_file_path = os.path.join(app.srcdir, app.config.redirects_file) if not os.path.exists(redirect_file_path): - raise ExtensionError(f"Could not find redirects file at '{redirect_file_path}'") + log.info("Could not found the redirect file: %s", redirect_file_path) + return in_suffix = next(iter(app.config.source_suffix.keys()))