diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index b46f17323d64f..a6f27228049f0 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -127,26 +127,26 @@ integrations: - integration-name: Amazon EMR external-doc-url: https://aws.amazon.com/emr/ how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/emr.rst + - /docs/apache-airflow-providers-amazon/operators/emr/emr.rst logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon EMR on EKS external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/emr-eks.html how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/emr_eks.rst + - /docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon EMR Serverless external-doc-url: https://docs.aws.amazon.com/emr/latest/EMR-Serverless-UserGuide/emr-serverless.html how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/emr_serverless.rst + - /docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst logo: /integration-logos/aws/Amazon-EMR_light-bg@4x.png tags: [aws] - integration-name: Amazon Glacier external-doc-url: https://aws.amazon.com/glacier/ logo: /integration-logos/aws/Amazon-S3-Glacier_light-bg@4x.png how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/glacier.rst + - /docs/apache-airflow-providers-amazon/operators/s3/glacier.rst tags: [aws] - integration-name: Amazon Kinesis Data Firehose external-doc-url: https://aws.amazon.com/kinesis/data-firehose/ @@ -162,14 +162,14 @@ integrations: external-doc-url: https://aws.amazon.com/redshift/ logo: /integration-logos/aws/Amazon-Redshift_light-bg@4x.png how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/redshift_sql.rst - - /docs/apache-airflow-providers-amazon/operators/redshift_cluster.rst + - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst + - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst tags: [aws] - integration-name: Amazon Redshift Data external-doc-url: https://aws.amazon.com/redshift/ logo: /integration-logos/aws/Amazon-Redshift_light-bg@4x.png how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/redshift_data.rst + - /docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst tags: [aws] - integration-name: Amazon SageMaker external-doc-url: https://aws.amazon.com/sagemaker/ @@ -203,7 +203,7 @@ integrations: external-doc-url: https://aws.amazon.com/s3/ logo: /integration-logos/aws/Amazon-Simple-Storage-Service-S3_light-bg@4x.png how-to-guide: - - /docs/apache-airflow-providers-amazon/operators/s3.rst + - /docs/apache-airflow-providers-amazon/operators/s3/s3.rst tags: [aws] - integration-name: Amazon Systems Manager (SSM) external-doc-url: https://aws.amazon.com/systems-manager/ @@ -506,74 +506,74 @@ hooks: transfers: - source-integration-name: Amazon DynamoDB target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.dynamodb_to_s3 - source-integration-name: Google Cloud Storage (GCS) target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.gcs_to_s3 - source-integration-name: Amazon Glacier target-integration-name: Google Cloud Storage (GCS) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/glacier_to_gcs.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst python-module: airflow.providers.amazon.aws.transfers.glacier_to_gcs - source-integration-name: Google target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.google_api_to_s3 - source-integration-name: Apache Hive target-integration-name: Amazon DynamoDB - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/hive_to_dynamodb.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst python-module: airflow.providers.amazon.aws.transfers.hive_to_dynamodb - source-integration-name: Internet Message Access Protocol (IMAP) target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.imap_attachment_to_s3 - source-integration-name: MongoDB target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3 - source-integration-name: Amazon Redshift target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.redshift_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Amazon Redshift - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_redshift - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: SSH File Transfer Protocol (SFTP) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_sftp - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.sftp_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: File Transfer Protocol (FTP) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_ftp - source-integration-name: Exasol target-integration-name: Amazon Simple Storage Service (S3) python-module: airflow.providers.amazon.aws.transfers.exasol_to_s3 - source-integration-name: File Transfer Protocol (FTP) target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.ftp_to_s3 - source-integration-name: Salesforce target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.salesforce_to_s3 - source-integration-name: Local target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.local_to_s3 - source-integration-name: Common SQL target-integration-name: Amazon Simple Storage Service (S3) - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.sql_to_s3 - source-integration-name: Amazon Simple Storage Service (S3) target-integration-name: Common SQL - how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sql.rst + how-to-guide: /docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst python-module: airflow.providers.amazon.aws.transfers.s3_to_sql - source-integration-name: Amazon Web Services target-integration-name: Amazon Web Services diff --git a/docs/apache-airflow-providers-amazon/operators/_partials/prerequisite_tasks.rst b/docs/apache-airflow-providers-amazon/_partials/prerequisite_tasks.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/_partials/prerequisite_tasks.rst rename to docs/apache-airflow-providers-amazon/_partials/prerequisite_tasks.rst diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index 9abb85bb3c763..2bae9ea118f4e 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -27,6 +27,7 @@ Content Connection types Operators + Transfers Deferrable Operators Secrets backends Logging for Tasks diff --git a/docs/apache-airflow-providers-amazon/operators/appflow.rst b/docs/apache-airflow-providers-amazon/operators/appflow.rst index 65604ee97b7ab..8ef7e9c288fc6 100644 --- a/docs/apache-airflow-providers-amazon/operators/appflow.rst +++ b/docs/apache-airflow-providers-amazon/operators/appflow.rst @@ -33,7 +33,7 @@ AWS PrivateLink, reducing exposure to security threats. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/athena.rst b/docs/apache-airflow-providers-amazon/operators/athena.rst index 7e1d4e45e6b92..a47227988291c 100644 --- a/docs/apache-airflow-providers-amazon/operators/athena.rst +++ b/docs/apache-airflow-providers-amazon/operators/athena.rst @@ -28,7 +28,7 @@ to your data in S3, define the schema, and start querying using standard SQL. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/batch.rst b/docs/apache-airflow-providers-amazon/operators/batch.rst index ba280cb38d37e..bcfb86dbf73f6 100644 --- a/docs/apache-airflow-providers-amazon/operators/batch.rst +++ b/docs/apache-airflow-providers-amazon/operators/batch.rst @@ -27,7 +27,7 @@ infrastructure. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/cloudformation.rst b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst index 6b2730c348192..4051be0ccd225 100644 --- a/docs/apache-airflow-providers-amazon/operators/cloudformation.rst +++ b/docs/apache-airflow-providers-amazon/operators/cloudformation.rst @@ -29,7 +29,7 @@ create and delete a collection of resources together as a single unit (a stack). Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/datasync.rst b/docs/apache-airflow-providers-amazon/operators/datasync.rst index b0b6100f4497b..aca6d5d755145 100644 --- a/docs/apache-airflow-providers-amazon/operators/datasync.rst +++ b/docs/apache-airflow-providers-amazon/operators/datasync.rst @@ -26,7 +26,7 @@ the internet or AWS Direct Connect. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/dms.rst b/docs/apache-airflow-providers-amazon/operators/dms.rst index c741735b866bd..3a9f38e72ad25 100644 --- a/docs/apache-airflow-providers-amazon/operators/dms.rst +++ b/docs/apache-airflow-providers-amazon/operators/dms.rst @@ -34,7 +34,7 @@ different database engines, such as from an Oracle database to a PostgreSQL data Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/dynamodb.rst b/docs/apache-airflow-providers-amazon/operators/dynamodb.rst index 1a3693a45567c..3333a6648daf4 100644 --- a/docs/apache-airflow-providers-amazon/operators/dynamodb.rst +++ b/docs/apache-airflow-providers-amazon/operators/dynamodb.rst @@ -28,7 +28,7 @@ data import and export tools. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Sensors diff --git a/docs/apache-airflow-providers-amazon/operators/ec2.rst b/docs/apache-airflow-providers-amazon/operators/ec2.rst index 5796c514e0238..2018d8113fcf9 100644 --- a/docs/apache-airflow-providers-amazon/operators/ec2.rst +++ b/docs/apache-airflow-providers-amazon/operators/ec2.rst @@ -15,9 +15,9 @@ specific language governing permissions and limitations under the License. -========================================= -Amazon Amazon Elastic Compute Cloud (EC2) -========================================= +================================== +Amazon Elastic Compute Cloud (EC2) +================================== `Amazon Elastic Compute Cloud (Amazon EC2) `__ is a web service that provides resizable computing capacity—literally, servers in Amazon's data centers—that you use to build and host your software systems. @@ -25,7 +25,7 @@ computing capacity—literally, servers in Amazon's data centers—that you use Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/ecs.rst b/docs/apache-airflow-providers-amazon/operators/ecs.rst index 2ba584cb77fdf..d513485a9a990 100644 --- a/docs/apache-airflow-providers-amazon/operators/ecs.rst +++ b/docs/apache-airflow-providers-amazon/operators/ecs.rst @@ -28,7 +28,7 @@ Airflow provides operators to run Task Definitions on an ECS cluster. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/eks.rst b/docs/apache-airflow-providers-amazon/operators/eks.rst index 7861ad60948f3..e11b721be37b5 100644 --- a/docs/apache-airflow-providers-amazon/operators/eks.rst +++ b/docs/apache-airflow-providers-amazon/operators/eks.rst @@ -29,7 +29,7 @@ Airflow provides operators to create and interact with the EKS clusters and comp Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/emr.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr.rst similarity index 99% rename from docs/apache-airflow-providers-amazon/operators/emr.rst rename to docs/apache-airflow-providers-amazon/operators/emr/emr.rst index 6597413700a8e..4b517dab8ff3b 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr.rst @@ -30,7 +30,7 @@ Storage Service (Amazon S3) and Amazon DynamoDB. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/emr_eks.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst similarity index 99% rename from docs/apache-airflow-providers-amazon/operators/emr_eks.rst rename to docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst index ee2ee50ec13b1..dc31c8f984618 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr_eks.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_eks.rst @@ -26,7 +26,7 @@ Amazon EKS. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/emr_serverless.rst b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst similarity index 98% rename from docs/apache-airflow-providers-amazon/operators/emr_serverless.rst rename to docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst index 15b0c6de0f478..6a30f52825cb4 100644 --- a/docs/apache-airflow-providers-amazon/operators/emr_serverless.rst +++ b/docs/apache-airflow-providers-amazon/operators/emr/emr_serverless.rst @@ -29,7 +29,7 @@ and manage clusters. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/emr/index.rst b/docs/apache-airflow-providers-amazon/operators/emr/index.rst new file mode 100644 index 0000000000000..57d517d3b155d --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/emr/index.rst @@ -0,0 +1,28 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Amazon EMR Operators +==================== + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-amazon/operators/glue.rst b/docs/apache-airflow-providers-amazon/operators/glue.rst index 0690ec5b72397..e582e9d415b67 100644 --- a/docs/apache-airflow-providers-amazon/operators/glue.rst +++ b/docs/apache-airflow-providers-amazon/operators/glue.rst @@ -27,7 +27,7 @@ your data and putting it to use in minutes instead of months. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/index.rst b/docs/apache-airflow-providers-amazon/operators/index.rst index 428d3758a3649..7c7082148e2fd 100644 --- a/docs/apache-airflow-providers-amazon/operators/index.rst +++ b/docs/apache-airflow-providers-amazon/operators/index.rst @@ -25,5 +25,7 @@ Amazon AWS Operators :maxdepth: 1 :glob: + emr/index + redshift/index + s3/index * - transfer/index diff --git a/docs/apache-airflow-providers-amazon/operators/lambda.rst b/docs/apache-airflow-providers-amazon/operators/lambda.rst index 2ec14eb5ba4b2..79649f106fdee 100644 --- a/docs/apache-airflow-providers-amazon/operators/lambda.rst +++ b/docs/apache-airflow-providers-amazon/operators/lambda.rst @@ -28,7 +28,7 @@ You can set up your code to automatically trigger from other AWS services or cal Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/quicksight.rst b/docs/apache-airflow-providers-amazon/operators/quicksight.rst index 79c7dc500cf38..cbca98d7d59a4 100644 --- a/docs/apache-airflow-providers-amazon/operators/quicksight.rst +++ b/docs/apache-airflow-providers-amazon/operators/quicksight.rst @@ -28,7 +28,7 @@ QuickSight Super-fast, Parallel, In-Memory, Calculation Engine (SPICE). Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/rds.rst b/docs/apache-airflow-providers-amazon/operators/rds.rst index 1cd1cd40701dd..bca9c64af1e0e 100644 --- a/docs/apache-airflow-providers-amazon/operators/rds.rst +++ b/docs/apache-airflow-providers-amazon/operators/rds.rst @@ -27,7 +27,7 @@ common database administration tasks. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/redshift/index.rst b/docs/apache-airflow-providers-amazon/operators/redshift/index.rst new file mode 100644 index 0000000000000..35b0542b8a423 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/redshift/index.rst @@ -0,0 +1,28 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Amazon Redshift Operators +========================= + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-amazon/operators/redshift_cluster.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst similarity index 97% rename from docs/apache-airflow-providers-amazon/operators/redshift_cluster.rst rename to docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst index d61aa4acfcd9d..7c5d22486b39f 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift_cluster.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_cluster.rst @@ -15,9 +15,9 @@ specific language governing permissions and limitations under the License. -=============== -Amazon Redshift -=============== +========================= +Amazon Redshift (Cluster) +========================= `Amazon Redshift `__ manages all the work of setting up, operating, and scaling a data warehouse: provisioning capacity, monitoring and backing up the cluster, and applying patches and upgrades to @@ -27,7 +27,7 @@ business and customers. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/redshift_data.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst similarity index 97% rename from docs/apache-airflow-providers-amazon/operators/redshift_data.rst rename to docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst index 9ec14dbaeea19..782d0cf6992fb 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift_data.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_data.rst @@ -27,7 +27,7 @@ business and customers. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/redshift_sql.rst b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst similarity index 95% rename from docs/apache-airflow-providers-amazon/operators/redshift_sql.rst rename to docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst index 17d43f4097391..1eb25939afcb5 100644 --- a/docs/apache-airflow-providers-amazon/operators/redshift_sql.rst +++ b/docs/apache-airflow-providers-amazon/operators/redshift/redshift_sql.rst @@ -17,9 +17,9 @@ .. _howto/operator:RedshiftSQLOperator: -=============== -Amazon Redshift -=============== +=================== +Amazon Redshift SQL +=================== `Amazon Redshift `__ manages all the work of setting up, operating, and scaling a data warehouse: provisioning capacity, monitoring and backing up the cluster, and applying patches and upgrades to @@ -29,7 +29,7 @@ business and customers. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/glacier.rst b/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst similarity index 98% rename from docs/apache-airflow-providers-amazon/operators/glacier.rst rename to docs/apache-airflow-providers-amazon/operators/s3/glacier.rst index 3a8de01a452ab..9dca7a776cff8 100644 --- a/docs/apache-airflow-providers-amazon/operators/glacier.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3/glacier.rst @@ -25,7 +25,7 @@ and extremely low-cost Amazon S3 cloud storage class for data archiving and long Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/s3/index.rst b/docs/apache-airflow-providers-amazon/operators/s3/index.rst new file mode 100644 index 0000000000000..f0d7f7d0871c3 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/s3/index.rst @@ -0,0 +1,28 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +Amazon S3 Operators +=================== + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/apache-airflow-providers-amazon/operators/s3.rst b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst similarity index 99% rename from docs/apache-airflow-providers-amazon/operators/s3.rst rename to docs/apache-airflow-providers-amazon/operators/s3/s3.rst index 3d594fcdcb2bb..ef9d0864490f9 100644 --- a/docs/apache-airflow-providers-amazon/operators/s3.rst +++ b/docs/apache-airflow-providers-amazon/operators/s3/s3.rst @@ -25,7 +25,7 @@ You can use Amazon S3 to store and retrieve any amount of data at any time, from Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/sagemaker.rst b/docs/apache-airflow-providers-amazon/operators/sagemaker.rst index 1aa82b6cdc697..933d162df8fa4 100644 --- a/docs/apache-airflow-providers-amazon/operators/sagemaker.rst +++ b/docs/apache-airflow-providers-amazon/operators/sagemaker.rst @@ -29,7 +29,7 @@ Airflow provides operators to create and interact with SageMaker Jobs and Pipeli Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/sns.rst b/docs/apache-airflow-providers-amazon/operators/sns.rst index caaa48452795d..903b8d7630b4a 100644 --- a/docs/apache-airflow-providers-amazon/operators/sns.rst +++ b/docs/apache-airflow-providers-amazon/operators/sns.rst @@ -30,7 +30,7 @@ messages (SMS). Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/sqs.rst b/docs/apache-airflow-providers-amazon/operators/sqs.rst index abe55163c97fa..13c806626ee65 100644 --- a/docs/apache-airflow-providers-amazon/operators/sqs.rst +++ b/docs/apache-airflow-providers-amazon/operators/sqs.rst @@ -15,9 +15,9 @@ specific language governing permissions and limitations under the License. -========== -Amazon SQS -========== +================================= +Amazon Simple Queue Service (SQS) +================================= `Amazon Simple Queue Service (SQS) `__ is a fully managed message queuing service that enables you to decouple and scale microservices, distributed systems, and serverless @@ -29,7 +29,7 @@ or requiring other services to be available. Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/operators/step_functions.rst b/docs/apache-airflow-providers-amazon/operators/step_functions.rst index 91984dc5986b6..1f207b4576f98 100644 --- a/docs/apache-airflow-providers-amazon/operators/step_functions.rst +++ b/docs/apache-airflow-providers-amazon/operators/step_functions.rst @@ -26,7 +26,7 @@ machines to execute the steps of your application in a reliable and scalable fas Prerequisite Tasks ------------------ -.. include:: _partials/prerequisite_tasks.rst +.. include:: ../_partials/prerequisite_tasks.rst Operators --------- diff --git a/docs/apache-airflow-providers-amazon/redirects.txt b/docs/apache-airflow-providers-amazon/redirects.txt index e8de475dd33f4..bf35801e962e9 100644 --- a/docs/apache-airflow-providers-amazon/redirects.txt +++ b/docs/apache-airflow-providers-amazon/redirects.txt @@ -15,8 +15,33 @@ # specific language governing permissions and limitations # under the License. -operators/s3_to_redshift.rst operators/transfer/s3_to_redshift.rst -operators/google_api_to_s3_transfer.rst operators/transfer/google_api_to_s3.rst -operators/imap_attachment_to_s3.rst operators/transfer/imap_attachment_to_s3.rst -operators/salesforce_to_s3.rst operators/transfer/salesforce_to_s3.rst +operators/s3_to_redshift.rst transfer/s3_to_redshift.rst +operators/google_api_to_s3_transfer.rst transfer/google_api_to_s3.rst +operators/imap_attachment_to_s3.rst transfer/imap_attachment_to_s3.rst +operators/salesforce_to_s3.rst transfer/salesforce_to_s3.rst operators/sqs_publish.rst operators/sqs.rst +operators/emr.rst operators/emr/emr.rst +operators/emr_eks.rst operators/emr/emr_eks.rst +operators/emr_serverless.rst operators/emr/emr_serverless.rst +operators/redshift_sql.rst operators/redshift/redshift_sql.rst +operators/redshift_data.rst operators/redshift/redshift_data.rst +operators/redshift_cluster.rst operators/redshift/redshift_cluster.rst +operators/s3.rst operators/s3/s3.rst +operators/glacier.rst operators/s3/glacier.rst +operators/transfer/dynamodb_to_s3.rst.rst transfer/dynamodb_to_s3.rst.rst +operators/transfer/ftp_to_s3.rst.rst transfer/ftp_to_s3.rst.rst +operators/transfer/gcs_to_s3.rst.rst transfer/gcs_to_s3.rst.rst +operators/transfer/glacier_to_gcs.rst transfer/glacier_to_gcs.rst +operators/transfer/google_api_to_s3.rst transfer/google_api_to_s3.rst +operators/transfer/hive_to_dynamodb.rst transfer/hive_to_dynamodb.rst +operators/transfer/imap_attachment_to_s3.rst transfer/imap_attachment_to_s3.rst +operators/transfer/local_to_s3.rst transfer/local_to_s3.rst +operators/transfer/mongo_to_s3.rst transfer/mongo_to_s3.rst +operators/transfer/redshift_to_s3.rst transfer/redshift_to_s3.rst +operators/transfer/s3_to_ftp.rst transfer/s3_to_ftp.rst +operators/transfer/s3_to_redshift.rst transfer/s3_to_redshift.rst +operators/transfer/s3_to_sftp.rst transfer/s3_to_sftp.rst +operators/transfer/s3_to_sql.rst transfer/s3_to_sql.rst +operators/transfer/salesforce_to_s3.rst transfer/salesforce_to_s3.rst +operators/transfer/sftp_to_s3.rst transfer/sftp_to_s3.rst +operators/transfer/sql_to_s3.rst transfer/sql_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/dynamodb_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/dynamodb_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/ftp_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/ftp_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/gcs_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/gcs_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/glacier_to_gcs.rst b/docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/glacier_to_gcs.rst rename to docs/apache-airflow-providers-amazon/transfer/glacier_to_gcs.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/google_api_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/google_api_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/hive_to_dynamodb.rst b/docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/hive_to_dynamodb.rst rename to docs/apache-airflow-providers-amazon/transfer/hive_to_dynamodb.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/imap_attachment_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/imap_attachment_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/index.rst b/docs/apache-airflow-providers-amazon/transfer/index.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/index.rst rename to docs/apache-airflow-providers-amazon/transfer/index.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/local_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/local_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/mongo_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/mongo_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/redshift_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/redshift_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/s3_to_ftp.rst rename to docs/apache-airflow-providers-amazon/transfer/s3_to_ftp.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/s3_to_redshift.rst rename to docs/apache-airflow-providers-amazon/transfer/s3_to_redshift.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sftp.rst rename to docs/apache-airflow-providers-amazon/transfer/s3_to_sftp.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sql.rst b/docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/s3_to_sql.rst rename to docs/apache-airflow-providers-amazon/transfer/s3_to_sql.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/salesforce_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/salesforce_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/sftp_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/sftp_to_s3.rst diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst b/docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst similarity index 100% rename from docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst rename to docs/apache-airflow-providers-amazon/transfer/sql_to_s3.rst diff --git a/scripts/in_container/run_provider_yaml_files_check.py b/scripts/in_container/run_provider_yaml_files_check.py index 9f821ce74dd58..5140c53cb3ded 100755 --- a/scripts/in_container/run_provider_yaml_files_check.py +++ b/scripts/in_container/run_provider_yaml_files_check.py @@ -395,9 +395,15 @@ def check_doc_files(yaml_files: dict[str, dict]): ) console.print("[yellow]Suspended providers:[/]") console.print(suspended_providers) + + expected_doc_files = chain( + DOCS_DIR.glob("apache-airflow-providers-*/operators/**/*.rst"), + DOCS_DIR.glob("apache-airflow-providers-*/transfer/**/*.rst"), + ) + expected_doc_urls = { f"/docs/{f.relative_to(DOCS_DIR).as_posix()}" - for f in DOCS_DIR.glob("apache-airflow-providers-*/operators/**/*.rst") + for f in expected_doc_files if f.name != "index.rst" and "_partials" not in f.parts and not any(f.relative_to(DOCS_DIR).as_posix().startswith(s) for s in suspended_providers)