Add example dag and system test for LocalFilesystemToGCSOperator (#9043)

airflow/providers/google/cloud/example_dags/example_local_to_gcs.py

@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from airflow import models
+from airflow.providers.google.cloud.operators.local_to_gcs import LocalFilesystemToGCSOperator
+from airflow.utils import dates
+
+# [START howto_gcs_environment_variables]
+BUCKET_NAME = os.environ.get('GCP_GCS_BUCKET', 'example-bucket-name')
+PATH_TO_UPLOAD_FILE = os.environ.get('GCP_GCS_PATH_TO_UPLOAD_FILE', 'example-text.txt')
+DESTINATION_FILE_LOCATION = os.environ.get('GCP_GCS_DESTINATION_FILE_LOCATION', 'example-text.txt')
+# [END howto_gcs_environment_variables]
+
+with models.DAG(
+    'example_local_to_gcs',
+    default_args=dict(start_date=dates.days_ago(1)),
+    schedule_interval=None,
+    tags=['example']
+) as dag:
+    # [START howto_operator_local_filesystem_to_gcs]
+    upload_file = LocalFilesystemToGCSOperator(
+        task_id="upload_file",
+        src=PATH_TO_UPLOAD_FILE,
+        dst=DESTINATION_FILE_LOCATION,
+        bucket=BUCKET_NAME,
+    )
+    # [END howto_operator_local_filesystem_to_gcs]

docs/howto/operator/gcp/local_to_gcs.rst

@@ -0,0 +1,58 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+
+Upload data from Local Filesystem to Google Cloud Storage
+=========================================================
+The `Google Cloud Storage <https://cloud.google.com/storage/>`__  (GCS) is used to store large data from various applications.
+This page shows how to upload data from local filesystem to GCS.
+
+.. contents::
+  :depth: 1
+  :local:
+
+
+Prerequisite Tasks
+^^^^^^^^^^^^^^^^^^
+
+.. include:: _partials/prerequisite_tasks.rst
+
+.. _howto/operator:LocalFilesystemToGCSOperator:
+
+LocalFileSystemToGCSOperator
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:class:`~airflow.providers.google.cloud.operators.local_to_gcs.LocalFilesystemToGCSOperator` allows you to upload
+data from local filesystem to GCS.
+
+When you use this operator, you can optionally compress the data being uploaded.
+
+Below is an example of using this operator to upload a file to GCS.
+
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_local_to_gcs.py
+    :language: python
+    :dedent: 0
+    :start-after: [START howto_operator_local_filesystem_to_gcs]
+    :end-before: [END howto_operator_local_filesystem_to_gcs]
+
+
+Reference
+---------
+
+For further information, look at:
+
+* `Google Cloud Storage Documentation <https://cloud.google.com/storage/>`__

docs/operators-and-hooks-ref.rst

@@ -918,7 +918,7 @@ These integrations allow you to copy data from/to Google Cloud Platform.
 
    * - Local
      - `Google Cloud Storage (GCS) <https://cloud.google.com/gcs/>`__
-     -
+     - :doc:`How to use <howto/operator/gcp/local_to_gcs>`
      - :mod:`airflow.providers.google.cloud.operators.local_to_gcs`
 
    * - `Microsoft SQL Server (MSSQL) <https://www.microsoft.com/pl-pl/sql-server/sql-server-downloads>`__

tests/providers/google/cloud/operators/test_local_to_gcs_system.py

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+
+import pytest
+
+from airflow.providers.google.cloud.example_dags.example_local_to_gcs import BUCKET_NAME, PATH_TO_UPLOAD_FILE
+from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY
+from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
+
+
+@pytest.mark.backend("mysql", "postgres")
+@pytest.mark.credential_file(GCP_GCS_KEY)
+class LocalFilesystemToGCSOperatorExampleDagsTest(GoogleSystemTest):
+
+    @provide_gcp_context(GCP_GCS_KEY)
+    def setUp(self):
+        super().setUp()
+        self.create_gcs_bucket(BUCKET_NAME)
+        with open(PATH_TO_UPLOAD_FILE, 'w+') as file:
+            file.writelines(['example test files'])
+
+    @provide_gcp_context(GCP_GCS_KEY)
+    def tearDown(self):
+        self.delete_gcs_bucket(BUCKET_NAME)
+        os.remove(PATH_TO_UPLOAD_FILE)
+        super().tearDown()
+
+    @provide_gcp_context(GCP_GCS_KEY)
+    def test_run_example_dag(self):
+        self.run_dag('example_local_to_gcs', CLOUD_DAG_FOLDER)

tests/test_project_structure.py

@@ -135,7 +135,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
         ('cloud', 'cassandra_to_gcs'),
         ('cloud', 'mysql_to_gcs'),
         ('cloud', 'mssql_to_gcs'),
-        ('cloud', 'local_to_gcs'),
     }
 
     MISSING_DOC_GUIDES = {
@@ -150,7 +149,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
         'dlp',
         'gcs_to_bigquery',
         'kubernetes_engine',
-        'local_to_gcs',
         'mlengine',
         'mssql_to_gcs',
         'mysql_to_gcs',