From 9b52b5c06cdb5fb8d61b4987552b87e4ee5434c2 Mon Sep 17 00:00:00 2001 From: Omair Khan Date: Sat, 4 Jul 2020 22:37:41 +0530 Subject: [PATCH] Documentation for Google Cloud Data Loss Prevention (#8201) --- .../google/cloud/example_dags/example_dlp.py | 7 +- .../providers/google/cloud/operators/dlp.py | 12 +++ .../operator/gcp/data_loss_prevention.rst | 92 +++++++++++++++++++ docs/operators-and-hooks-ref.rst | 2 +- 4 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 docs/howto/operator/gcp/data_loss_prevention.rst diff --git a/airflow/providers/google/cloud/example_dags/example_dlp.py b/airflow/providers/google/cloud/example_dags/example_dlp.py index 43abbef45832b..91a5ac8dbfe44 100644 --- a/airflow/providers/google/cloud/example_dags/example_dlp.py +++ b/airflow/providers/google/cloud/example_dags/example_dlp.py @@ -57,7 +57,7 @@ default_args=default_args, schedule_interval=None, # Override to match your needs tags=['example'], -) as dag: +) as dag: # [START howto_operator_dlp_create_inspect_template] create_template = CloudDLPCreateInspectTemplateOperator( project_id=GCP_PROJECT, inspect_template=INSPECT_TEMPLATE, @@ -66,7 +66,9 @@ do_xcom_push=True, dag=dag, ) + # [END howto_operator_dlp_create_inspect_template] + # [START howto_operator_dlp_use_inspect_template] inspect_content = CloudDLPInspectContentOperator( task_id="inpsect_content", project_id=GCP_PROJECT, @@ -74,12 +76,15 @@ inspect_template_name="{{ task_instance.xcom_pull('create_template', key='return_value')['name'] }}", dag=dag, ) + # [END howto_operator_dlp_use_inspect_template] + # [START howto_operator_dlp_delete_inspect_template] delete_template = CloudDLPDeleteInspectTemplateOperator( task_id="delete_template", template_id=TEMPLATE_ID, project_id=GCP_PROJECT, dag=dag, ) + # [END howto_operator_dlp_delete_inspect_template] create_template >> inspect_content >> delete_template diff --git a/airflow/providers/google/cloud/operators/dlp.py b/airflow/providers/google/cloud/operators/dlp.py index 9617e8ede494e..210b6d142051d 100644 --- a/airflow/providers/google/cloud/operators/dlp.py +++ b/airflow/providers/google/cloud/operators/dlp.py @@ -269,6 +269,10 @@ class CloudDLPCreateInspectTemplateOperator(BaseOperator): Creates an InspectTemplate for re-using frequently used configuration for inspecting content, images, and storage. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateInspectTemplateOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -730,6 +734,10 @@ class CloudDLPDeleteInspectTemplateOperator(BaseOperator): """ Deletes an InspectTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteInspectTemplateOperator` + :param template_id: The ID of the inspect template to be deleted. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -1238,6 +1246,10 @@ class CloudDLPInspectContentOperator(BaseOperator): Finds potentially sensitive info in content. This method has limits on input size, processing time, and output size. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPInspectContentOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. diff --git a/docs/howto/operator/gcp/data_loss_prevention.rst b/docs/howto/operator/gcp/data_loss_prevention.rst new file mode 100644 index 0000000000000..728709d17d5f7 --- /dev/null +++ b/docs/howto/operator/gcp/data_loss_prevention.rst @@ -0,0 +1,92 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Google Cloud Data Loss Prevention Operator +========================================== +`Google Cloud DLP `__, provides tools to classify, mask, tokenize, and transform sensitive +elements to help you better manage the data that you collect, store, or use for business or analytics. + +.. contents:: + :depth: 1 + :local: + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: _partials/prerequisite_tasks.rst + +Templates +^^^^^^^^^ + +Templates can be used to create and persist +configuration information to use with the Cloud Data Loss Prevention. +There are two types of templates supported by Cloud DLP: + +* `Inspection Template `__, +* `De-Identification Template `__, + +Here we will be using identification template for our example + +Creating Template +""""""""""""""""" + +To create a inspection template you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateInspectTemplateOperator`. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_create_inspect_template] + :end-before: [END howto_operator_dlp_create_inspect_template] + +.. _howto/operator:CloudDLPCreateInspectTemplateOperator: + +Using Template +"""""""""""""" + +To find potentially sensitive info using the inspection template we just created, we can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPInspectContentOperator` + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_use_inspect_template] + :end-before: [END howto_operator_dlp_use_inspect_template] + +.. _howto/operator:CloudDLPInspectContentOperator: + +Deleting Template +""""""""""""""""" + +To delete the template you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteInspectTemplateOperator`. + +.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_delete_inspect_template] + :end-before: [END howto_operator_dlp_delete_inspect_template] + +.. _howto/operator:CloudDLPDeleteInspectTemplateOperator: + +Reference +^^^^^^^^^ + +For further information, look at: + +* `Client Library Documentation `__ +* `Product Documentation `__ diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 9c791d38de361..3c9c6a7d37a65 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -675,7 +675,7 @@ These integrations allow you to perform various operations within the Google Clo - * - `Cloud Data Loss Prevention (DLP) `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.dlp` - :mod:`airflow.providers.google.cloud.operators.dlp` -