diff --git a/airflow/providers/google/cloud/example_dags/example_dlp.py b/airflow/providers/google/cloud/example_dags/example_dlp.py index a8dce29290050..1dffaf788a94f 100644 --- a/airflow/providers/google/cloud/example_dags/example_dlp.py +++ b/airflow/providers/google/cloud/example_dags/example_dlp.py @@ -31,8 +31,14 @@ from airflow import models from airflow.providers.google.cloud.operators.dlp import ( CloudDLPCreateInspectTemplateOperator, + CloudDLPCreateJobTriggerOperator, + CloudDLPCreateStoredInfoTypeOperator, CloudDLPDeleteInspectTemplateOperator, + CloudDLPDeleteJobTriggerOperator, + CloudDLPDeleteStoredInfoTypeOperator, CloudDLPInspectContentOperator, + CloudDLPUpdateJobTriggerOperator, + CloudDLPUpdateStoredInfoTypeOperator, ) from airflow.utils.dates import days_ago @@ -54,6 +60,7 @@ start_date=days_ago(1), tags=['example'], ) as dag: + # [START howto_operator_dlp_create_inspect_template] create_template = CloudDLPCreateInspectTemplateOperator( project_id=GCP_PROJECT, inspect_template=INSPECT_TEMPLATE, @@ -62,7 +69,9 @@ do_xcom_push=True, dag=dag, ) + # [END howto_operator_dlp_create_inspect_template] + # [START howto_operator_dlp_use_inspect_template] inspect_content = CloudDLPInspectContentOperator( task_id="inpsect_content", project_id=GCP_PROJECT, @@ -70,9 +79,94 @@ inspect_template_name="{{ task_instance.xcom_pull('create_template', key='return_value')['name'] }}", dag=dag, ) + # [END howto_operator_dlp_use_inspect_template] + # [START howto_operator_dlp_delete_inspect_template] delete_template = CloudDLPDeleteInspectTemplateOperator( task_id="delete_template", template_id=TEMPLATE_ID, project_id=GCP_PROJECT, dag=dag, ) + # [END howto_operator_dlp_delete_inspect_template] create_template >> inspect_content >> delete_template + + +CUSTOM_INFO_TYPES = [{"info_type": {"name": "C_MRN"}, "regex": {"pattern": "[1-9]{3}-[1-9]{1}-[1-9]{5}"},}] +CUSTOM_INFO_TYPE_ID = "custom_info_type" +UPDATE_CUSTOM_INFO_TYPE = [ + {"info_type": {"name": "C_MRN"}, "regex": {"pattern": "[a-z]{3}-[a-z]{1}-[a-z]{5}"},} +] + +with models.DAG( + "example_gcp_dlp_info_types", + schedule_interval=None, + start_date=days_ago(1), + tags=["example", "dlp", "info-types"], +) as dag: + # [START howto_operator_dlp_create_info_type] + create_info_type = CloudDLPCreateStoredInfoTypeOperator( + project_id=GCP_PROJECT, + config=CUSTOM_INFO_TYPES, + stored_info_type_id=CUSTOM_INFO_TYPE_ID, + dag=dag, + task_id="create_info_type", + ) + # [END howto_operator_dlp_create_info_type] + # [START howto_operator_dlp_update_info_type] + update_info_type = CloudDLPUpdateStoredInfoTypeOperator( + project_id=GCP_PROJECT, + stored_info_type_id=CUSTOM_INFO_TYPE_ID, + config=UPDATE_CUSTOM_INFO_TYPE, + dag=dag, + task_id="update_info_type", + ) + # [END howto_operator_dlp_update_info_type] + # [START howto_operator_dlp_delete_info_type] + delete_info_type = CloudDLPDeleteStoredInfoTypeOperator( + project_id=GCP_PROJECT, stored_info_type_id=CUSTOM_INFO_TYPE_ID, dag=dag, task_id="delete_info_type", + ) + # [END howto_operator_dlp_delete_info_type] + create_info_type >> update_info_type >> delete_info_type + +SCHEDULE = {"recurrence_period_duration": {"seconds": 60 * 60 * 24}} +JOB = { + "inspect_config": INSPECT_CONFIG, +} + +JOB_TRIGGER = { + "inspect_job": JOB, + "triggers": [{"schedule": SCHEDULE}], + "status": "HEALTHY", +} + +TRIGGER_ID = "example_trigger" + +with models.DAG( + "example_gcp_dlp_job", schedule_interval=None, start_date=days_ago(1), tags=["example", "dlp_job"], +) as dag: # [START howto_operator_dlp_create_job_trigger] + create_trigger = CloudDLPCreateJobTriggerOperator( + project_id=GCP_PROJECT, + job_trigger=JOB_TRIGGER, + trigger_id=TRIGGER_ID, + dag=dag, + task_id="create_trigger", + ) + # [END howto_operator_dlp_create_job_trigger] + UPDATED_SCHEDULE = {"recurrence_period_duration": {"seconds": 2 * 60 * 60 * 24}} + + JOB_TRIGGER["triggers"] = [{"schedule": UPDATED_SCHEDULE}] + + # [START howto_operator_dlp_update_job_trigger] + update_trigger = CloudDLPUpdateJobTriggerOperator( + project_id=GCP_PROJECT, + job_trigger_id=TRIGGER_ID, + job_trigger=JOB_TRIGGER, + dag=dag, + task_id="update_info_type", + ) + # [END howto_operator_dlp_update_job_trigger] + # [START howto_operator_dlp_delete_job_trigger] + delete_trigger = CloudDLPDeleteJobTriggerOperator( + project_id=GCP_PROJECT, job_trigger_id=TRIGGER_ID, dag=dag, task_id="delete_info_type", + ) + # [END howto_operator_dlp_delete_job_trigger] + create_trigger >> update_trigger >> delete_trigger diff --git a/airflow/providers/google/cloud/operators/dlp.py b/airflow/providers/google/cloud/operators/dlp.py index 7a5aef4e0a71a..384354a80368e 100644 --- a/airflow/providers/google/cloud/operators/dlp.py +++ b/airflow/providers/google/cloud/operators/dlp.py @@ -51,6 +51,10 @@ class CloudDLPCancelDLPJobOperator(BaseOperator): """ Starts asynchronous cancellation on a long-running DlpJob. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCancelDLPJobOperator` + :param dlp_job_id: ID of the DLP job resource to be cancelled. :type dlp_job_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -124,6 +128,10 @@ class CloudDLPCreateDeidentifyTemplateOperator(BaseOperator): Creates a DeidentifyTemplate for re-using frequently used configuration for de-identifying content, images, and storage. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateDeidentifyTemplateOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -223,6 +231,10 @@ class CloudDLPCreateDLPJobOperator(BaseOperator): """ Creates a new job to inspect storage or calculate risk metrics. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateDLPJobOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -326,6 +338,10 @@ class CloudDLPCreateInspectTemplateOperator(BaseOperator): Creates an InspectTemplate for re-using frequently used configuration for inspecting content, images, and storage. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateInspectTemplateOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -425,6 +441,10 @@ class CloudDLPCreateJobTriggerOperator(BaseOperator): Creates a job trigger to run DLP actions such as scanning storage for sensitive information on a set schedule. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateJobTriggerOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -515,6 +535,10 @@ class CloudDLPCreateStoredInfoTypeOperator(BaseOperator): """ Creates a pre-built stored infoType to be used for inspection. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPCreateStoredInfoTypeOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -614,6 +638,10 @@ class CloudDLPDeidentifyContentOperator(BaseOperator): De-identifies potentially sensitive info from a ContentItem. This method has limits on input size and output size. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeidentifyContentOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -719,6 +747,10 @@ class CloudDLPDeleteDeidentifyTemplateOperator(BaseOperator): """ Deletes a DeidentifyTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteDeidentifyTemplateOperator` + :param template_id: The ID of deidentify template to be deleted. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -802,6 +834,10 @@ class CloudDLPDeleteDLPJobOperator(BaseOperator): Deletes a long-running DlpJob. This method indicates that the client is no longer interested in the DlpJob result. The job will be cancelled if possible. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteDLPJobOperator` + :param dlp_job_id: The ID of the DLP job resource to be cancelled. :type dlp_job_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -877,6 +913,10 @@ class CloudDLPDeleteInspectTemplateOperator(BaseOperator): """ Deletes an InspectTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteInspectTemplateOperator` + :param template_id: The ID of the inspect template to be deleted. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -959,6 +999,10 @@ class CloudDLPDeleteJobTriggerOperator(BaseOperator): """ Deletes a job trigger. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteJobTriggerOperator` + :param job_trigger_id: The ID of the DLP job trigger to be deleted. :type job_trigger_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -1034,6 +1078,10 @@ class CloudDLPDeleteStoredInfoTypeOperator(BaseOperator): """ Deletes a stored infoType. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPDeleteStoredInfoTypeOperator` + :param stored_info_type_id: The ID of the stored info type to be deleted. :type stored_info_type_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -1116,6 +1164,10 @@ class CloudDLPGetDeidentifyTemplateOperator(BaseOperator): """ Gets a DeidentifyTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPGetDeidentifyTemplateOperator` + :param template_id: The ID of deidentify template to be read. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -1198,6 +1250,10 @@ class CloudDLPGetDLPJobOperator(BaseOperator): """ Gets the latest state of a long-running DlpJob. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPGetDLPJobOperator` + :param dlp_job_id: The ID of the DLP job resource to be read. :type dlp_job_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -1273,6 +1329,10 @@ class CloudDLPGetInspectTemplateOperator(BaseOperator): """ Gets an InspectTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPGetInspectTemplateOperator` + :param template_id: The ID of inspect template to be read. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -1355,6 +1415,10 @@ class CloudDLPGetDLPJobTriggerOperator(BaseOperator): """ Gets a job trigger. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPGetDLPJobTriggerOperator` + :param job_trigger_id: The ID of the DLP job trigger to be read. :type job_trigger_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -1430,6 +1494,10 @@ class CloudDLPGetStoredInfoTypeOperator(BaseOperator): """ Gets a stored infoType. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPGetStoredInfoTypeOperator` + :param stored_info_type_id: The ID of the stored info type to be read. :type stored_info_type_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -1513,6 +1581,10 @@ class CloudDLPInspectContentOperator(BaseOperator): Finds potentially sensitive info in content. This method has limits on input size, processing time, and output size. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPInspectContentOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -1602,6 +1674,10 @@ class CloudDLPListDeidentifyTemplatesOperator(BaseOperator): """ Lists DeidentifyTemplates. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListDeidentifyTemplatesOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -1690,6 +1766,10 @@ class CloudDLPListDLPJobsOperator(BaseOperator): """ Lists DlpJobs that match the specified filter in the request. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListDLPJobsOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -1781,6 +1861,10 @@ class CloudDLPListInfoTypesOperator(BaseOperator): """ Returns a list of the sensitive information types that the DLP API supports. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListInfoTypesOperator` + :param language_code: (Optional) Optional BCP-47 language code for localized infoType friendly names. If omitted, or if localized strings are not available, en-US strings will be returned. @@ -1855,6 +1939,10 @@ class CloudDLPListInspectTemplatesOperator(BaseOperator): """ Lists InspectTemplates. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListInspectTemplatesOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -1943,6 +2031,10 @@ class CloudDLPListJobTriggersOperator(BaseOperator): """ Lists job triggers. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListJobTriggersOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -2029,6 +2121,10 @@ class CloudDLPListStoredInfoTypesOperator(BaseOperator): """ Lists stored infoTypes. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPListStoredInfoTypesOperator` + :param organization_id: (Optional) The organization ID. Required to set this field if parent resource is an organization. :type organization_id: str @@ -2118,6 +2214,10 @@ class CloudDLPRedactImageOperator(BaseOperator): Redacts potentially sensitive info from an image. This method has limits on input size, processing time, and output size. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPRedactImageOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -2215,6 +2315,10 @@ class CloudDLPReidentifyContentOperator(BaseOperator): """ Re-identifies content that has been de-identified. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPReidentifyContentOperator` + :param project_id: (Optional) Google Cloud Platform project ID where the DLP Instance exists. If set to None or missing, the default project_id from the GCP connection is used. @@ -2318,6 +2422,10 @@ class CloudDLPUpdateDeidentifyTemplateOperator(BaseOperator): """ Updates the DeidentifyTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPUpdateDeidentifyTemplateOperator` + :param template_id: The ID of deidentify template to be updated. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -2412,6 +2520,10 @@ class CloudDLPUpdateInspectTemplateOperator(BaseOperator): """ Updates the InspectTemplate. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPUpdateInspectTemplateOperator` + :param template_id: The ID of the inspect template to be updated. :type template_id: str :param organization_id: (Optional) The organization ID. Required to set this @@ -2506,6 +2618,10 @@ class CloudDLPUpdateJobTriggerOperator(BaseOperator): """ Updates a job trigger. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPUpdateJobTriggerOperator` + :param job_trigger_id: The ID of the DLP job trigger to be updated. :type job_trigger_id: str :param project_id: (Optional) Google Cloud Platform project ID where the @@ -2593,6 +2709,10 @@ class CloudDLPUpdateStoredInfoTypeOperator(BaseOperator): """ Updates the stored infoType by creating a new version. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CloudDLPUpdateStoredInfoTypeOperator` + :param stored_info_type_id: The ID of the stored info type to be updated. :type stored_info_type_id: str :param organization_id: (Optional) The organization ID. Required to set this diff --git a/docs/howto/operator/google/cloud/data_loss_prevention.rst b/docs/howto/operator/google/cloud/data_loss_prevention.rst new file mode 100644 index 0000000000000..5c40f1a622fdd --- /dev/null +++ b/docs/howto/operator/google/cloud/data_loss_prevention.rst @@ -0,0 +1,327 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Google Cloud Data Loss Prevention Operator +========================================== +`Google Cloud DLP `__, provides tools to classify, mask, tokenize, and transform sensitive +elements to help you better manage the data that you collect, store, or use for business or analytics. + +.. contents:: + :depth: 1 + :local: + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: /howto/operator/google/_partials/prerequisite_tasks.rst + +Info-Types +^^^^^^^^^^ +Google Cloud DLP uses info-types to define what scans for. + +.. _howto/operator:CloudDLPCreateStoredInfoTypeOperator: + +Create Stored Info-Type +""""""""""""""""""""""" + +To create a custom info-type you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateStoredInfoTypeOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_create_info_type] + :end-before: [END howto_operator_dlp_create_info_type] + + + +.. _howto/operator:CloudDLPGetStoredInfoTypeOperator: +.. _howto/operator:CloudDLPListInfoTypesOperator: +.. _howto/operator:CloudDLPListStoredInfoTypesOperator: + +Retrieve Stored Info-Type +""""""""""""""""""""""""" + +To retrieve the lists of sensitive info-types supported by DLP-API for reference, you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListInfoTypesOperator`. + +Similarly to retrieve the list custom info-types, you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListStoredInfoTypesOperator`. + +To retrieve a single info-type +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPGetStoredInfoTypeOperator` + + +.. _howto/operator:CloudDLPUpdateStoredInfoTypeOperator: + +Update Stored Info-Type +""""""""""""""""""""""" + +To update a info-type you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPUpdateStoredInfoTypeOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_update_info_type] + :end-before: [END howto_operator_dlp_update_info_type] + + +.. _howto/operator:CloudDLPDeleteStoredInfoTypeOperator: + +Deleting Stored Info-Type +""""""""""""""""""""""""" + +To delete a info-type you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteStoredInfoTypeOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_delete_info_type] + :end-before: [END howto_operator_dlp_delete_info_type] + + +Templates +^^^^^^^^^ + +Templates can be used to create and persist +configuration information to use with the Cloud Data Loss Prevention. +There are two types of DLP templates supported by Airflow: + +* Inspection Template +* De-Identification Template + +Here we will be using identification template for our example + +.. _howto/operator:CloudDLPCreateInspectTemplateOperator: + +Creating Template +""""""""""""""""" + +To create a inspection template you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateInspectTemplateOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_create_inspect_template] + :end-before: [END howto_operator_dlp_create_inspect_template] + + + +.. _howto/operator:CloudDLPGetInspectTemplateOperator: +.. _howto/operator:CloudDLPListInspectTemplatesOperator: + +Retrieving Template +""""""""""""""""""" + +If you already have an existing inspect template you can retrieve it by use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPGetInspectTemplateOperator` +List of existing inspect templates can be retrieved by +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListInspectTemplatesOperator` + +.. _howto/operator:CloudDLPInspectContentOperator: + +Using Template +"""""""""""""" + +To find potentially sensitive info using the inspection template we just created, we can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPInspectContentOperator` + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_use_inspect_template] + :end-before: [END howto_operator_dlp_use_inspect_template] + +.. _howto/operator:CloudDLPUpdateInspectTemplateOperator: + +Updating Template +""""""""""""""""" + +To update the template you can use +:class:`~airflow.providers.google.cloud.operators.cloud.CloudDLPUpdateInspectTemplateOperator`. + +.. _howto/operator:CloudDLPDeleteInspectTemplateOperator: + +Deleting Template +""""""""""""""""" + +To delete the template you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteInspectTemplateOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_delete_inspect_template] + :end-before: [END howto_operator_dlp_delete_inspect_template] + +.. _howto/operator:CloudDLPCreateDeidentifyTemplateOperator: +.. _howto/operator:CloudDLPDeleteDeidentifyTemplateOperator: +.. _howto/operator:CloudDLPUpdateDeidentifyTemplateOperator: +.. _howto/operator:CloudDLPGetDeidentifyTemplateOperator: +.. _howto/operator:CloudDLPListDeidentifyTemplatesOperator: + +De-Identification Template +"""""""""""""""""""""""""" + +Like Inspect templates, De-Identification templates also have CRUD operators + +* :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateDeidentifyTemplateOperator` +* :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteDeidentifyTemplateOperator` +* :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPUpdateDeidentifyTemplateOperator` +* :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPGetDeidentifyTemplateOperator` +* :class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListDeidentifyTemplatesOperator` + + +Jobs & Job Triggers +^^^^^^^^^^^^^^^^^^^ + +Cloud Data Loss Protection uses a job to run actions to scan content for sensitive data or +calculate the risk of re-identification. You can schedule these jobs using job triggers. + +.. _howto/operator:CloudDLPCreateDLPJobOperator: + +Creating Job +"""""""""""" + +To create a job you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateDLPJobOperator`. + + +.. _howto/operator:CloudDLPListDLPJobsOperator: +.. _howto/operator:CloudDLPGetDLPJobOperator: + +Retrieving Job +"""""""""""""" + +To retrieve the list of jobs you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListDLPJobsOperator`. +To retrieve a single job +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPGetDLPJobOperator`. + +.. _howto/operator:CloudDLPDeleteDLPJobOperator: + +Deleting Job +"""""""""""" + +To delete a job you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteDLPJobOperator`. + +.. _howto/operator:CloudDLPCancelDLPJobOperator: + +Canceling a Job +"""""""""""""""" + +To start asynchronous cancellation of a long-running DLP job you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCancelDLPJobOperator`. + + +.. _howto/operator:CloudDLPCreateJobTriggerOperator: + +Creating Job Trigger +"""""""""""""""""""" + +To create a job trigger you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPCreateJobTriggerOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_create_job_trigger] + :end-before: [END howto_operator_dlp_create_job_trigger] + +.. _howto/operator:CloudDLPListJobTriggersOperator: +.. _howto/operator:CloudDLPGetDLPJobTriggerOperator: + +Retrieving Job Trigger +"""""""""""""""""""""" + +To retrieve list of job triggers you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPListJobTriggersOperator`. +To retrieve a single job trigger you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPGetDLPJobTriggerOperator`. + +.. _howto/operator:CloudDLPUpdateJobTriggerOperator: + +Updating Job Trigger +"""""""""""""""""""" + +To update a job trigger you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPUpdateJobTriggerOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_update_job_trigger] + :end-before: [END howto_operator_dlp_update_job_trigger] + +.. _howto/operator:CloudDLPDUpdateJobTriggerOperator: + +Deleting Job Trigger +""""""""""""""""""""" + +To delete a job trigger you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeleteJobTriggerOperator`. + +.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_dlp.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dlp_delete_job_trigger] + :end-before: [END howto_operator_dlp_delete_job_trigger] + +.. _howto/operator:CloudDLPDeleteJobTriggerOperator: + +Content Method +^^^^^^^^^^^^^^ + +Unlike storage methods (Jobs) content method are synchronous, stateless methods. + +.. _howto/operator:CloudDLPDeidentifyContentOperator: + +De-identify Content +""""""""""""""""""" + +To de-identify potentially sensitive info from a content item, you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPDeidentifyContentOperator`. + +.. _howto/operator:CloudDLPReidentifyContentOperator: + +Re-identify Content +""""""""""""""""""" + +To re-identify the content that has been de-identified you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPReidentifyContentOperator`. + +.. _howto/operator:CloudDLPReIdentifyOperator: + +Redact Image +"""""""""""" + +To redact potentially sensitive information from the content image you can use +:class:`~airflow.providers.google.cloud.operators.cloud.dlp.CloudDLPRedactImageOperator`. + +.. _howto/operator:CloudDLPRedactImageOperator: + +Reference +^^^^^^^^^ + +For further information, look at: + +* `Client Library Documentation `__ +* `Product Documentation `__ diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index d1c600e2e222c..66571e6d3a52f 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -731,7 +731,7 @@ These integrations allow you to perform various operations within the Google Clo - * - `Cloud Data Loss Prevention (DLP) `__ - - + - :doc:`How to use ` - :mod:`airflow.providers.google.cloud.hooks.dlp` - :mod:`airflow.providers.google.cloud.operators.dlp` -