Skip to content

Commit

Permalink
Merge pull request #19372 from DrFaust92/r/glue_catalog_table_target
Browse files Browse the repository at this point in the history
r/glue_catalog_table - add `target_table` argument
YakDriver authored May 18, 2021
2 parents 002e91f + 5ce71fd commit dda7736
Showing 4 changed files with 211 additions and 50 deletions.
3 changes: 3 additions & 0 deletions .changelog/19372.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_glue_catalog_table: Add `target_table` argument
```
78 changes: 78 additions & 0 deletions aws/resource_aws_glue_catalog_table.go
Original file line number Diff line number Diff line change
@@ -283,6 +283,28 @@ func resourceAwsGlueCatalogTable() *schema.Resource {
Type: schema.TypeString,
Optional: true,
},
"target_table": {
Type: schema.TypeList,
Optional: true,
ForceNew: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"catalog_id": {
Type: schema.TypeString,
Required: true,
},
"database_name": {
Type: schema.TypeString,
Required: true,
},
"name": {
Type: schema.TypeString,
Required: true,
},
},
},
},
"view_original_text": {
Type: schema.TypeString,
Optional: true,
@@ -407,6 +429,14 @@ func resourceAwsGlueCatalogTableRead(d *schema.ResourceData, meta interface{}) e
return fmt.Errorf("error setting parameters: %w", err)
}

if table.TargetTable != nil {
if err := d.Set("target_table", []interface{}{flattenGlueTableTargetTable(table.TargetTable)}); err != nil {
return fmt.Errorf("error setting target_table: %w", err)
}
} else {
d.Set("target_table", nil)
}

partIndexInput := &glue.GetPartitionIndexesInput{
CatalogId: out.Table.CatalogId,
TableName: out.Table.Name,
@@ -510,6 +540,10 @@ func expandGlueTableInput(d *schema.ResourceData) *glue.TableInput {
tableInput.Parameters = expandStringMap(v.(map[string]interface{}))
}

if v, ok := d.GetOk("target_table"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
tableInput.TargetTable = expandGlueTableTargetTable(v.([]interface{})[0].(map[string]interface{}))
}

return tableInput
}

@@ -945,3 +979,47 @@ func flattenGlueTableSchemaReferenceSchemaID(s *glue.SchemaId) []map[string]inte

return schemaIDInfoSlice
}

func expandGlueTableTargetTable(tfMap map[string]interface{}) *glue.TableIdentifier {
if tfMap == nil {
return nil
}

apiObject := &glue.TableIdentifier{}

if v, ok := tfMap["catalog_id"].(string); ok && v != "" {
apiObject.CatalogId = aws.String(v)
}

if v, ok := tfMap["database_name"].(string); ok && v != "" {
apiObject.DatabaseName = aws.String(v)
}

if v, ok := tfMap["name"].(string); ok && v != "" {
apiObject.Name = aws.String(v)
}

return apiObject
}

func flattenGlueTableTargetTable(apiObject *glue.TableIdentifier) map[string]interface{} {
if apiObject == nil {
return nil
}

tfMap := map[string]interface{}{}

if v := apiObject.CatalogId; v != nil {
tfMap["catalog_id"] = aws.StringValue(v)
}

if v := apiObject.DatabaseName; v != nil {
tfMap["database_name"] = aws.StringValue(v)
}

if v := apiObject.Name; v != nil {
tfMap["name"] = aws.StringValue(v)
}

return tfMap
}
71 changes: 71 additions & 0 deletions aws/resource_aws_glue_catalog_table_test.go
Original file line number Diff line number Diff line change
@@ -12,6 +12,16 @@ import (
"github.com/terraform-providers/terraform-provider-aws/aws/internal/service/glue/finder"
)

func init() {
RegisterServiceErrorCheckFunc(glue.EndpointsID, testAccErrorCheckSkipGlue)
}

func testAccErrorCheckSkipGlue(t *testing.T) resource.ErrorCheckFunc {
return testAccErrorCheckSkipMessagesContaining(t,
"AccessDeniedException: Operation not allowed",
)
}

func TestAccAWSGlueCatalogTable_basic(t *testing.T) {
rName := acctest.RandomWithPrefix("tf-acc-test")
resourceName := "aws_glue_catalog_table.test"
@@ -31,7 +41,10 @@ func TestAccAWSGlueCatalogTable_basic(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "name", rName),
resource.TestCheckResourceAttr(resourceName, "database_name", rName),
resource.TestCheckResourceAttr(resourceName, "partition_keys.#", "0"),
resource.TestCheckResourceAttr(resourceName, "target_table.#", "0"),
testAccCheckResourceAttrAccountID(resourceName, "catalog_id"),
resource.TestCheckResourceAttr(resourceName, "storage_descriptor.#", "0"),
resource.TestCheckResourceAttr(resourceName, "partition_index.#", "0"),
),
},
{
@@ -580,6 +593,36 @@ func TestAccAWSGlueCatalogTable_disappears_database(t *testing.T) {
})
}

func TestAccAWSGlueCatalogTable_targetTable(t *testing.T) {
rName := acctest.RandomWithPrefix("tf-acc-test")
resourceName := "aws_glue_catalog_table.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
ErrorCheck: testAccErrorCheck(t, glue.EndpointsID),
Providers: testAccProviders,
CheckDestroy: testAccCheckGlueTableDestroy,
Steps: []resource.TestStep{
{
Config: testAccGlueCatalogTableConfigTargetTable(rName),
Destroy: false,
Check: resource.ComposeTestCheckFunc(
testAccCheckGlueCatalogTableExists(resourceName),
resource.TestCheckResourceAttr(resourceName, "target_table.#", "1"),
resource.TestCheckResourceAttrPair(resourceName, "target_table.0.catalog_id", "aws_glue_catalog_table.test2", "catalog_id"),
resource.TestCheckResourceAttrPair(resourceName, "target_table.0.database_name", "aws_glue_catalog_table.test2", "database_name"),
resource.TestCheckResourceAttrPair(resourceName, "target_table.0.name", "aws_glue_catalog_table.test2", "name"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccAWSGlueCatalogTable_disappears(t *testing.T) {
rName := acctest.RandomWithPrefix("tf-acc-test")
resourceName := "aws_glue_catalog_table.test"
@@ -1301,3 +1344,31 @@ resource "aws_glue_catalog_table" "test" {
}
`, rName)
}

func testAccGlueCatalogTableConfigTargetTable(rName string) string {
return fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
name = %[1]q
}
resource "aws_glue_catalog_table" "test" {
name = %[1]q
database_name = aws_glue_catalog_database.test.name
target_table {
catalog_id = aws_glue_catalog_table.test2.catalog_id
database_name = aws_glue_catalog_table.test2.database_name
name = aws_glue_catalog_table.test2.name
}
}
resource "aws_glue_catalog_database" "test2" {
name = "%[1]s-2"
}
resource "aws_glue_catalog_table" "test2" {
name = "%[1]s-2"
database_name = aws_glue_catalog_database.test2.name
}
`, rName)
}
109 changes: 59 additions & 50 deletions website/docs/r/glue_catalog_table.html.markdown
Original file line number Diff line number Diff line change
@@ -82,92 +82,101 @@ resource "aws_glue_catalog_table" "aws_glue_catalog_table" {

## Argument Reference

The following arguments are supported:
The following arguments are required:

* `name` - (Required) Name of the table. For Hive compatibility, this must be entirely lowercase.
* `database_name` - (Required) Name of the metadata database where the table metadata resides. For Hive compatibility, this must be all lowercase.

The follow arguments are optional:

* `catalog_id` - (Optional) ID of the Glue Catalog and database to create the table in. If omitted, this defaults to the AWS Account ID plus the database name.
* `description` - (Optional) Description of the table.
* `owner` - (Optional) Owner of the table.
* `parameters` - (Optional) Properties associated with this table, as a list of key-value pairs.
* `partition_index` - (Optional) Configuration block for a maximum of 3 partition indexes. See [`partition_index`](#partition_index) below.
* `partition_keys` - (Optional) Configuration block of columns by which the table is partitioned. Only primitive types are supported as partition keys. See [`partition_keys`](#partition_keys) below.
* `retention` - (Optional) Retention time for this table.
* `storage_descriptor` - (Optional) A [storage descriptor](#storage-descriptor) object containing information about the physical storage of this table. You can refer to the [Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-tables.html#aws-glue-api-catalog-tables-StorageDescriptor) for a full explanation of this object.
* `partition_keys` - (Optional) A list of columns by which the table is partitioned. Only primitive types are supported as partition keys. see [Partition Keys](#partition-keys) below.
* `view_original_text` - (Optional) If the table is a view, the original text of the view; otherwise null.
* `storage_descriptor` - (Optional) Configuration block for information about the physical storage of this table. For more information, refer to the [Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-tables. html#aws-glue-api-catalog-tables-StorageDescriptor). See [`storage_descriptor`](#storage_descriptor) below.
* `table_type` - (Optional) Type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc.). While optional, some Athena DDL queries such as `ALTER TABLE` and `SHOW CREATE TABLE` will fail if this argument is empty.
* `target_table` - (Optional) Configuration block of a target table for resource linking. See [`target_table`](#target_table) below.
* `view_expanded_text` - (Optional) If the table is a view, the expanded text of the view; otherwise null.
* `table_type` - (Optional) The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc.). While optional, some Athena DDL queries such as `ALTER TABLE` and `SHOW CREATE TABLE` will fail if this argument is empty.
* `parameters` - (Optional) Properties associated with this table, as a list of key-value pairs.
* `partition_index` - (Optional) A list of partition indexes. see [Partition Index](#partition-index) below.
* `view_original_text` - (Optional) If the table is a view, the original text of the view; otherwise null.

### Partition Index
### partition_index

* `index_name` - (Required) The name of the partition index.
* `keys` - (Required) The keys for the partition index.
* `index_name` - (Required) Name of the partition index.
* `keys` - (Required) Keys for the partition index.

### Partition Keys
### partition_keys

* `name` - (Required) The name of the Partition Key.
* `type` - (Optional) The datatype of data in the Partition Key.
* `comment` - (Optional) Free-form text comment.
* `name` - (Required) Name of the Partition Key.
* `type` - (Optional) Datatype of data in the Partition Key.

### Storage Descriptor
### storage_descriptor

* `columns` - (Optional) A list of the [Columns](#column) in the table.
* `location` - (Optional) The physical location of the table. By default this takes the form of the warehouse location, followed by the database location in the warehouse, followed by the table name.
* `input_format` - (Optional) The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format.
* `output_format` - (Optional) The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format.
* `compressed` - (Optional) True if the data in the table is compressed, or False if not.
* `bucket_columns` - (Optional) List of reducer grouping columns, clustering columns, and bucketing columns in the table.
* `columns` - (Optional) Configuration block for columns in the table. See [`columns`](#columns) below.
* `compressed` - (Optional) Whether the data in the table is compressed.
* `input_format` - (Optional) Input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format.
* `location` - (Optional) Physical location of the table. By default this takes the form of the warehouse location, followed by the database location in the warehouse, followed by the table name.
* `number_of_buckets` - (Optional) Must be specified if the table contains any dimension columns.
* `ser_de_info` - (Optional) [Serialization/deserialization (SerDe)](#ser-de-info) information.
* `bucket_columns` - (Optional) A list of reducer grouping columns, clustering columns, and bucketing columns in the table.
* `sort_columns` - (Optional) A list of [Order](#sort-column) objects specifying the sort order of each bucket in the table.
* `output_format` - (Optional) Output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format.
* `parameters` - (Optional) User-supplied properties in key-value form.
* `skewed_info` - (Optional) Information about values that appear very frequently in a column (skewed values).
* `stored_as_sub_directories` - (Optional) True if the table data is stored in subdirectories, or False if not.
* `schema_reference` - (Optional) An object that references a schema stored in the AWS Glue Schema Registry. When creating a table, you can pass an empty list of columns for the schema, and instead use a schema reference. See [Schema Reference](#schema-reference) below.
* `schema_reference` - (Optional) Object that references a schema stored in the AWS Glue Schema Registry. When creating a table, you can pass an empty list of columns for the schema, and instead use a schema reference. See [Schema Reference](#schema-reference) below.
* `ser_de_info` - (Optional) Configuration block for serialization and deserialization ("SerDe") information. See [`ser_de_info`](#ser_de_info) below.
* `skewed_info` - (Optional) Configuration block with information about values that appear very frequently in a column (skewed values). See [`skewed_info`](#skewed_info) below.
* `sort_columns` - (Optional) Configuration block for the sort order of each bucket in the table. See [`sort_columns`](#sort_columns) below.
* `stored_as_sub_directories` - (Optional) Whether the table data is stored in subdirectories.

##### Column
#### columns

* `name` - (Required) The name of the Column.
* `type` - (Optional) The datatype of data in the Column.
* `comment` - (Optional) Free-form text comment.
* `parameters` - (Optional) These key-value pairs define properties associated with the column.
* `name` - (Required) Name of the Column.
* `parameters` - (Optional) Key-value pairs defining properties associated with the column.
* `type` - (Optional) Datatype of data in the Column.

##### Ser De Info
#### schema_reference

* `name` - (Optional) Name of the SerDe.
* `parameters` - (Optional) A map of initialization parameters for the SerDe, in key-value form.
* `serialization_library` - (Optional) Usually the class that implements the SerDe. An example is: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.
* `schema_id` - (Optional) Configuration block that contains schema identity fields. Either this or the `schema_version_id` has to be provided. See [`schema_id`](#schema_id) below.
* `schema_version_id` - (Optional) Unique ID assigned to a version of the schema. Either this or the `schema_id` has to be provided.
* `schema_version_number` - (Required) Version number of the schema.

##### Sort Columns
##### schema_id

* `column` - (Required) The name of the column.
* `sort_order` - (Required) Indicates that the column is sorted in ascending order (== 1), or in descending order (==0).
* `registry_name` - (Optional) Name of the schema registry that contains the schema. Must be provided when `schema_name` is specified and conflicts with `schema_arn`.
* `schema_arn` - (Optional) ARN of the schema. One of `schema_arn` or `schema_name` has to be provided.
* `schema_name` - (Optional) Name of the schema. One of `schema_arn` or `schema_name` has to be provided.

##### Skewed Info
#### ser_de_info

* `skewed_column_names` - (Optional) A list of names of columns that contain skewed values.
* `skewed_column_value_location_maps` - (Optional) A list of values that appear so frequently as to be considered skewed.
* `skewed_column_values` - (Optional) A map of skewed values to the columns that contain them.
* `name` - (Optional) Name of the SerDe.
* `parameters` - (Optional) Map of initialization parameters for the SerDe, in key-value form.
* `serialization_library` - (Optional) Usually the class that implements the SerDe. An example is `org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe`.

#### sort_columns

* `column` - (Required) Name of the column.
* `sort_order` - (Required) Whether the column is sorted in ascending (`1`) or descending order (`0`).

##### Schema Reference
#### skewed_info

* `schema_id` - (Optional) A structure that contains schema identity fields. Either this or the `schema_version_id` has to be provided. See [Schema ID](#schema-id) below.
* `schema_version_id` - (Optional) The unique ID assigned to a version of the schema. Either this or the `schema_id` has to be provided.
* `schema_version_number` - (Required) The version number of the schema.
* `skewed_column_names` - (Optional) List of names of columns that contain skewed values.
* `skewed_column_value_location_maps` - (Optional) List of values that appear so frequently as to be considered skewed.
* `skewed_column_values` - (Optional) Map of skewed values to the columns that contain them.

###### Schema ID
### target_table

* `schema_arn` - (Optional) The Amazon Resource Name (ARN) of the schema. One of `schema_arn` or `schema_name` has to be provided.
* `schema_name` - (Optional) The name of the schema. One of `schema_arn` or `schema_name` has to be provided.
* `registry_name` - (Optional) The name of the schema registry that contains the schema. Must be provided when `schema_name` is specified and conflicts with `schema_arn`.
* `catalog_id` - (Required) ID of the Data Catalog in which the table resides.
* `database_name` - (Required) Name of the catalog database that contains the target table.
* `name` - (Required) Name of the target table.

## Attributes Reference

In addition to all arguments above, the following attributes are exported:

* `id` - Catalog ID, Database name and of the name table.
* `arn` - The ARN of the Glue Table.

* `id` - Catalog ID, Database name and of the name table.

## Import

0 comments on commit dda7736

Please sign in to comment.