Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 147/ Programmatic Documentation Notes #187

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
187 changes: 118 additions & 69 deletions databuilder/models/table_metadata.py
Original file line number Diff line number Diff line change
@@ -8,8 +8,6 @@
RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
from databuilder.publisher.neo4j_csv_publisher import UNQUOTED_SUFFIX

DESCRIPTION_NODE_LABEL = 'Description'


class TagMetadata(Neo4jCsvSerializable):
TAG_NODE_LABEL = 'Tag'
@@ -59,18 +57,88 @@ def create_next_relation(self):
return None


class DescriptionMetadata:
DESCRIPTION_NODE_LABEL = 'Description'
PROGRAMMATIC_DESCRIPTION_NODE_LABEL = 'Programmatic_Description'
DESCRIPTION_KEY_FORMAT = '{description}'
DESCRIPTION_TEXT = 'description'
DESCRIPTION_SOURCE = 'description_source'

DESCRIPTION_RELATION_TYPE = 'DESCRIPTION'
INVERSE_DESCRIPTION_RELATION_TYPE = 'DESCRIPTION_OF'

# The default editable source.
DEFAULT_SOURCE = "description"

def __init__(self,
text, # type: Union[None, str]
source=DEFAULT_SOURCE # type: str
):
"""
:param source: The unique source of what is populating this description.
:param text: the description text. Markdown supported.
"""
self._source = source
self._text = text
# There are so many dependencies on Description node, that it is probably easier to just separate the rest out.
if (self._source == self.DEFAULT_SOURCE):
self._label = self.DESCRIPTION_NODE_LABEL
else:
self._label = self.PROGRAMMATIC_DESCRIPTION_NODE_LABEL

@staticmethod
def create_description_metadata(text, source=DEFAULT_SOURCE):
# type: (Union[None,str], str) -> ProgrammaticDescription

# We do not want to create a node if there is no description text!
if text is None:
return None
if not source:
description_node = DescriptionMetadata(text=text, source=DescriptionMetadata.DEFAULT_SOURCE)
else:
description_node = DescriptionMetadata(text=text, source=source)
return description_node

def get_description_id(self):
# type: () -> str
if self._source == self.DEFAULT_SOURCE:
return "_description"
else:
return "_" + self._source + "_description"

def __repr__(self):
# type: () -> str
return 'DescriptionMetadata({!r}, {!r})'.format(self._source, self._text)

def get_node_dict(self, node_key):
# (str) -> Dict
return {
NODE_LABEL: self._label,
NODE_KEY: node_key,
DescriptionMetadata.DESCRIPTION_SOURCE: self._source,
DescriptionMetadata.DESCRIPTION_TEXT: self._text,
}

def get_relation(self, start_node, start_key, end_key):
# (str, str) => Dict
return {
RELATION_START_LABEL: start_node,
RELATION_END_LABEL: self._label,
RELATION_START_KEY: start_key,
RELATION_END_KEY: end_key,
RELATION_TYPE: DescriptionMetadata.DESCRIPTION_RELATION_TYPE,
RELATION_REVERSE_TYPE: DescriptionMetadata.INVERSE_DESCRIPTION_RELATION_TYPE
}


class ColumnMetadata:
COLUMN_NODE_LABEL = 'Column'
COLUMN_KEY_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{col}'
COLUMN_NAME = 'name'
COLUMN_TYPE = 'type'
COLUMN_ORDER = 'sort_order{}'.format(UNQUOTED_SUFFIX) # int value needs to be unquoted when publish to neo4j
COLUMN_DESCRIPTION = 'description'
COLUMN_DESCRIPTION_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{col}/_description'

# pair of nodes makes relationship where name of variable represents order of relationship.
COL_DESCRIPTION_RELATION_TYPE = 'DESCRIPTION'
DESCRIPTION_COL_RELATION_TYPE = 'DESCRIPTION_OF'
COLUMN_DESCRIPTION_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{col}/{description_id}'

# Relation between column and tag
COL_TAG_RELATION_TYPE = 'TAGGED_BY'
@@ -81,7 +149,7 @@ def __init__(self,
description, # type: Union[str, None]
col_type, # type: str
sort_order, # type: int
tags=None, # Union[List[str], None]
tags=None # type: Union[List[str], None]
):
# type: (...) -> None
"""
@@ -92,7 +160,8 @@ def __init__(self,
:param sort_order:
"""
self.name = name
self.description = description
self.description = DescriptionMetadata.create_description_metadata(source=None,
text=description)
self.type = col_type
self.sort_order = sort_order
self.tags = tags
@@ -126,10 +195,7 @@ class TableMetadata(Neo4jCsvSerializable):
TABLE_NAME = 'name'
IS_VIEW = 'is_view{}'.format(UNQUOTED_SUFFIX) # bool value needs to be unquoted when publish to neo4j

TABLE_DESCRIPTION = 'description'
TABLE_DESCRIPTION_FORMAT = '{db}://{cluster}.{schema}/{tbl}/_description'
TABLE_DESCRIPTION_RELATION_TYPE = 'DESCRIPTION'
DESCRIPTION_TABLE_RELATION_TYPE = 'DESCRIPTION_OF'
TABLE_DESCRIPTION_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{description_id}'

DATABASE_NODE_LABEL = 'Database'
DATABASE_KEY_FORMAT = 'database://{db}'
@@ -165,6 +231,7 @@ def __init__(self,
columns=None, # type: Iterable[ColumnMetadata]
is_view=False, # type: bool
tags=None, # type: Union[List, str]
description_source=None, # type: Union[str, None]
**kwargs # type: Dict
):
# type: (...) -> None
@@ -177,13 +244,14 @@ def __init__(self,
:param columns:
:param is_view: Indicate whether the table is a view or not
:param tags:
:param description_source: Optional. Where the description is coming from. Used to compose unique id.
:param kwargs: Put additional attributes to the table model if there is any.
"""
self.database = database
self.cluster = cluster
self.schema_name = schema_name
self.name = name
self.description = description
self.description = DescriptionMetadata.create_description_metadata(text=description, source=description_source)
self.columns = columns if columns else []
self.is_view = is_view
self.attrs = None
@@ -218,12 +286,13 @@ def _get_table_key(self):
schema=self.schema_name,
tbl=self.name)

def _get_table_description_key(self):
# type: () -> str
def _get_table_description_key(self, description):
# type: (DescriptionMetadata) -> str
return TableMetadata.TABLE_DESCRIPTION_FORMAT.format(db=self.database,
cluster=self.cluster,
schema=self.schema_name,
tbl=self.name)
tbl=self.name,
description_id=description.get_description_id())

def _get_database_key(self):
# type: () -> str
@@ -248,13 +317,14 @@ def _get_col_key(self, col):
tbl=self.name,
col=col.name)

def _get_col_description_key(self, col):
# type: (ColumnMetadata) -> str
def _get_col_description_key(self, col, description):
# type: (ColumnMetadata, DescriptionMetadata) -> str
return ColumnMetadata.COLUMN_DESCRIPTION_FORMAT.format(db=self.database,
cluster=self.cluster,
schema=self.schema_name,
tbl=self.name,
col=col.name)
col=col.name,
description_id=description.get_description_id())

def create_next_node(self):
# type: () -> Union[Dict[str, Any], None]
@@ -277,9 +347,8 @@ def _create_next_node(self): # noqa: C901
yield table_node

if self.description:
yield {NODE_LABEL: DESCRIPTION_NODE_LABEL,
NODE_KEY: self._get_table_description_key(),
TableMetadata.TABLE_DESCRIPTION: self.description}
node_key = self._get_table_description_key(self.description)
yield self.description.get_node_dict(node_key)

# Create the table tag node
if self.tags:
@@ -294,21 +363,15 @@ def _create_next_node(self): # noqa: C901
ColumnMetadata.COLUMN_TYPE: col.type,
ColumnMetadata.COLUMN_ORDER: col.sort_order}

if not col.description:
continue

yield {
NODE_LABEL: DESCRIPTION_NODE_LABEL,
NODE_KEY: self._get_col_description_key(col),
ColumnMetadata.COLUMN_DESCRIPTION: col.description}

if not col.tags:
continue
if col.description:
node_key = self._get_col_description_key(col, col.description)
yield col.description.get_node_dict(node_key)

for tag in col.tags:
yield {NODE_LABEL: TagMetadata.TAG_NODE_LABEL,
NODE_KEY: TagMetadata.get_tag_key(tag),
TagMetadata.TAG_TYPE: 'default'}
if col.tags:
for tag in col.tags:
yield {NODE_LABEL: TagMetadata.TAG_NODE_LABEL,
NODE_KEY: TagMetadata.get_tag_key(tag),
TagMetadata.TAG_TYPE: 'default'}

# Database, cluster, schema
others = [NodeTuple(key=self._get_database_key(),
@@ -351,14 +414,9 @@ def _create_next_relation(self):
}

if self.description:
yield {
RELATION_START_LABEL: TableMetadata.TABLE_NODE_LABEL,
RELATION_END_LABEL: DESCRIPTION_NODE_LABEL,
RELATION_START_KEY: self._get_table_key(),
RELATION_END_KEY: self._get_table_description_key(),
RELATION_TYPE: TableMetadata.TABLE_DESCRIPTION_RELATION_TYPE,
RELATION_REVERSE_TYPE: TableMetadata.DESCRIPTION_TABLE_RELATION_TYPE
}
yield self.description.get_relation(TableMetadata.TABLE_NODE_LABEL,
self._get_table_key(),
self._get_table_description_key(self.description))

if self.tags:
for tag in self.tags:
@@ -381,30 +439,21 @@ def _create_next_relation(self):
RELATION_REVERSE_TYPE: TableMetadata.COL_TABLE_RELATION_TYPE
}

if not col.description:
continue

yield {
RELATION_START_LABEL: ColumnMetadata.COLUMN_NODE_LABEL,
RELATION_END_LABEL: DESCRIPTION_NODE_LABEL,
RELATION_START_KEY: self._get_col_key(col),
RELATION_END_KEY: self._get_col_description_key(col),
RELATION_TYPE: ColumnMetadata.COL_DESCRIPTION_RELATION_TYPE,
RELATION_REVERSE_TYPE: ColumnMetadata.DESCRIPTION_COL_RELATION_TYPE
}

if not col.tags:
continue

for tag in col.tags:
yield {
RELATION_START_LABEL: TableMetadata.TABLE_NODE_LABEL,
RELATION_END_LABEL: TagMetadata.TAG_NODE_LABEL,
RELATION_START_KEY: self._get_table_key(),
RELATION_END_KEY: TagMetadata.get_tag_key(tag),
RELATION_TYPE: ColumnMetadata.COL_TAG_RELATION_TYPE,
RELATION_REVERSE_TYPE: ColumnMetadata.TAG_COL_RELATION_TYPE,
}
if col.description:
yield col.description.get_relation(ColumnMetadata.COLUMN_NODE_LABEL,
self._get_col_key(col),
self._get_col_description_key(col, col.description))

if col.tags:
for tag in col.tags:
yield {
RELATION_START_LABEL: TableMetadata.TABLE_NODE_LABEL,
RELATION_END_LABEL: TagMetadata.TAG_NODE_LABEL,
RELATION_START_KEY: self._get_table_key(),
RELATION_END_KEY: TagMetadata.get_tag_key(tag),
RELATION_TYPE: ColumnMetadata.COL_TAG_RELATION_TYPE,
RELATION_REVERSE_TYPE: ColumnMetadata.TAG_COL_RELATION_TYPE,
}

others = [
RelTuple(start_label=TableMetadata.DATABASE_NODE_LABEL,
3 changes: 2 additions & 1 deletion example/sample_data/sample_col.csv
Original file line number Diff line number Diff line change
@@ -8,4 +8,5 @@ col1,"col1 description","string",1,dynamo,gold,test_schema,test_table2
col2,"col2 description","string",2,dynamo,gold,test_schema,test_table2
col3,"col3 description","string",3,dynamo,gold,test_schema,test_table2
col4,"col4 description","int",4,dynamo,gold,test_schema,test_table2
col1,"view col description","int",1,hive,gold,test_schema,test_view1
col1,"view col description","int",1,hive,gold,test_schema,test_view1
col1,"col1 description","int",1,hive,gold,test_schema,test_table3,""
9 changes: 5 additions & 4 deletions example/sample_data/sample_table.csv
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
database,cluster,schema_name,name,description,tags,is_view
hive,gold,test_schema,test_table1,"1st test table","tag1,tag2,pii,high_quality",false
dynamo,gold,test_schema,test_table2,"2nd test table","high_quality,recommended",false
hive,gold,test_schema,test_view1,"1st test view","tag1",true
database,cluster,schema_name,name,description,tags,is_view,description_source
hive,gold,test_schema,test_table1,"1st test table","tag1,tag2,pii,high_quality",false,
dynamo,gold,test_schema,test_table2,"2nd test table","high_quality,recommended",false,
hive,gold,test_schema,test_view1,"1st test view","tag1",true,
hive,gold,test_schema,test_table3,"3rd test","needs_documentation",false,
4 changes: 4 additions & 0 deletions example/sample_data/sample_table_programmatic_source.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
database,cluster,schema_name,name,description,tags,description_source
hive,gold,test_schema,test_table1,"**Size**: 50T\n\n**Monthly Cost**: $5000","expensive","s3_crawler"
dynamo,gold,test_schema,test_table2,"**Size**: 1T\n\n**Monthly Cost**: $50","cheap","s3_crawler"
hive,gold,test_schema,test_table1,"### Quality Report:\n --- \n Ipsus enom. Ipsus enom ipsus lorenum.\n ---\n[![Build Status](https://api.travis-ci.com/lyft/amundsendatabuilder.svg?branch=master)](https://travis-ci.com/lyft/amundsendatabuilder)","low_quality","quality_service"
60 changes: 17 additions & 43 deletions example/scripts/sample_data_loader.py
Original file line number Diff line number Diff line change
@@ -63,18 +63,19 @@ def create_connection(db_file):
return None


def load_table_data_from_csv(file_name):
def load_table_data_from_csv(file_name, table_name):
conn = create_connection(DB_FILE)
if conn:
cur = conn.cursor()
cur.execute('drop table if exists test_table_metadata')
cur.execute('create table if not exists test_table_metadata '
cur.execute('drop table if exists {}'.format(table_name))
cur.execute('create table if not exists {} '
'(database VARCHAR(64) NOT NULL , '
'cluster VARCHAR(64) NOT NULL, '
'schema_name VARCHAR(64) NOT NULL,'
'name VARCHAR(64) NOT NULL,'
'description VARCHAR(64) NOT NULL, '
'tags VARCHAR(128) NOT NULL)')
'tags VARCHAR(128) NOT NULL,'
'description_source VARCHAR(32))'.format(table_name))
file_loc = 'example/sample_data/' + file_name
with open(file_loc, 'r') as fin:
dr = csv.DictReader(fin)
@@ -83,10 +84,13 @@ def load_table_data_from_csv(file_name):
i['schema_name'],
i['name'],
i['description'],
i['tags']) for i in dr]
i['tags'],
i['description_source']) for i in dr]

cur.executemany("INSERT INTO test_table_metadata (database, cluster, "
"schema_name, name, description, tags) VALUES (?, ?, ?, ?, ?, ?);", to_db)
cur.executemany("INSERT INTO {} (database, cluster, "
"schema_name, name, description, tags, "
"description_source) "
"VALUES (?, ?, ?, ?, ?, ?, ?);".format(table_name), to_db)
conn.commit()


@@ -108,42 +112,6 @@ def load_tag_data_from_csv(file_name):
conn.commit()


def load_col_data_from_csv(file_name):
conn = create_connection(DB_FILE)
if conn:
cur = conn.cursor()
cur.execute('drop table if exists test_col_metadata')
cur.execute('create table if not exists test_col_metadata '
'(name VARCHAR(64) NOT NULL , '
'description VARCHAR(64) NOT NULL , '
'col_type VARCHAR(64) NOT NULL , '
'sort_order INTEGER NOT NULL , '
'database VARCHAR(64) NOT NULL , '
'cluster VARCHAR(64) NOT NULL, '
'schema_name VARCHAR(64) NOT NULL,'
'table_name VARCHAR(64) NOT NULL,'
'table_description VARCHAR(64) NOT NULL)')
file_loc = 'example/sample_data/' + file_name
with open(file_loc, 'r') as fin:
dr = csv.DictReader(fin)
to_db = [(i['name'],
i['description'],
i['col_type'],
i['sort_order'],
i['database'],
i['cluster'],
i['schema_name'],
i['table_name'],
i['table_description']) for i in dr]

cur.executemany("INSERT INTO test_col_metadata ("
"name, description, col_type, sort_order,"
"database, cluster, "
"schema_name, table_name, table_description) VALUES "
"(?, ?, ?, ?, ?, ?, ?, ?, ?);", to_db)
conn.commit()


def load_table_column_stats_from_csv(file_name):
conn = create_connection(DB_FILE)
if conn:
@@ -671,6 +639,7 @@ def create_table_column_job(table_path, column_path):
# Uncomment next line to get INFO level logging
# logging.basicConfig(level=logging.INFO)

load_table_data_from_csv('sample_table_programmatic_source.csv', 'programmatic')
load_table_column_stats_from_csv('sample_table_column_stats.csv')
load_watermark_data_from_csv('sample_watermark.csv')
load_table_owner_data_from_csv('sample_table_owner.csv')
@@ -693,6 +662,11 @@ def create_table_column_job(table_path, column_path):
)
table_and_col_job.launch()

# start programmatic table job
job2 = create_sample_job('programmatic',
'databuilder.models.table_metadata.TableMetadata')
job2.launch()

# start table stats job
job_table_stats = create_sample_job('test_table_column_stats',
'databuilder.models.table_stats.TableColumnStats')
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
from setuptools import setup, find_packages


__version__ = '1.5.2'
__version__ = '1.6.1'


requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
8 changes: 4 additions & 4 deletions tests/unit/extractor/test_bigquery_metadata_extractor.py
Original file line number Diff line number Diff line change
@@ -189,7 +189,7 @@ def test_table_without_schema(self, mock_build):
self.assertEquals(result.cluster, 'your-project-here')
self.assertEquals(result.schema_name, 'fdgdfgh')
self.assertEquals(result.name, 'nested_recs')
self.assertEquals(result.description, '')
self.assertEquals(result.description._text, '')
self.assertEquals(result.columns, [])
self.assertEquals(result.is_view, False)

@@ -205,7 +205,7 @@ def test_table_without_columns(self, mock_build):
self.assertEquals(result.cluster, 'your-project-here')
self.assertEquals(result.schema_name, 'fdgdfgh')
self.assertEquals(result.name, 'nested_recs')
self.assertEquals(result.description, '')
self.assertEquals(result.description._text, "")
self.assertEquals(result.columns, [])
self.assertEquals(result.is_view, False)

@@ -231,12 +231,12 @@ def test_normal_table(self, mock_build):
self.assertEquals(result.cluster, 'your-project-here')
self.assertEquals(result.schema_name, 'fdgdfgh')
self.assertEquals(result.name, 'nested_recs')
self.assertEquals(result.description, '')
self.assertEquals(result.description._text, "")

first_col = result.columns[0]
self.assertEquals(first_col.name, 'test')
self.assertEquals(first_col.type, 'STRING')
self.assertEquals(first_col.description, 'some_description')
self.assertEquals(first_col.description._text, 'some_description')
self.assertEquals(result.is_view, False)

@patch('databuilder.extractor.base_bigquery_extractor.build')
2 changes: 1 addition & 1 deletion tests/unit/extractor/test_csv_extractor.py
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ def test_extraction_with_model_class(self):

result = extractor.extract()
self.assertEquals(result.name, 'test_table1')
self.assertEquals(result.description, '1st test table')
self.assertEquals(result.description._text, '1st test table')
self.assertEquals(result.database, 'hive')
self.assertEquals(result.cluster, 'gold')
self.assertEquals(result.schema_name, 'test_schema')
44 changes: 35 additions & 9 deletions tests/unit/models/test_table_metadata.py
Original file line number Diff line number Diff line change
@@ -31,25 +31,28 @@ def test_serialize(self):
{'name': 'test_table1', 'KEY': 'hive://gold.test_schema1/test_table1', 'LABEL': 'Table',
'is_view:UNQUOTED': False},
{'description': 'test_table1', 'KEY': 'hive://gold.test_schema1/test_table1/_description',
'LABEL': 'Description'},
'LABEL': 'Description', 'description_source': 'description'},
{'sort_order:UNQUOTED': 0, 'type': 'bigint', 'name': 'test_id1',
'KEY': 'hive://gold.test_schema1/test_table1/test_id1', 'LABEL': 'Column'},
{'description': 'description of test_table1',
'KEY': 'hive://gold.test_schema1/test_table1/test_id1/_description', 'LABEL': 'Description'},
'KEY': 'hive://gold.test_schema1/test_table1/test_id1/_description', 'LABEL': 'Description',
'description_source': 'description'},
{'sort_order:UNQUOTED': 1, 'type': 'bigint', 'name': 'test_id2',
'KEY': 'hive://gold.test_schema1/test_table1/test_id2', 'LABEL': 'Column'},
{'description': 'description of test_id2',
'KEY': 'hive://gold.test_schema1/test_table1/test_id2/_description', 'LABEL': 'Description'},
'KEY': 'hive://gold.test_schema1/test_table1/test_id2/_description',
'LABEL': 'Description', 'description_source': 'description'},
{'sort_order:UNQUOTED': 2, 'type': 'boolean', 'name': 'is_active',
'KEY': 'hive://gold.test_schema1/test_table1/is_active', 'LABEL': 'Column'},
{'sort_order:UNQUOTED': 3, 'type': 'varchar', 'name': 'source',
'KEY': 'hive://gold.test_schema1/test_table1/source', 'LABEL': 'Column'},
{'description': 'description of source', 'KEY': 'hive://gold.test_schema1/test_table1/source/_description',
'LABEL': 'Description'},
'LABEL': 'Description', 'description_source': 'description'},
{'sort_order:UNQUOTED': 4, 'type': 'timestamp', 'name': 'etl_created_at',
'KEY': 'hive://gold.test_schema1/test_table1/etl_created_at', 'LABEL': 'Column'},
{'description': 'description of etl_created_at',
'KEY': 'hive://gold.test_schema1/test_table1/etl_created_at/_description', 'LABEL': 'Description'},
'KEY': 'hive://gold.test_schema1/test_table1/etl_created_at/_description', 'LABEL': 'Description',
'description_source': 'description'},
{'sort_order:UNQUOTED': 5, 'type': 'varchar', 'name': 'ds',
'KEY': 'hive://gold.test_schema1/test_table1/ds', 'LABEL': 'Column'}
]
@@ -106,16 +109,18 @@ def test_serialize(self):
while node_row:
actual.append(node_row)
node_row = self.table_metadata.next_node()

self.assertEqual(self.expected_nodes, actual)
for i in range(0, len(self.expected_nodes)):
self.assertEqual(actual[i], self.expected_nodes[i])

relation_row = self.table_metadata.next_relation()
actual = []
while relation_row:
actual.append(relation_row)
relation_row = self.table_metadata.next_relation()

self.assertEqual(self.expected_rels, actual)
for i in range(0, len(self.expected_rels)):
print(self.expected_rels[i])
print(actual[i])
self.assertEqual(actual[i], self.expected_rels[i])

# 2nd record should not show already serialized database, cluster, and schema
node_row = self.table_metadata2.next_node()
@@ -153,6 +158,27 @@ def test_table_attributes(self):
self.assertEqual(actual[0].get('attr1'), 'uri')
self.assertEqual(actual[0].get('attr2'), 'attr2')

# TODO NO test can run before serialiable... need to fix
def test_z_custom_sources(self):
# type: () -> None
self.custom_source = TableMetadata('hive', 'gold', 'test_schema3', 'test_table4', 'test_table4', [
ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0),
ColumnMetadata('test_id2', 'description of test_id2', 'bigint', 1),
ColumnMetadata('is_active', None, 'boolean', 2),
ColumnMetadata('source', 'description of source', 'varchar', 3),
ColumnMetadata('etl_created_at', 'description of etl_created_at', 'timestamp', 4),
ColumnMetadata('ds', None, 'varchar', 5)], is_view=False, description_source="custom")

node_row = self.custom_source.next_node()
actual = []
while node_row:
actual.append(node_row)
node_row = self.custom_source.next_node()
expected = {'LABEL': 'Programmatic_Description',
'KEY': 'hive://gold.test_schema3/test_table4/_custom_description',
'description_source': 'custom', 'description': 'test_table4'}
self.assertEqual(actual[1], expected)

def test_tags_field(self):
# type: () -> None
self.table_metadata4 = TableMetadata('hive', 'gold', 'test_schema4', 'test_table4', 'test_table4', [