Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: Added "is_public" to tabular_datasets table #501

Merged
merged 7 commits into from
Oct 7, 2022
Merged

Fix: Added "is_public" to tabular_datasets table #501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8817672
Adding a boolean column to show if a dataset is public or not
happyhuman Oct 5, 2022
b045b0c
Removing extra files
happyhuman Oct 5, 2022
6737ef9
Generated the terraform file.
happyhuman Oct 5, 2022
2573c0a
Fixed the logging message
happyhuman Oct 6, 2022
f820745
Added the new column name
happyhuman Oct 6, 2022
4a247d0
After running black.
happyhuman Oct 6, 2022
8cfbbe9
After running black.
happyhuman Oct 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion datasets/cloud_datasets/infra/pdp_extract_tabular_metadata_pipeline.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,12 @@ resource "google_bigquery_table" "_cloud_datasets_tabular_datasets" {
"name": "num_tables",
"description": "Number of tables contained in this dataset",
"type": "INTEGER"
}
},
{
"name": "is_public",
"description": "Whether or not the dataset is public to all users",
"type": "BOOLEAN"
}
]
EOF
depends_on = [
Expand Down
8 changes: 7 additions & 1 deletion datasets/cloud_datasets/pipelines/_images/pdp_extract_tabular_metadata/script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
"dataset_id",
"description",
"num_tables",
"is_public",
]

TABLES_COLUMNS = [
Expand Down Expand Up @@ -148,6 +149,7 @@ class DatasetInfo:
dataset_id: str = None
description: str = None
num_tables: int = None
is_public: bool = None

def __init__(
self,
Expand All @@ -161,6 +163,10 @@ def __init__(
self.description = np.nan
self.created_at = dataset_reference.created
self.modified_at = dataset_reference.modified
entries = list(dataset_reference.access_entries)
self.is_public = any(
map(lambda e: e.entity_id in {"allAuthenticatedUsers", "allUsers"}, entries)
)

def __repr__(self) -> str:
return f"{self.project_id}.{self.dataset_id}"
Expand Down Expand Up @@ -344,7 +350,7 @@ def main(
extractor.write_datasets_to_bq(tabular_dataset_table_name, extracted)
extractor.write_tables_to_bq(tables_table_name, extracted)
extractor.write_tables_fields_to_bq(tables_fields_table_name, extracted)
logging.info("Total time to run this function: ", time.time() - st)
logging.info("Total time to run this function: %s", time.time() - st)


if __name__ == "__main__":
Expand Down
5 changes: 5 additions & 0 deletions datasets/cloud_datasets/pipelines/pdp_extract_tabular_metadata/pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ resources:
"name": "num_tables",
"description": "Number of tables contained in this dataset",
"type": "INTEGER"
},
{
"name": "is_public",
"description": "Whether or not the dataset is public to all users",
"type": "BOOLEAN"
}
]
- type: bigquery_table
Expand Down