-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(ingest): use entry point for registering transformers
- Loading branch information
1 parent
5e971ce
commit b41430e
Showing
3 changed files
with
47 additions
and
71 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 22 additions & 70 deletions
92
metadata-ingestion/src/datahub/ingestion/transformer/transform_registry.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,74 +1,26 @@ | ||
| from datahub.ingestion.api.registry import PluginRegistry | ||
| from datahub.ingestion.api.transform import Transformer | ||
| from datahub.ingestion.transformer import dataset_domain | ||
| from datahub.ingestion.transformer.add_dataset_browse_path import ( | ||
| AddDatasetBrowsePathTransformer, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_ownership import ( | ||
| AddDatasetOwnership, | ||
| PatternAddDatasetOwnership, | ||
| SimpleAddDatasetOwnership, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_properties import ( | ||
| AddDatasetProperties, | ||
| SimpleAddDatasetProperties, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_schema_tags import ( | ||
| PatternAddDatasetSchemaTags, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_schema_terms import ( | ||
| PatternAddDatasetSchemaTerms, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_tags import ( | ||
| AddDatasetTags, | ||
| PatternAddDatasetTags, | ||
| SimpleAddDatasetTags, | ||
| ) | ||
| from datahub.ingestion.transformer.add_dataset_terms import ( | ||
| AddDatasetTerms, | ||
| PatternAddDatasetTerms, | ||
| SimpleAddDatasetTerms, | ||
| ) | ||
| from datahub.ingestion.transformer.mark_dataset_status import MarkDatasetStatus | ||
| from datahub.ingestion.transformer.remove_dataset_ownership import ( | ||
| SimpleRemoveDatasetOwnership, | ||
| ) | ||
|
|
||
| transform_registry = PluginRegistry[Transformer]() | ||
|
|
||
| transform_registry.register( | ||
| "simple_remove_dataset_ownership", SimpleRemoveDatasetOwnership | ||
| ) | ||
| transform_registry.register("mark_dataset_status", MarkDatasetStatus) | ||
| transform_registry.register("set_dataset_browse_path", AddDatasetBrowsePathTransformer) | ||
|
|
||
| transform_registry.register("add_dataset_ownership", AddDatasetOwnership) | ||
| transform_registry.register("simple_add_dataset_ownership", SimpleAddDatasetOwnership) | ||
| transform_registry.register("pattern_add_dataset_ownership", PatternAddDatasetOwnership) | ||
|
|
||
| transform_registry.register("add_dataset_domain", dataset_domain.AddDatasetDomain) | ||
| transform_registry.register( | ||
| "simple_add_dataset_domain", dataset_domain.SimpleAddDatasetDomain | ||
| ) | ||
| transform_registry.register( | ||
| "pattern_add_dataset_domain", dataset_domain.PatternAddDatasetDomain | ||
| ) | ||
|
|
||
|
|
||
| transform_registry.register("add_dataset_tags", AddDatasetTags) | ||
| transform_registry.register("simple_add_dataset_tags", SimpleAddDatasetTags) | ||
| transform_registry.register("pattern_add_dataset_tags", PatternAddDatasetTags) | ||
|
|
||
| transform_registry.register("add_dataset_terms", AddDatasetTerms) | ||
| transform_registry.register("simple_add_dataset_terms", SimpleAddDatasetTerms) | ||
| transform_registry.register("pattern_add_dataset_terms", PatternAddDatasetTerms) | ||
|
|
||
| transform_registry.register("add_dataset_properties", AddDatasetProperties) | ||
| transform_registry.register("simple_add_dataset_properties", SimpleAddDatasetProperties) | ||
|
|
||
| transform_registry.register( | ||
| "pattern_add_dataset_schema_terms", PatternAddDatasetSchemaTerms | ||
| ) | ||
| transform_registry.register( | ||
| "pattern_add_dataset_schema_tags", PatternAddDatasetSchemaTags | ||
| ) | ||
| transform_registry.register_from_entrypoint("datahub.ingestion.transformer.plugins") | ||
|
|
||
| # These transformers are always enabled | ||
| assert transform_registry.get("simple_remove_dataset_ownership") | ||
| assert transform_registry.get("mark_dataset_status") | ||
| assert transform_registry.get("set_dataset_browse_path") | ||
| assert transform_registry.get("add_dataset_ownership") | ||
| assert transform_registry.get("simple_add_dataset_ownership") | ||
| assert transform_registry.get("pattern_add_dataset_ownership") | ||
| assert transform_registry.get("add_dataset_domain") | ||
| assert transform_registry.get("simple_add_dataset_domain") | ||
| assert transform_registry.get("pattern_add_dataset_domain") | ||
| assert transform_registry.get("add_dataset_tags") | ||
| assert transform_registry.get("simple_add_dataset_tags") | ||
| assert transform_registry.get("pattern_add_dataset_tags") | ||
| assert transform_registry.get("add_dataset_terms") | ||
| assert transform_registry.get("simple_add_dataset_terms") | ||
| assert transform_registry.get("pattern_add_dataset_terms") | ||
| assert transform_registry.get("add_dataset_properties") | ||
| assert transform_registry.get("simple_add_dataset_properties") | ||
| assert transform_registry.get("pattern_add_dataset_schema_terms") | ||
| assert transform_registry.get("pattern_add_dataset_schema_tags") |