Skip to content

Commit

Permalink
Fix typos
Browse files Browse the repository at this point in the history
  • Loading branch information
@treff7es
treff7es committed Aug 5, 2024
1 parent e401ee9 commit 56e14e2
Show file tree
Hide file tree
Showing 10 changed files with 14 additions and 14 deletions.
2 changes: 1 addition & 1 deletion metadata-ingestion/src/datahub/ingestion/source/data_lake_common/path_spec.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ class Config:
description="Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`",
)

taversal_method: FolderTraversalMethod = Field(
traversal_method: FolderTraversalMethod = Field(
default=FolderTraversalMethod.MAX,
description="Method to traverse the folder. ALL: Traverse all the folders, MIN_MAX: Traverse the folders by finding min and max value, MAX: Traverse the folder with max value",
)
Expand Down
10 changes: 5 additions & 5 deletions metadata-ingestion/src/datahub/ingestion/source/s3/source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ def ingest_table(
if table_data.partitions:
customProperties.update(
{
"number_of_partitons": str(
"number_of_partitions": str(
len(table_data.partitions) if table_data.partitions else 0
),
"partitions": str(
Expand Down Expand Up @@ -1051,13 +1051,13 @@ def s3_browser(
):
dirs_to_process = []
logger.info(f"Processing folder: {f}")
if path_spec.taversal_method == FolderTraversalMethod.ALL:
if path_spec.traversal_method == FolderTraversalMethod.ALL:
dirs_to_process.append(f)
else:
if (
path_spec.taversal_method
path_spec.traversal_method
== FolderTraversalMethod.MIN_MAX
or path_spec.taversal_method
or path_spec.traversal_method
== FolderTraversalMethod.MAX
):
protocol = ContainerWUCreator.get_protocol(
Expand All @@ -1072,7 +1072,7 @@ def s3_browser(
dirs_to_process.append(dirs_to_process_max[0])

if (
path_spec.taversal_method
path_spec.traversal_method
== FolderTraversalMethod.MIN_MAX
):
dirs_to_process_min = self.get_dir_to_process(
Expand Down
2 changes: 1 addition & 1 deletion ...ta-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "1",
"number_of_partitions": "1",
"partitions": "{'min_partition': {'id': [('partition[0]', 'year=2022'), ('partition[1]', 'month=jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}, 'max_partition': {'id': [('partition[0]', 'year=2022'), ('partition[1]', 'month=jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion ...a-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "1",
"number_of_partitions": "1",
"partitions": "{'min_partition': {'id': [('partition_key[0]', 'year'), ('partition[0]', '2022'), ('partition_key[1]', 'month'), ('partition[1]', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}, 'max_partition': {'id': [('partition_key[0]', 'year'), ('partition[0]', '2022'), ('partition_key[1]', 'month'), ('partition[1]', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion ...tion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_update_schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "1",
"number_of_partitions": "1",
"partitions": "{'min_partition': {'id': [('partition[0]', 'year=2022'), ('partition[1]', 'month=jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}, 'max_partition': {'id': [('partition[0]', 'year=2022'), ('partition[1]', 'month=jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion ...golden-files/s3/golden_mces_folder_partition_update_schema_with_partition_autodetect.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "1",
"number_of_partitions": "1",
"partitions": "{'min_partition': {'id': [('year', '2022'), ('month', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}, 'max_partition': {'id': [('year', '2022'), ('month', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion .../golden-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "7",
"number_of_partitions": "7",
"partitions": "{'min_partition': {'id': [('year', '2019'), ('month', 'feb')], 'creation_time': '2020-04-14 07:04:20+00:00', 'modification_time': '2020-04-14 07:04:30+00:00', 'size': '29010', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part2.json'}, 'max_partition': {'id': [('year', '2022'), ('month', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion ...den-files/s3/golden_mces_folder_partition_with_partition_autodetect_traverse_min_max.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,7 +774,7 @@
"json": {
"customProperties": {
"schema_inferred_from": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json",
"number_of_partitons": "2",
"number_of_partitions": "2",
"partitions": "{'min_partition': {'id': [('year', '2019'), ('month', 'feb')], 'creation_time': '2020-04-14 07:04:20+00:00', 'modification_time': '2020-04-14 07:04:30+00:00', 'size': '29010', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part2.json'}, 'max_partition': {'id': [('year', '2022'), ('month', 'jan')], 'creation_time': '2020-04-14 07:06:20+00:00', 'modification_time': '2020-04-14 07:06:20+00:00', 'size': '14540', 'sample_file': 's3://my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2022/month=jan/part3.json'}}"
},
"name": "folder_aaa.pokemon_abilities_json",
Expand Down
2 changes: 1 addition & 1 deletion ...ts/integration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/",
"sample_files": true,
"autodetect_partitions": true,
"taversal_method": "ALL",
"traversal_method": "ALL",
"table_name": "{dept}.{table}",
"exclude":[
"**/folder_aaaa/**"
Expand Down
2 changes: 1 addition & 1 deletion ...ntegration/s3/sources/s3/folder_partition_with_partition_autodetect_traverse_min_max.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/",
"sample_files": true,
"autodetect_partitions": true,
"taversal_method": "MIN_MAX",
"traversal_method": "MIN_MAX",
"table_name": "{dept}.{table}",
"exclude":[
"**/folder_aaaa/**"
Expand Down

0 comments on commit 56e14e2

Please sign in to comment.