From 4a649aa61cd59d06b66b232a45505dd38478d8e2 Mon Sep 17 00:00:00 2001 From: "David W. Streever" Date: Tue, 12 Nov 2024 06:15:18 -0500 Subject: [PATCH] Fixed property overrides placement to correct cluster. --- pom.xml | 2 +- .../mirror/datastrategy/DataStrategyBase.java | 5 +-- .../SQLAcidDowngradeInPlaceDataStrategy.java | 9 +++-- .../StorageMigrationDataStrategy.java | 7 ++-- .../utils/hms/mirror/domain/Overrides.java | 36 +++++++++---------- .../HmsMirrorConfigCloneFullTest_01.java | 4 +-- .../end_to_end/legacy_to_cdp/Test_sql_is.java | 3 ++ 7 files changed, 36 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 2a9c8846..aa5c2c1e 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ com.cloudera.utils.hadoop hms-mirror - 2.2.0.18.0 + 2.2.0.18.1 jar hms-mirror diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java index bc85f2d9..bc352fe8 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java @@ -569,8 +569,9 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina // Set Override Properties. if (config.getOptimization().getOverrides() != null) { - for (String key : config.getOptimization().getOverrides().getLeft().keySet()) { - targetEnvTable.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key)); + Map overrides = config.getOptimization().getOverrides().getFor(targetEnv2); + for (String key : overrides.keySet()) { + targetEnvTable.addSql("Setting " + key, "set " + key + "=" + overrides.get(key)); } } diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java index beb648d7..c3b5c6a8 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java @@ -37,6 +37,7 @@ import org.springframework.stereotype.Component; import java.text.MessageFormat; +import java.util.Map; import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION; import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION_THRESHOLD; @@ -125,9 +126,11 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep let.addSql(TableUtils.USE_DESC, useDb); // Set Override Properties. - if (!hmsMirrorConfig.getOptimization().getOverrides().getLeft().isEmpty()) { - for (String key : hmsMirrorConfig.getOptimization().getOverrides().getLeft().keySet()) { - let.addSql("Setting " + key, "set " + key + "=" + hmsMirrorConfig.getOptimization().getOverrides().getLeft().get(key)); + // Get the LEFT overrides for the DOWNGRADE. + Map overrides = hmsMirrorConfig.getOptimization().getOverrides().getFor(Environment.LEFT); + if (!overrides.isEmpty()) { + for (String key : overrides.keySet()) { + let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key)); } } diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java index ac1253ad..eb8611a9 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java @@ -489,9 +489,10 @@ public Boolean execute(TableMirror tableMirror) { let.addSql(new Pair(TEZ_EXECUTION_DESC, SET_TEZ_AS_EXECUTION_ENGINE)); } // Set Override Properties. - if (config.getOptimization().getOverrides() != null) { - for (String key : config.getOptimization().getOverrides().getLeft().keySet()) { - let.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key)); + Map overrides = config.getOptimization().getOverrides().getFor(Environment.LEFT); + if (!overrides.isEmpty()) { + for (String key : overrides.keySet()) { + let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key)); } } diff --git a/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java b/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java index 5402fb90..b944e6e7 100644 --- a/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java +++ b/src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java @@ -17,6 +17,7 @@ package com.cloudera.utils.hms.mirror.domain; +import com.cloudera.utils.hms.mirror.domain.support.Environment; import com.cloudera.utils.hms.mirror.domain.support.SideType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -47,31 +48,28 @@ public void addProperty(String key, String value, SideType side) { } @JsonIgnore - public Map getLeft() { - Map left = new TreeMap(); - for (Map.Entry> entry : properties.entrySet()) { - if (entry.getValue().containsKey(SideType.LEFT)) { - left.put(entry.getKey(), entry.getValue().get(SideType.LEFT)); - } - if (entry.getValue().containsKey(SideType.BOTH)) { - left.put(entry.getKey(), entry.getValue().get(SideType.BOTH)); - } - } - return left; - } + public Map getFor(Environment environment) { - @JsonIgnore - public Map getRight() { - Map right = new TreeMap(); + Map rtn = new TreeMap(); for (Map.Entry> entry : properties.entrySet()) { - if (entry.getValue().containsKey(SideType.RIGHT)) { - right.put(entry.getKey(), entry.getValue().get(SideType.RIGHT)); + switch (environment) { + case LEFT: + if (entry.getValue().containsKey(SideType.LEFT)) { + rtn.put(entry.getKey(), entry.getValue().get(SideType.LEFT)); + } + break; + case RIGHT: + if (entry.getValue().containsKey(SideType.RIGHT)) { + rtn.put(entry.getKey(), entry.getValue().get(SideType.RIGHT)); + } + break; } if (entry.getValue().containsKey(SideType.BOTH)) { - right.put(entry.getKey(), entry.getValue().get(SideType.BOTH)); + rtn.put(entry.getKey(), entry.getValue().get(SideType.BOTH)); } } - return right; + + return rtn; } public void setPropertyOverridesStr(String[] inPropsStr, SideType side) { diff --git a/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java b/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java index 1865432b..82689dfa 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java @@ -147,8 +147,8 @@ public void cloneTest_01() { assertEquals(clone.getOptimization().isAutoTune(), config.getOptimization().isAutoTune()); assertEquals(clone.getOptimization().isCompressTextOutput(), config.getOptimization().isCompressTextOutput()); assertEquals(clone.getOptimization().isSkipStatsCollection(), config.getOptimization().isSkipStatsCollection()); - assertEquals(clone.getOptimization().getOverrides().getLeft(), config.getOptimization().getOverrides().getLeft()); - assertEquals(clone.getOptimization().getOverrides().getRight(), config.getOptimization().getOverrides().getRight()); + assertEquals(clone.getOptimization().getOverrides().getFor(Environment.LEFT), config.getOptimization().getOverrides().getFor(Environment.LEFT)); + assertEquals(clone.getOptimization().getOverrides().getFor(Environment.RIGHT), config.getOptimization().getOverrides().getFor(Environment.RIGHT)); assertEquals(clone.getOptimization().isBuildShadowStatistics(), config.getOptimization().isBuildShadowStatistics()); assertEquals(clone.getOutputDirectory(), config.getOutputDirectory()); diff --git a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java index 3aca3051..11895c7b 100644 --- a/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java +++ b/src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java @@ -43,6 +43,9 @@ // "--hms-mirror.config.target-namespace=s3a://my_cs_bucket", // "--hms-mirror.config.reset-to-default-location=true", // "--hms-mirror.config.distcp=true", + "--hms-mirror.config.property-overrides=hive.exec.orc.split.strategy=BI", + "--hms-mirror.config.property-overrides-left=tez.queue.name=left-side", + "--hms-mirror.config.property-overrides-right=tez.queue.name=right-side", "--hms-mirror.conversion.test-filename=/test_data/assorted_tbls_01.yaml", "--hms-mirror.config.filename=/config/default.yaml.hdp2-cdp", "--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/legacy_cdp/sql_is"