Completed -swt with cli and web ui access.

Writerside/topics/Index-of-Settings.md

@@ -261,12 +261,35 @@ This does NOT prevent data from being manipulated on the target cluster after th
 <tab title="Config File">
 
 ```yaml
-  readOnly: true|false
+readOnly: true|false
 ```
 
 </tab>
 </tabs>
 
+## Save Working Tables
+
+`saveWorkingTables` is a boolean value that determines if the working tables should be saved. The default value is `false`.
+
+<tabs>
+<tab title="CLI">
+
+`-swt|--save-working-tables`
+
+</tab>
+<tab title="Web UI">
+
+![save-working-tables.png](save-working-tables.png)
+
+</tab>
+<tab title="Config File">
+
+```yaml
+saveWorkingTables: true|false
+```
+
+</tab>
+</tabs>
 
 ## Skip Features
 

Writerside/topics/Release-Notes.md

@@ -9,10 +9,16 @@ The latest set of known issues can be found [here](https://github.com/cloudera-l
 The latest set of enhancement requests can be found [here](https://github.com/cloudera-labs/hms-mirror/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement). 
 
 If there is something you'd like to see, add a new issue [here](https://github.com/cloudera-labs/hms-mirror/issues)
-## 2.2.0.19.4 (release)
+
+## 2.2.0.19.6 (pre-release)
 
 **What's New**
 [BEHAVIOR CHANGE - Drop any shadow table definitions created during migration automatically](https://github.com/cloudera-labs/hms-mirror/issues/163)
+> This adjustment will remove shadow tables that were created during the migration process.  This will help to keep 
+> the source and target clusters clean of any artifacts created during the migration process. An option 
+> `saveWorkingTables` has been added to the configuration to allow you to keep these tables if you need them for any 
+> audits or other reasons. The default is `false`, which means that the tables will be dropped automatically. An audit 
+> 'cleanup' file with the drop commands will be created in the report directory, regardless of the setting.
 
 ## 2.2.0.19.2 (pre-release)
 

pom.xml

@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.2.0.19.4</version>
+    <version>2.2.0.19.6</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java

@@ -985,6 +985,18 @@ CommandLineRunner configRightIsDisconnectedFalse(HmsMirrorConfig hmsMirrorConfig
         };
     }
 
+    @Bean
+    @Order(1)
+    @ConditionalOnProperty(
+            name = "hms-mirror.config.save-working-tables")
+    CommandLineRunner configSaveWorkingTables(HmsMirrorConfig hmsMirrorConfig, @Value("${hms-mirror.config.save-working-tables}") boolean value) {
+        return args -> {
+            log.warn("save-working-tables: {}", value);
+            hmsMirrorConfig.setSaveWorkingTables(value);
+        };
+    }
+
+
     @Bean
     @Order(1)
     @ConditionalOnProperty(
@@ -1461,6 +1473,11 @@ private Options getOptions() {
         resetToDefaultLocation.setRequired(Boolean.FALSE);
         options.addOption(resetToDefaultLocation);
 
+        Option saveWorkingTables = new Option("swt", "save-working-tables", false,
+                "Save working tables (shadow tables) created during the migration process.");
+        saveWorkingTables.setRequired(Boolean.FALSE);
+        options.addOption(saveWorkingTables);
+
         Option skipLegacyTranslation = new Option("slt", "skip-legacy-translation", false,
                 "Skip Schema Upgrades and Serde Translations");
         skipLegacyTranslation.setRequired(Boolean.FALSE);

src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptionsEnum.java

@@ -75,6 +75,7 @@ public enum HmsMirrorCommandLineOptionsEnum {
     READ_ONLY("ro", "read-only", null, ""),
     RESET_RIGHT("rr", "reset-right", null, ""),
     SYNC("s", "sync", null, ""),
+    SAVE_WORKING_TABLES("swt", "save-working-tables", null, ""),
     SORT_DYNAMIC_PARTITION_INSERTS("sdpi", "sort-dynamic-partition-inserts", null, ""),
     SKIP_FEATURES("sf", "skip-features", null, ""),
     SKIP_LINK_CHECK("slc", "skip-link-check", null, ""),

src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java

@@ -628,23 +628,12 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
                 } else {
                     // Otherwise, we're on the RIGHT cluster and need to cleanup the 'shadow' table.
                     // Add USE clause to the SQL
+                    String rightDatabase = HmsMirrorConfigUtil.getResolvedDB(tableMirror.getParent().getName(), config);
                     if (config.isSaveWorkingTables()) {
                         Pair cleanUp = new Pair("Post Migration Cleanup", "-- To be run AFTER final RIGHT SQL statements.");
                         targetEnvTable.addCleanUpSql(cleanUp);
-
-                        String rightDatabase = HmsMirrorConfigUtil.getResolvedDB(tableMirror.getParent().getName(), config);
-
-                        String useRightDb = MessageFormat.format(MirrorConf.USE, rightDatabase);
-                        Pair leftRightPair = new Pair(TableUtils.USE_DESC, useRightDb);
-                        targetEnvTable.addCleanUpSql(leftRightPair);
-
-                        // Drop Shadow Table.
-                        String dropTransferSql = MessageFormat.format(MirrorConf.DROP_TABLE, source.getName());
-                        targetEnvTable.getCleanUpSql().add(new Pair(TableUtils.DROP_SHADOW_TABLE, dropTransferSql));
                     } else {
                         // Remove Shadow table as a part of the automatic cleanup.
-                        String rightDatabase = HmsMirrorConfigUtil.getResolvedDB(tableMirror.getParent().getName(), config);
-
                         String useRightDb = MessageFormat.format(MirrorConf.USE, rightDatabase);
                         Pair leftRightPair = new Pair(TableUtils.USE_DESC, useRightDb);
                         targetEnvTable.addSql(leftRightPair);
@@ -653,6 +642,14 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
                         String dropTransferSql = MessageFormat.format(MirrorConf.DROP_TABLE, source.getName());
                         targetEnvTable.getSql().add(new Pair(TableUtils.DROP_SHADOW_TABLE, dropTransferSql));
                     }
+                    // Always generate the drop working table scripts.
+                    String useRightDb = MessageFormat.format(MirrorConf.USE, rightDatabase);
+                    Pair leftRightPair = new Pair(TableUtils.USE_DESC, useRightDb);
+                    targetEnvTable.addCleanUpSql(leftRightPair);
+
+                    // Drop Shadow Table.
+                    String dropTransferSql = MessageFormat.format(MirrorConf.DROP_TABLE, source.getName());
+                    targetEnvTable.getCleanUpSql().add(new Pair(TableUtils.DROP_SHADOW_TABLE, dropTransferSql));
                 }
             }
         }

src/main/resources/application.yaml

@@ -22,6 +22,7 @@ hms-mirror:
     filename: "default.yaml"
     setup: "false"
     suppress-cli-warnings: "false"
+    save-working-tables: "false"
     yaml:
       # This controls the maximum number of bytes that can be read from a YAML file.
       # The default value in SnakeYaml is 3MB.  We've reset this to 100MB.

src/main/resources/messages.properties

@@ -353,6 +353,7 @@ config.transfer.storageMigration.postfix=Storage Migration Postfix
 config.sync=Sync
 config.sqlOutput=SQL Output
 config.skipLegacyTranslation=Skip Legacy Translation
+config.saveWorkingTables=Save Working Tables
 config.skipFeatures=Skip Features
 config.resetToDefaultLocation=Reset to Default Location
 config.resetRight=Reset Right

src/main/resources/templates/fragments/config/misc.html

@@ -84,6 +84,12 @@
                    th:checked="${CONFIG.resetRight}"/>
             <!--        <label for="resetRight" th:text="#{config.resetRight}"></label>-->
             <!--    </div>-->
+            <div class="pure-control-group">
+                <label for="saveWorkingTables" th:text="#{config.saveWorkingTables}"></label>
+                <input type="checkbox" name="saveWorkingTables"
+                       th:field="*{saveWorkingTables}"
+                       th:checked="${CONFIG.saveWorkingTables}"/>
+            </div>
             <div class="pure-control-group">
                 <label for="skipFeatures" th:text="#{config.skipFeatures}"></label>
                 <input type="checkbox" name="skipFeatures"

src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp/Test_sm_smn_wd_epl_glm_fel_dc.java

@@ -142,7 +142,7 @@ public void returnCodeTest() {
     @Test
     public void validateTableIssueCount() {
         validateTableIssueCount("ext_purge_odd_parts", "web_sales",
-                Environment.LEFT, 2);
+                Environment.LEFT, 3);
 
 //        assertEquals("Issue Count not as expected", 3,
 //                getConversion().getDatabase("ext_purge_odd_parts")

src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/cdp_to_cdp/Test_so_ma_exist_sync_01.java

@@ -57,8 +57,8 @@ public void returnCodeTest() {
         // Get Runtime Return Code.
         long rtn = getReturnCode();
         // Verify the return code.
-        long check = 0L;
-        assertEquals("Return Code Failure: " + rtn, check * -1, rtn);
+        long check = 1L;
+        assertEquals("Return Code Failure: " + rtn, check, rtn);
     }
 
 //    @Test