[Second run in WebUI Fails](cloudera-labs/hms-mirror#181)

[dbRegEx not being processed. Throws MISC_ERROR because it can't find any databases.](cloudera-labs/hms-mirror#180) [Legacy DBPROPERTIES are causing ERROR when attempting to set on CDP](cloudera-labs/hms-mirror#179)
dstreev · Feb 6, 2025 · 0252fe4 · 0252fe4
1 parent ce0e1e8
commit 0252fe4
Show file tree

Hide file tree

Showing 16 changed files with 277 additions and 15 deletions.
diff --git a/pom.xml b/pom.xml
@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.3.0.9</version>
+    <version>2.3.0.10</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/MirrorConf.java b/src/main/java/com/cloudera/utils/hms/mirror/MirrorConf.java
@@ -43,7 +43,7 @@ public interface MirrorConf {
     String ALTER_DB_LOCATION_DESC =
             "Alter Database Location";
     String ALTER_DB_PROPERTIES =
-            "ALTER DATABASE {0} SET DBPROPERTIES ({1}={2})";
+            "ALTER DATABASE {0} SET DBPROPERTIES (''{1}''=''{2}'')";
     String ALTER_DB_PROPERTIES_DESC = "Set database property";
     String DEFAULT_MANAGED_BASE_DIR = "/warehouse/tablespace/managed/hive";
     String DEFAULT_FS = "fs.defaultFS";

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java b/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptions.java
@@ -328,6 +328,20 @@ CommandLineRunner configDatabasePrefix(HmsMirrorConfig hmsMirrorConfig, @Value("
         };
     }
 
+    @Bean
+    @Order(1)
+    @ConditionalOnProperty(
+            name = "hms-mirror.config.database-skip-properties")
+    CommandLineRunner configDatabasePropertySkip(HmsMirrorConfig hmsMirrorConfig, @Value("${hms-mirror.config.database-skip-properties}") String values) {
+        return args -> {
+            log.info("database-skip-properties: {}", values);
+            for (String prop: values.split(",")) {
+                hmsMirrorConfig.getFilter().getDbPropertySkipList().add(prop);
+            }
+        };
+    }
+
+
     @Bean
     @Order(1)
     @ConditionalOnProperty(
@@ -1678,6 +1692,15 @@ private Options getOptions() {
 
         options.addOptionGroup(storageOptionsGroup);
 
+        Option dbPropertySkipOption = new Option("dbsp", "database-skip-properties", true,
+                "Comma separated list of database properties (regex) to skip during the migration process.  " +
+                        "This will prevent the property from being set on the target cluster.");
+        dbPropertySkipOption.setArgs(1);
+        dbPropertySkipOption.setOptionalArg(Boolean.TRUE);
+        dbPropertySkipOption.setArgName("properties");
+        dbPropertySkipOption.setRequired(Boolean.FALSE);
+        options.addOption(dbPropertySkipOption);
+
         // External Warehouse Dir
         Option externalWarehouseDirOption = new Option("ewd", "external-warehouse-directory", true,
                 "The external warehouse directory path.  Should not include the namespace OR the database directory. " +

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptionsEnum.java b/src/main/java/com/cloudera/utils/hms/mirror/cli/HmsMirrorCommandLineOptionsEnum.java
@@ -36,6 +36,7 @@ public enum HmsMirrorCommandLineOptionsEnum {
     DATABASE("db", "database", "databases", ""),
     DATABASE_ONLY("dbo", "database-only", null, ""),
     DATABASE_PREFIX("dbp", "db-prefix", "prefix", ""),
+    DATABASE_SKIP_PROPERTIES("dbsp", "database-skip-properties", "key(s)", ""),
     DATABASE_RENAME("dbr", "db-rename", "rename", ""),
     DATABASE_REGEX("dbRegEx", "database-regex", "regex", ""),
     //TODO: Double check conversion from legacy.

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/domain/Filter.java b/src/main/java/com/cloudera/utils/hms/mirror/domain/Filter.java
@@ -22,6 +22,10 @@
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import static java.util.Objects.isNull;
@@ -32,8 +36,14 @@
 public class Filter implements Cloneable {
     @JsonIgnore
     private Pattern dbFilterPattern = null;
-    @JsonIgnore // wip
+
     private String dbRegEx = null;
+    private List<String> dbPropertySkipList = new ArrayList<>();
+
+    @JsonIgnore
+    // Build a set of Pattern Objects based on the dbPropertySkipList
+    private Set<Pattern> dbPropertySkipListPattern = new HashSet<>();
+
     @JsonIgnore
     private Pattern tblExcludeFilterPattern = null;
     @JsonIgnore
@@ -58,9 +68,50 @@ public Filter clone() {
         }
     }
 
-//    public void setDbRegEx(String dbRegEx) {
-//        this.dbRegEx = dbRegEx;
+    public Set<Pattern> getDbPropertySkipListPattern() {
+        // Whenever this is retrieved this way, we need to ensure that the dbPropertySkipListPattern is in sync
+        dbPropertySkipListPattern.clear();
+        for (String dbPropertySkip : dbPropertySkipList) {
+            dbPropertySkipListPattern.add(Pattern.compile(dbPropertySkip));
+        }
+        return dbPropertySkipListPattern;
+    }
+
+    public void setDbPropertySkipList(List<String> dbPropertySkipList) {
+        this.dbPropertySkipList = dbPropertySkipList;
+        dbPropertySkipListPattern.clear();
+        for (String dbPropertySkip : dbPropertySkipList) {
+            dbPropertySkipListPattern.add(Pattern.compile(dbPropertySkip));
+        }
+    }
+
+    public void addDbPropertySkipItem(String dbPropertySkipItem) {
+        if (!isBlank(dbPropertySkipItem)) {
+            this.dbPropertySkipList.add(dbPropertySkipItem);
+            dbPropertySkipListPattern.add(Pattern.compile(dbPropertySkipItem));
+        }
+    }
+
+    public void setDbPropertySkipListPattern(Set<Pattern> dbPropertySkipListPattern) {
+        // Do nothing here, as this is a derived field
+    }
+
+    //    public void removeDbPropertySkipItem(String dbPropertySkipItem) {
+//        if (!isBlank(dbPropertySkipItem)) {
+//            this.dbPropertySkipList.remove(dbPropertySkipItem);
+//            dbPropertySkipListPattern.remove(Pattern.compile(dbPropertySkipItem));
+//        }
 //    }
+//
+    public void removeDbPropertySkipItemByIndex(int index) {
+        try {
+            this.dbPropertySkipList.remove(index);
+        } catch (IndexOutOfBoundsException e) {
+            // Nothing to do.
+        } finally {
+            this.dbPropertySkipListPattern.clear();
+        }
+    }
 
     public Pattern getDbFilterPattern() {
         if (!isBlank(dbRegEx) && isNull(dbFilterPattern)) {

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/ConfigService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/ConfigService.java
@@ -906,7 +906,7 @@ public Boolean validate(ExecuteSession session, CliEnvironment cli) {
             session.addError(CONFLICTING_PROPERTIES, "dbRename", "dbPrefix");
         }
 
-        if (isNull(config.getDatabases()) || config.getDatabases().isEmpty()) {
+        if ((isNull(config.getDatabases()) || config.getDatabases().isEmpty()) && (isBlank(config.getFilter().getDbRegEx()))) {
 //            log.error("No databases specified OR found if you used dbRegEx");
             session.addError(MISC_ERROR, "No databases specified OR found if you used dbRegEx");
             rtn.set(Boolean.FALSE);

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/ConnectionPoolService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/ConnectionPoolService.java
@@ -98,6 +98,11 @@ public void close() {
             // Set State of Connection.
             connected = false;
             getConnectionPools().close();
+            try {
+                getExecuteSession().setConnected(Boolean.FALSE);
+            } catch (SessionException e) {
+                throw new RuntimeException(e);
+            }
             // Set to null to allow for reset.
             connectionPools = null;
         }

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/DatabaseService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/DatabaseService.java
@@ -67,7 +67,7 @@ public class DatabaseService {
     private WarehouseService warehouseService;
     private ConfigService configService;
 
-    private final List<String> skipList = Arrays.asList(DB_LOCATION, DB_MANAGED_LOCATION, COMMENT, DB_NAME, OWNER_NAME, OWNER_TYPE);
+    public static final Set<String> skipList = new HashSet<String>(Arrays.asList(DB_LOCATION, DB_MANAGED_LOCATION, COMMENT, DB_NAME, OWNER_NAME, OWNER_TYPE));
 
     @Autowired
     public void setConfigService(ConfigService configService) {
@@ -877,7 +877,9 @@ public boolean buildDBStatements(DBMirror dbMirror) {
                     }
 
                     // Build the DBPROPERITES
-                    Map<String, String> dbProperties = DatabaseUtils.getParameters(dbDefRight, skipList);
+                    // Check if the user has specified any DB Properties to skip.
+                    Set<String> lclSkipList = new HashSet<>(skipList);
+                    Map<String, String> dbProperties = DatabaseUtils.getParameters(dbDefRight, lclSkipList, config.getFilter().getDbPropertySkipListPattern());
                     if (!dbProperties.isEmpty()) {
                         for (Map.Entry<String, String> entry : dbProperties.entrySet()) {
                             String alterDbProps = MessageFormat.format(ALTER_DB_PROPERTIES, targetDatabase, entry.getKey(), entry.getValue());

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/HMSMirrorAppService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/HMSMirrorAppService.java
@@ -125,8 +125,8 @@ public Future<Boolean> run() {
             runStatus.setStage(StageEnum.VALIDATING_CONFIG, CollectionEnum.COMPLETED);
         } else {
             runStatus.setStage(StageEnum.VALIDATING_CONFIG, CollectionEnum.ERRORED);
-            runStatus.setProgress(ProgressEnum.FAILED);
             reportWriterService.wrapup();
+            runStatus.setProgress(ProgressEnum.FAILED);
             return new AsyncResult<>(Boolean.FALSE);
         }
 
@@ -150,37 +150,43 @@ public Future<Boolean> run() {
                     runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                     runStatus.setProgress(ProgressEnum.FAILED);
                     connectionPoolService.close();
+                    runStatus.setProgress(ProgressEnum.FAILED);
                     return new AsyncResult<>(Boolean.FALSE);
                 }
             } catch (SQLException sqle) {
                 log.error("Issue refreshing connections pool", sqle);
                 runStatus.addError(CONNECTION_ISSUE, sqle.getMessage());
                 runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             } catch (URISyntaxException e) {
                 log.error("URI issue with connections pool", e);
                 runStatus.addError(CONNECTION_ISSUE, e.getMessage());
                 runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             } catch (SessionException se) {
                 log.error("Issue with Session", se);
                 runStatus.addError(SESSION_ISSUE, se.getMessage());
                 runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             } catch (EncryptionException ee) {
                 log.error("Issue with Decryption", ee);
                 runStatus.addError(ENCRYPTION_ISSUE, ee.getMessage());
                 runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             } catch (RuntimeException rte) {
                 log.error("Runtime Issue", rte);
                 runStatus.addError(SESSION_ISSUE, rte.getMessage());
                 runStatus.setStage(StageEnum.CONNECTION, CollectionEnum.ERRORED);
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             }
         } else {
@@ -252,6 +258,7 @@ public Future<Boolean> run() {
                 executeSessionService.getSession().addError(MISC_ERROR, "LEFT:Issue getting databases for dbRegEx");
                 reportWriterService.wrapup();
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             } finally {
                 if (nonNull(conn)) {
@@ -276,6 +283,7 @@ public Future<Boolean> run() {
                 runStatus.setStage(StageEnum.ENVIRONMENT_VARS, CollectionEnum.ERRORED);
                 reportWriterService.wrapup();
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             }
         } else {
@@ -286,6 +294,7 @@ public Future<Boolean> run() {
             log.error("No databases specified OR found if you used dbRegEx");
             runStatus.addError(MISC_ERROR, "No databases specified OR found if you used dbRegEx");
             connectionPoolService.close();
+            runStatus.setProgress(ProgressEnum.FAILED);
             return new AsyncResult<>(Boolean.FALSE);
         }
 
@@ -316,6 +325,7 @@ public Future<Boolean> run() {
                     runStatus.setStage(StageEnum.DATABASES, CollectionEnum.ERRORED);
                     reportWriterService.wrapup();
                     connectionPoolService.close();
+                    runStatus.setProgress(ProgressEnum.FAILED);
                     return new AsyncResult<>(Boolean.FALSE);
                 } catch (RuntimeException rte) {
                     log.error("Runtime Issue", rte);
@@ -324,6 +334,7 @@ public Future<Boolean> run() {
                     runStatus.setStage(StageEnum.DATABASES, CollectionEnum.ERRORED);
                     reportWriterService.wrapup();
                     connectionPoolService.close();
+                    runStatus.setProgress(ProgressEnum.FAILED);
                     return new AsyncResult<>(Boolean.FALSE);
                 }
 
@@ -366,9 +377,10 @@ public Future<Boolean> run() {
             // Failure, report and exit with FALSE
             if (!rtn) {
                 runStatus.setStage(StageEnum.TABLES, CollectionEnum.ERRORED);
-                runStatus.getErrors().set(MessageCode.COLLECTING_TABLES);
+                runStatus.addError(MessageCode.COLLECTING_TABLES);
                 reportWriterService.wrapup();
                 connectionPoolService.close();
+                runStatus.setProgress(ProgressEnum.FAILED);
                 return new AsyncResult<>(Boolean.FALSE);
             }
         } else {
@@ -401,6 +413,9 @@ public Future<Boolean> run() {
 
                 reportWriterService.wrapup();
                 connectionPoolService.close();
+
+                runStatus.setProgress(ProgressEnum.FAILED);
+
                 return new AsyncResult<>(Boolean.FALSE);
             }
         }

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/web/controller/ControllerReferences.java b/src/main/java/com/cloudera/utils/hms/mirror/web/controller/ControllerReferences.java
@@ -73,6 +73,7 @@ public interface ControllerReferences {
     String MESSAGE = "MESSAGE";
     String ARTIFACTS = "ARTIFACTS";
     String DATABASES = "DATABASES";
+    String DBPROPERTIES = "DBPROPERTIES";
     String OTHERS = "OTHERS";
     String DISTCP = "DISTCP";
     String SQL = "SQL";

diff --git a/src/main/java/com/cloudera/utils/hms/mirror/web/controller/DatabaseMVController.java b/src/main/java/com/cloudera/utils/hms/mirror/web/controller/DatabaseMVController.java
@@ -88,5 +88,35 @@ public String deleteDatabase(Model model,
         return "redirect:/config/edit";
     }
 
+    @RequestMapping(value = "/property/add", method = RequestMethod.POST)
+    public String addDatabaseSkipProperty(Model model,
+                              @RequestParam(value = DBPROPERTIES, required = true) String properties) throws SessionException {
+        executeSessionService.closeSession();
+
+        ExecuteSession session = executeSessionService.getSession();
+        HmsMirrorConfig config = session.getConfig();
+
+        String[] props = properties.split(",");
+        for (String property: props) {
+            config.getFilter().addDbPropertySkipItem(property);
+        }
+
+        configService.validate(session, null);
+
+        return "redirect:/config/edit";
+    }
+
+    @RequestMapping(value = "/property/{index}/delete", method = RequestMethod.GET)
+    public String deleteDatabaseSkipProperty(Model model,
+                                 @PathVariable @NotNull Integer index) throws SessionException {
+        executeSessionService.closeSession();
+
+        ExecuteSession session = executeSessionService.getSession();
+        HmsMirrorConfig config = session.getConfig();
+
+        config.getFilter().removeDbPropertySkipItemByIndex(index);
+
+        return "redirect:/config/edit";
+    }
 
 }
diff --git a/src/main/java/com/cloudera/utils/hms/util/DatabaseUtils.java b/src/main/java/com/cloudera/utils/hms/util/DatabaseUtils.java
@@ -18,8 +18,9 @@
 package com.cloudera.utils.hms.util;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
 
 import static java.util.Objects.nonNull;
 
@@ -40,7 +41,7 @@ public static Map<String, String> parametersToMap(String parametersStr) {
         return map;
     }
 
-    public static boolean upsertParameters(Map<String, String> source, Map<String, String> target, List<String> skipList) {
+    public static boolean upsertParameters(Map<String, String> source, Map<String, String> target, Set<String> skipList) {
         boolean changed = false;
         for (Map.Entry<String, String> entry : source.entrySet()) {
             if (skipList.contains(entry.getKey())) {
@@ -54,11 +55,26 @@ public static boolean upsertParameters(Map<String, String> source, Map<String, S
         return changed;
     }
 
-    public static Map<String, String> getParameters(Map<String, String> parameters, List<String> skipList) {
+    public static Map<String, String> getParameters(Map<String, String> parameters, Set<String> staticSkipList, Set<Pattern> dynamicSkipList) {
         Map<String, String> result = new HashMap<>();
         for (Map.Entry<String, String> entry : parameters.entrySet()) {
-            if (!skipList.contains(entry.getKey())) {
-                result.put(entry.getKey(), entry.getValue());
+            String gKey = entry.getKey().trim().replaceAll("^\"|\"$", "");
+            gKey = gKey.replaceAll("^'|'$", "");
+            if (!staticSkipList.contains(gKey)) {
+                // Now check against the user supplied dynamic skip list
+                boolean skip = false;
+                for (Pattern pattern : dynamicSkipList) {
+                    if (pattern.matcher(gKey).matches()) {
+                        skip = true;
+                        break;
+                    }
+                }
+                if (!skip) {
+                    // Strip the quotes from the key and value
+                    String gValue = entry.getValue().trim().replaceAll("^\"|\"$", "");
+                    gValue = gValue.replaceAll("^'|'$", "");
+                    result.put(gKey, gValue);
+                }
             }
         }
         return result;