From c5b3a5d9639a5d232a8ddc7b4009a85857179693 Mon Sep 17 00:00:00 2001 From: Andre Curione Date: Tue, 16 Jan 2018 11:03:47 -0600 Subject: [PATCH 1/2] #13309 - Changes in the Upsert Command for Postgres -9.4 --- .../business/PermissionBitFactoryImpl.java | 3 +- .../com/dotmarketing/common/util/SQLUtil.java | 13 +++- .../dotmarketing/db/DbConnectionFactory.java | 18 +++++ .../db/commands/DatabaseCommand.java | 2 +- .../db/commands/UpsertCommand.java | 70 ++++++++++++++++++- 5 files changed, 98 insertions(+), 8 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java index 4fa3a991d6e9..ea7c77595b60 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/PermissionBitFactoryImpl.java @@ -2331,8 +2331,7 @@ private void upsertPermission(DotConnect dc, String permissionId, Permissionable replacements.setAttribute(QueryReplacements.ID_VALUE, "permission_reference_seq.NEXTVAL"); } - String query = upsertCommand.generateSQLQuery(replacements); - upsertCommand.execute(dc, query, permissionId, newReference.getPermissionId(), type); + upsertCommand.execute(dc, replacements, permissionId, newReference.getPermissionId(), type); } private List filterOnlyNonInheritablePermissions(List permissions, String permissionableId) { diff --git a/dotCMS/src/main/java/com/dotmarketing/common/util/SQLUtil.java b/dotCMS/src/main/java/com/dotmarketing/common/util/SQLUtil.java index 54629eb0ee15..34ca4a64c0c1 100644 --- a/dotCMS/src/main/java/com/dotmarketing/common/util/SQLUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/common/util/SQLUtil.java @@ -41,7 +41,8 @@ public class SQLUtil { public static final String _DESC = " " + DESC; public static final String PARAMETER = "?"; - private static final String SQL_STATE_UNIQUE_CONSTRAINT = "23000"; + private static final String ORACLE_SQL_STATE_UNIQUE_CONSTRAINT = "23000"; + private static final String POSTGRE_SQL_STATE_UNIQUE_CONSTRAINT = "23505"; private static final Set EVIL_SQL_CONDITION_WORDS = ImmutableSet.of( "insert", "delete", "update", "replace", "create", "drop", "alter", "truncate", "declare", "exec", "--", "procedure", "pg_", "lock", @@ -315,9 +316,17 @@ private static boolean isValidSQLCharacter (final char c) { return Character.isLetterOrDigit(c) || '-' == c || '_' == c; } // isValidSQLCharacter. + /** + * Method to check if an exception is a Unique Constraint Exception + * It depends on the database engine. So far only Oracle and postgres has been implemented + * @param ex + * @return + */ public static boolean isUniqueConstraintException (DotDataException ex) { if (ex != null && ex.getCause() instanceof SQLException) { - return ((SQLException) ex.getCause()).getSQLState().equals(SQL_STATE_UNIQUE_CONSTRAINT); + final SQLException sqle = (SQLException) ex.getCause(); + return (DbConnectionFactory.isOracle() && sqle.getSQLState().equals(ORACLE_SQL_STATE_UNIQUE_CONSTRAINT)) || + (DbConnectionFactory.isPostgres() && sqle.getSQLState().equals(POSTGRE_SQL_STATE_UNIQUE_CONSTRAINT)); } return false; } diff --git a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java index f31b5d118874..3991e1927ecf 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java @@ -503,6 +503,24 @@ public static int getDbVersion() { return version; } + /** + * Method to get the Database Full Version + * @return Database Version (Major.Minor) + */ + public static float getDbFullVersion() { + try { + Connection con = getConnection(); + DatabaseMetaData meta = con.getMetaData(); + String version = "%d.%d"; + version = String.format(version, meta.getDatabaseMajorVersion(), meta.getDatabaseMinorVersion()); + return Float.parseFloat(version); + } catch (Exception e) { + Logger.error(DbConnectionFactory.class, + "---------- DBConnectionFactory: Error getting DB Full version " + "---------------", e); + throw new DotRuntimeException(e.toString()); + } + } + /** * Returns the correct MySQL system variable used to define the database * Storage Engine. The old {@code storage_variable} was deprecated as of diff --git a/dotCMS/src/main/java/com/dotmarketing/db/commands/DatabaseCommand.java b/dotCMS/src/main/java/com/dotmarketing/db/commands/DatabaseCommand.java index 634667ab514c..c69fb73bc08f 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/commands/DatabaseCommand.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/commands/DatabaseCommand.java @@ -29,5 +29,5 @@ enum QueryReplacements { * Execute the query * @throws DotDataException */ - void execute (DotConnect dotConnect, String query, Object... parameters) throws DotDataException; + void execute (DotConnect dotConnect, SimpleMapAppContext queryReplacements, Object... parameters) throws DotDataException; } diff --git a/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java b/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java index cbe0ad402038..389c606d983f 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java @@ -3,7 +3,9 @@ import com.dotcms.system.SimpleMapAppContext; import com.dotmarketing.common.db.DotConnect; import com.dotmarketing.common.util.SQLUtil; +import com.dotmarketing.db.DbConnectionFactory; import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Logger; import com.liferay.util.StringUtil; import java.util.ArrayList; import java.util.Arrays; @@ -23,8 +25,9 @@ public abstract class UpsertCommand implements DatabaseCommand { * @param parameters or values * @throws DotDataException */ - public void execute(DotConnect dotConnect, String query, Object... parameters) + public void execute(DotConnect dotConnect, SimpleMapAppContext queryReplacements, Object... parameters) throws DotDataException { + String query = generateSQLQuery(queryReplacements); DotConnect dc = (dotConnect != null) ? dotConnect : new DotConnect(); ArrayList params = new ArrayList<>(); Collections.addAll(params, parameters); //Update parameters @@ -129,7 +132,8 @@ public String generateSQLQuery(SimpleMapAppContext replacements) { } @Override - public void execute(DotConnect dotConnect, String query, Object... parameters) throws DotDataException { + public void execute(DotConnect dotConnect, SimpleMapAppContext queryReplacements, Object... parameters) throws DotDataException { + String query = generateSQLQuery(queryReplacements); DotConnect dc = (dotConnect != null) ? dotConnect : new DotConnect(); dc.executeUpdate(query, parameters); } @@ -143,11 +147,19 @@ final class PostgreUpsertCommand extends UpsertCommand { * ON CONFLICT (conditionalColumn) DO UPDATE SET column1=value1, column2=value2, etc... */ + private static final float POSTGRES_UPSERT_MINIMUM_VERSION = 9.5f; + private static final String POSTGRES_UPSERT_QUERY = "INSERT INTO %s (%s) " + "VALUES (%s) ON CONFLICT (%s) " + "DO UPDATE SET %s"; + private static final String POSTGRES_INSERT_QUERY = + "INSERT INTO %s (%s) VALUES (%s)"; + + private static final String POSTGRES_UPDATE_QUERY = + "UPDATE %s SET %s WHERE %s='%s'"; + @Override public String generateSQLQuery(SimpleMapAppContext replacements) { return @@ -159,6 +171,57 @@ public String generateSQLQuery(SimpleMapAppContext replacements) { getUpdateColumnValuePairs(replacements) ); } + + //If PostgreSQL gets upgraded to 9.5+ (to Support ON CONFLICT) , this override can be removed and let it use the super method execute() + @Override + public void execute(DotConnect dotConnect, SimpleMapAppContext queryReplacements, + Object... parameters) throws DotDataException { + DotConnect dc = (dotConnect != null) ? dotConnect : new DotConnect(); + + float version = DbConnectionFactory.getDbFullVersion(); + if (version >= POSTGRES_UPSERT_MINIMUM_VERSION) { + this.executeUpsert(dc, queryReplacements, parameters); + } else { + this.executeUpdateInsert(dc, queryReplacements, parameters); + } + } + + private void executeUpsert(DotConnect dotConnect, SimpleMapAppContext queryReplacements, + Object... parameters) throws DotDataException { + super.execute(dotConnect, queryReplacements, parameters); + } + + private void executeUpdateInsert(DotConnect dotConnect, SimpleMapAppContext queryReplacements, + Object... parameters) throws DotDataException { + try { + //In Postgre 9.4- Upsert Statement is not supported. Attempt to Insert first. + String insertQuery = + String.format( + POSTGRES_INSERT_QUERY, + queryReplacements.getAttribute(QueryReplacements.TABLE), + getInsertColumnsString(queryReplacements), + getInsertValuesString(queryReplacements) + ); + dotConnect.executeUpdate(insertQuery, false, parameters); + + } catch (DotDataException ex) { + if (SQLUtil.isUniqueConstraintException(ex)) { + //On Unique Constraint exception, attempt to update: + DbConnectionFactory.closeAndCommit(); + String updateQuery = + String.format( + POSTGRES_UPDATE_QUERY, + queryReplacements.getAttribute(QueryReplacements.TABLE), + getUpdateColumnValuePairs(queryReplacements), + queryReplacements.getAttribute(QueryReplacements.CONDITIONAL_COLUMN), + queryReplacements.getAttribute(QueryReplacements.CONDITIONAL_VALUE) + ); + dotConnect.executeUpdate(updateQuery, parameters); + } else { + throw ex; + } + } + } } final class MySQLUpsertCommand extends UpsertCommand { @@ -254,9 +317,10 @@ public String generateSQLQuery(SimpleMapAppContext replacements) { } @Override - public void execute(DotConnect dotConnect, String query, Object... parameters) + public void execute(DotConnect dotConnect, SimpleMapAppContext queryReplacements, Object... parameters) throws DotDataException { + String query = generateSQLQuery(queryReplacements); DotConnect dc = (dotConnect != null) ? dotConnect : new DotConnect(); ArrayList params = new ArrayList<>(); From 1c46907a31d9c260d2bed32a06113d106506dae6 Mon Sep 17 00:00:00 2001 From: Andre Curione Date: Tue, 16 Jan 2018 11:15:34 -0600 Subject: [PATCH 2/2] #13309 - Changes in the Upsert Command for Postgres -9.4 --- .../src/main/java/com/dotmarketing/db/DbConnectionFactory.java | 2 +- .../main/java/com/dotmarketing/db/commands/UpsertCommand.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java index 3991e1927ecf..d13f63678c48 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java @@ -514,7 +514,7 @@ public static float getDbFullVersion() { String version = "%d.%d"; version = String.format(version, meta.getDatabaseMajorVersion(), meta.getDatabaseMinorVersion()); return Float.parseFloat(version); - } catch (Exception e) { + } catch (SQLException e) { Logger.error(DbConnectionFactory.class, "---------- DBConnectionFactory: Error getting DB Full version " + "---------------", e); throw new DotRuntimeException(e.toString()); diff --git a/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java b/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java index 389c606d983f..1c1c7ab17e39 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/commands/UpsertCommand.java @@ -5,7 +5,6 @@ import com.dotmarketing.common.util.SQLUtil; import com.dotmarketing.db.DbConnectionFactory; import com.dotmarketing.exception.DotDataException; -import com.dotmarketing.util.Logger; import com.liferay.util.StringUtil; import java.util.ArrayList; import java.util.Arrays; @@ -147,7 +146,7 @@ final class PostgreUpsertCommand extends UpsertCommand { * ON CONFLICT (conditionalColumn) DO UPDATE SET column1=value1, column2=value2, etc... */ - private static final float POSTGRES_UPSERT_MINIMUM_VERSION = 9.5f; + private static final float POSTGRES_UPSERT_MINIMUM_VERSION = 9.5F; private static final String POSTGRES_UPSERT_QUERY = "INSERT INTO %s (%s) "