diff --git a/pom.xml b/pom.xml index a300d503..2ae4a614 100644 --- a/pom.xml +++ b/pom.xml @@ -106,18 +106,6 @@ - - genepi - genepi-db - 1.2.1 - - - log4j - log4j - - - - genepi genepi-hadoop @@ -138,6 +126,18 @@ + + commons-dbutils + commons-dbutils + 1.7 + + + + commons-dbcp + commons-dbcp + 1.4 + + mysql mysql-connector-java @@ -383,12 +383,6 @@ netty-resolver-dns-native-macos runtime - - - jakarta.annotation - jakarta.annotation-api - compile - ch.qos.logback diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java index 9b4b589a..bc2cc641 100644 --- a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java +++ b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java @@ -18,11 +18,11 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import cloudgene.mapred.core.User; +import cloudgene.mapred.database.util.DatabaseUpdater; import cloudgene.mapred.util.GitHubException; import cloudgene.mapred.util.GitHubUtil; import cloudgene.mapred.util.GitHubUtil.Repository; import cloudgene.mapred.wdl.WdlApp; -import genepi.db.DatabaseUpdater; import genepi.hadoop.S3Util; import genepi.io.FileUtil; import net.lingala.zip4j.ZipFile; diff --git a/src/main/java/cloudgene/mapred/database/CounterDao.java b/src/main/java/cloudgene/mapred/database/CounterDao.java index 06cfa5c9..455e6c67 100644 --- a/src/main/java/cloudgene/mapred/database/CounterDao.java +++ b/src/main/java/cloudgene/mapred/database/CounterDao.java @@ -8,10 +8,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.AbstractJob; -import genepi.db.Database; -import genepi.db.IRowMapMapper; -import genepi.db.JdbcDataAccessObject; public class CounterDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java b/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java index 12a7c510..97aaef62 100644 --- a/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java +++ b/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java @@ -14,8 +14,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import genepi.db.Database; -import genepi.db.JdbcDataAccessObject; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.JdbcDataAccessObject; public class CounterHistoryDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/DownloadDao.java b/src/main/java/cloudgene/mapred/database/DownloadDao.java index 9787aae0..3d511ac6 100644 --- a/src/main/java/cloudgene/mapred/database/DownloadDao.java +++ b/src/main/java/cloudgene/mapred/database/DownloadDao.java @@ -8,11 +8,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.CloudgeneParameterOutput; import cloudgene.mapred.jobs.Download; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class DownloadDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java index 090e8ae3..39132ec3 100644 --- a/src/main/java/cloudgene/mapred/database/JobDao.java +++ b/src/main/java/cloudgene/mapred/database/JobDao.java @@ -10,14 +10,14 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao.UserMapper; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneParameterInput; import cloudgene.mapred.jobs.CloudgeneParameterOutput; import cloudgene.mapred.jobs.CloudgeneStep; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class JobDao extends JdbcDataAccessObject { @@ -267,14 +267,19 @@ public List findAllNotRetiredJobs() { sql.append("select * "); sql.append("from job "); sql.append("join `user` on job.user_id = `user`.id "); - sql.append("where state != ? AND state != ? "); + sql.append("where state not in (?,?,?,?,?,?,?) "); sql.append("order by job.id desc "); List result = new Vector(); - Object[] params = new Object[2]; - params[0] = AbstractJob.STATE_RETIRED; - params[1] = AbstractJob.STATE_DELETED; + Object[] params = new Object[7]; + params[0] = AbstractJob.STATE_WAITING; + params[1] = AbstractJob.STATE_RUNNING; + params[2] = AbstractJob.STATE_EXPORTING; + params[3] = AbstractJob.STATE_RETIRED; + params[4] = AbstractJob.STATE_DELETED; + params[5] = AbstractJob.STATE_RETIRED; + params[6] = AbstractJob.STATE_DELETED; try { diff --git a/src/main/java/cloudgene/mapred/database/MessageDao.java b/src/main/java/cloudgene/mapred/database/MessageDao.java index 67398de8..80ea1c1e 100644 --- a/src/main/java/cloudgene/mapred/database/MessageDao.java +++ b/src/main/java/cloudgene/mapred/database/MessageDao.java @@ -8,11 +8,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class MessageDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java index 101b1b9f..8422d40f 100644 --- a/src/main/java/cloudgene/mapred/database/ParameterDao.java +++ b/src/main/java/cloudgene/mapred/database/ParameterDao.java @@ -8,15 +8,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneParameterInput; import cloudgene.mapred.jobs.CloudgeneParameterOutput; import cloudgene.mapred.jobs.Download; import cloudgene.mapred.wdl.WdlParameterInputType; import cloudgene.mapred.wdl.WdlParameterOutputType; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class ParameterDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/StepDao.java b/src/main/java/cloudgene/mapred/database/StepDao.java index c4c53e45..094318c3 100644 --- a/src/main/java/cloudgene/mapred/database/StepDao.java +++ b/src/main/java/cloudgene/mapred/database/StepDao.java @@ -8,13 +8,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.steps.EmptyStep; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class StepDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/TemplateDao.java b/src/main/java/cloudgene/mapred/database/TemplateDao.java index b0d6ef06..ee679603 100644 --- a/src/main/java/cloudgene/mapred/database/TemplateDao.java +++ b/src/main/java/cloudgene/mapred/database/TemplateDao.java @@ -9,9 +9,9 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.core.Template; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; public class TemplateDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/UserDao.java b/src/main/java/cloudgene/mapred/database/UserDao.java index 549d5aa9..7b94868a 100644 --- a/src/main/java/cloudgene/mapred/database/UserDao.java +++ b/src/main/java/cloudgene/mapred/database/UserDao.java @@ -9,10 +9,10 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.core.User; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IRowMapper; +import cloudgene.mapred.database.util.JdbcDataAccessObject; import cloudgene.mapred.util.PublicUser; -import genepi.db.Database; -import genepi.db.IRowMapper; -import genepi.db.JdbcDataAccessObject; public class UserDao extends JdbcDataAccessObject { diff --git a/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java b/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java index d387793b..d63addb6 100644 --- a/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java +++ b/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java @@ -8,8 +8,8 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; -import genepi.db.Database; -import genepi.db.IUpdateListener; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.IUpdateListener; public class BcryptHashUpdate implements IUpdateListener { diff --git a/src/main/java/cloudgene/mapred/database/util/AbstractDatabaseConnector.java b/src/main/java/cloudgene/mapred/database/util/AbstractDatabaseConnector.java new file mode 100644 index 00000000..2bb0b28c --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/AbstractDatabaseConnector.java @@ -0,0 +1,81 @@ +package cloudgene.mapred.database.util; + +import org.apache.commons.dbcp.BasicDataSource; + +public abstract class AbstractDatabaseConnector implements DatabaseConnector { + + private int maxActive = 10; + + private int maxWait = 10000; + + private boolean defaultAutoCommit = true; + + private boolean testWhileIdle = true; + + private int minEvictableIdleTimeMillis = 1800000; + + private int timeBetweenEvictionRunsMillis = 1800000; + + protected BasicDataSource createDataSource() { + + BasicDataSource dataSource = new BasicDataSource(); + dataSource.setMaxActive(maxActive); + dataSource.setMaxWait(maxWait); + dataSource.setMaxIdle(maxActive); + dataSource.setDefaultAutoCommit(defaultAutoCommit); + dataSource.setTestWhileIdle(testWhileIdle); + dataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + dataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + return dataSource; + + } + + public int getMaxActive() { + return maxActive; + } + + public void setMaxActive(int maxActive) { + this.maxActive = maxActive; + } + + public int getMaxWait() { + return maxWait; + } + + public void setMaxWait(int maxWait) { + this.maxWait = maxWait; + } + + public boolean isDefaultAutoCommit() { + return defaultAutoCommit; + } + + public void setDefaultAutoCommit(boolean defaultAutoCommit) { + this.defaultAutoCommit = defaultAutoCommit; + } + + public boolean isTestWhileIdle() { + return testWhileIdle; + } + + public void setTestWhileIdle(boolean testWhileIdle) { + this.testWhileIdle = testWhileIdle; + } + + public int getMinEvictableIdleTimeMillis() { + return minEvictableIdleTimeMillis; + } + + public void setMinEvictableIdleTimeMillis(int minEvictableIdleTimeMillis) { + this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; + } + + public int getTimeBetweenEvictionRunsMillis() { + return timeBetweenEvictionRunsMillis; + } + + public void setTimeBetweenEvictionRunsMillis(int timeBetweenEvictionRunsMillis) { + this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/Database.java b/src/main/java/cloudgene/mapred/database/util/Database.java new file mode 100644 index 00000000..c7c4b85b --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/Database.java @@ -0,0 +1,129 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Iterator; +import java.util.Vector; + +import org.apache.commons.dbcp.BasicDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Database + * + * @author Lukas Forer + * + */ +public class Database { + + private static final Logger log = LoggerFactory.getLogger(Database.class); + + private DatabaseConnector connector; + + public Database() { + } + + public void connect(DatabaseConnector connector) throws SQLException { + this.connector = connector; + try { + + connector.connect(); + + log.debug("Establish connection successful"); + + fireChangeEvent(DatabaseListener.AFTER_CONNECTION); + } catch (SQLException e) { + log.error("Establish connection failed", e); + throw e; + } + } + + public void disconnect() throws SQLException { + + if (connector != null) { + + if (connector.getDataSource() != null) { + + log.debug("Disconnecting"); + + fireChangeEvent(DatabaseListener.BEFORE_DISCONNECTION); + try { + + connector.disconnect(); + + log.debug("Disconnection successful"); + fireChangeEvent(DatabaseListener.AFTER_DISCONNECTION); + } catch (SQLException e) { + log.error("Disconnection failed", e); + throw e; + } + } + + } + } + + public boolean isConnected() { + if (connector != null) { + return !connector.getDataSource().isClosed(); + } else { + return false; + } + } + + public BasicDataSource getDataSource() { + return connector.getDataSource(); + } + + /* + * ListenerSupport + */ + + private Vector listeners = new Vector(); + + public void addDatabaseListener(DatabaseListener listener) { + if (!listeners.contains(listener)) { + listeners.add(listener); + } + } + + public void removeDatabaseListener(DatabaseListener listener) { + listeners.remove(listener); + } + + private void fireChangeEvent(int event) { + Iterator iter = listeners.iterator(); + while (iter.hasNext()) { + (iter.next()).onDatabaseEvent(event); + } + } + + public void executeSQL(InputStream is) throws SQLException, IOException, + URISyntaxException { + connector.executeSQL(is); + } + + public DatabaseConnector getConnector() { + return connector; + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/DatabaseConnector.java b/src/main/java/cloudgene/mapred/database/util/DatabaseConnector.java new file mode 100644 index 00000000..d95978e5 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/DatabaseConnector.java @@ -0,0 +1,42 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.sql.SQLException; + +import org.apache.commons.dbcp.BasicDataSource; + +public interface DatabaseConnector { + + public void connect() throws SQLException; + + public void disconnect() throws SQLException; + + public BasicDataSource getDataSource(); + + public void executeSQL(InputStream is) throws SQLException, IOException, + URISyntaxException; + + public String getSchema(); + + boolean existsTable(String table) throws SQLException; +} + diff --git a/src/main/java/cloudgene/mapred/database/util/DatabaseConnectorFactory.java b/src/main/java/cloudgene/mapred/database/util/DatabaseConnectorFactory.java index 1bd9a1eb..b49f7cbd 100644 --- a/src/main/java/cloudgene/mapred/database/util/DatabaseConnectorFactory.java +++ b/src/main/java/cloudgene/mapred/database/util/DatabaseConnectorFactory.java @@ -2,9 +2,8 @@ import java.util.Map; -import genepi.db.DatabaseConnector; -import genepi.db.h2.H2Connector; -import genepi.db.mysql.MySqlConnector; +import cloudgene.mapred.database.util.h2.H2Connector; +import cloudgene.mapred.database.util.mysql.MySqlConnector; public class DatabaseConnectorFactory { diff --git a/src/main/java/cloudgene/mapred/database/util/DatabaseListener.java b/src/main/java/cloudgene/mapred/database/util/DatabaseListener.java new file mode 100644 index 00000000..96fabb57 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/DatabaseListener.java @@ -0,0 +1,29 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +public interface DatabaseListener { + + public static final int AFTER_CONNECTION = 1; + public static final int AFTER_DISCONNECTION = 2; + public static final int BEFORE_DISCONNECTION = 3; + public static final int ERROR = 4; + + public void onDatabaseEvent(int event); + +} diff --git a/src/main/java/cloudgene/mapred/database/util/DatabaseUpdater.java b/src/main/java/cloudgene/mapred/database/util/DatabaseUpdater.java new file mode 100644 index 00000000..7d91c48e --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/DatabaseUpdater.java @@ -0,0 +1,380 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import genepi.io.FileUtil; + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DatabaseUpdater { + + protected static final Logger log = LoggerFactory.getLogger(DatabaseUpdater.class); + + private DatabaseConnector connector; + + private Database database; + + private String oldVersion; + + private String currentVersion; + + private String filename; + + private InputStream updateFileAsStream; + + private boolean needUpdate = false; + + private Map listeners = new HashMap(); + + public DatabaseUpdater(Database database, String filename, InputStream updateFileAsStream, String currentVersion) { + + this.filename = filename; + this.database = database; + this.connector = database.getConnector(); + this.updateFileAsStream = updateFileAsStream; + this.currentVersion = currentVersion; + + if (isVersionTableAvailable(database)) { + + oldVersion = readVersionDB(); + log.info("Read current DB version: " + oldVersion); + + // should not happen, since an entry is created when metadata table + // exists + if (oldVersion == null) { + oldVersion = readVersion(filename); + log.info("Read curent version from DB was not successful, read it from file: " + oldVersion); + } + + } else { + // check also file for backwards compatibility + oldVersion = readVersion(filename); + log.info("Read current version from file: " + oldVersion); + } + log.info("Current app version: " + currentVersion); + needUpdate = (compareVersion(currentVersion, oldVersion) > 0); + + } + + public void addUpdate(String version, IUpdateListener listener) { + listeners.put(version, listener); + } + + public boolean updateDB() { + + if (needUpdate()) { + log.info("Database needs update..."); + if (!update()) { + log.error("Updating database failed."); + try { + database.disconnect(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return false; + } + log.info("Update database done."); + } else { + log.info("Database is already up-to-date."); + if (!isVersionTableAvailable(database)) { + writeVersion(currentVersion); + } + } + + String dbVersion = readVersionDB(); + if (!dbVersion.equals(currentVersion)) { + log.error("App version (v" + currentVersion + ") and DB version (v" + dbVersion + + ") does not match. Update Application to latest version."); + return false; + } + + return true; + } + + public boolean update() { + if (needUpdate) { + + log.info("Updating database from " + oldVersion + " to " + currentVersion + "..."); + + try { + readAndPrepareSqlClasspath(updateFileAsStream, oldVersion, currentVersion); + } catch (IOException | URISyntaxException | SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // check if DB version match with Main version + if (isVersionTableAvailable(database)) { + String currentDBVersion = readVersionDB(); + if ((compareVersion(currentVersion, currentDBVersion) > 0)) { + writeVersion(currentVersion); + } + } else { + writeVersion(currentVersion); + } + + log.info("Updating database was successful."); + + } + + return true; + + } + + public boolean needUpdate() { + return needUpdate; + } + + public void writeVersion(String newVersion) { + + try { + + if (!isVersionTableAvailable(database)) { + createVersionTable(database); + } + + Connection connection = connector.getDataSource().getConnection(); + PreparedStatement ps = connection.prepareStatement("INSERT INTO database_versions (version) VALUES (?)"); + ps.setString(1, newVersion); + ps.executeUpdate(); + log.info("Version in DB updated to: " + newVersion); + + if (new File(filename).exists()) { + FileUtil.deleteFile(filename); + log.info("Deleted version.txt on file system."); + } + + connection.close(); + + } catch (SQLException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + } + + public String readVersion(String versionFile) { + + File file = new File(versionFile); + + if (file.exists()) { + + try { + + return readFileAsString(versionFile); + + } catch (Exception e) { + + return "0.0.0"; + + } + + } else { + + return "0.0.0"; + + } + + } + + public String readVersionDB() { + + String version = null; + + try { + Connection connection = connector.getDataSource().getConnection(); + PreparedStatement ps = connection.prepareStatement( + "select version from database_versions where updated_on = (SELECT MAX(updated_on) from database_versions) " + + "order by updated_on, id DESC"); + ResultSet result = ps.executeQuery(); + + if (result.next()) { + version = result.getString(1); + } + + connection.close(); + + } catch (SQLException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + return version; + } + + public static String readFileAsString(String filename) throws java.io.IOException, URISyntaxException { + + InputStream is = new FileInputStream(filename); + + DataInputStream in = new DataInputStream(is); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + StringBuilder builder = new StringBuilder(); + while ((strLine = br.readLine()) != null) { + // builder.append("\n"); + builder.append(strLine); + } + + in.close(); + + return builder.toString(); + } + + public String readAndPrepareSqlClasspath(InputStream filestream, String minVersion, String maxVersion) + throws java.io.IOException, URISyntaxException, SQLException { + + DataInputStream in = new DataInputStream(filestream); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + StringBuilder builder = new StringBuilder(); + boolean reading = false; + String version = null; + + while ((strLine = br.readLine()) != null) { + + if (strLine.startsWith("--")) { + + if (builder.length() > 0) { + executeSQLFile(builder.toString(), version); + builder.setLength(0); + IUpdateListener listener = listeners.get(version); + if (listener != null) { + listener.afterUpdate(database); + } + } + + version = strLine.replace("--", "").trim(); + reading = (compareVersion(version, minVersion) > 0 && compareVersion(version, maxVersion) <= 0); + if (reading) { + log.info("Loading SQL update for version " + version); + IUpdateListener listener = listeners.get(version); + if (listener != null) { + listener.beforeUpdate(database); + } + } + + } + + if (reading) { + builder.append("\n"); + builder.append(strLine); + } + } + + // last block + executeSQLFile(builder.toString(), version); + + in.close(); + + return builder.toString(); + + } + + public void executeSQLFile(String sqlContent, String version) throws SQLException { + + if (sqlContent.length() > 0) { + Connection connection; + connection = connector.getDataSource().getConnection(); + PreparedStatement ps = connection.prepareStatement(sqlContent); + ps.executeUpdate(); + connection.close(); + log.info("DB SQL Update " + version + " finished"); + writeVersion(version); + } + + } + + public static int compareVersion(String version1, String version2) { + + String parts1[] = version1.split("-", 2); + String parts2[] = version2.split("-", 2); + + String tiles1[] = parts1[0].split("\\."); + String tiles2[] = parts2[0].split("\\."); + + for (int i = 0; i < tiles1.length; i++) { + int number1 = Integer.parseInt(tiles1[i].trim()); + int number2 = Integer.parseInt(tiles2[i].trim()); + + if (number1 != number2) { + + return number1 > number2 ? 1 : -1; + + } + + } + + if (parts1.length > 1) { + if (parts2.length > 1) { + return parts1[1].compareTo(parts2[1]); + } else { + return -1; + } + } else { + if (parts2.length > 1) { + return 1; + } + } + + return 0; + + } + + public boolean isVersionTableAvailable(Database database) { + try { + return database.getConnector().existsTable("database_versions"); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return false; + } + } + + public void createVersionTable(Database database) { + try { + Connection connection = connector.getDataSource().getConnection(); + String statement = "create table database_versions ( \r\n" + + " id integer not null auto_increment primary key,\r\n" + + " version varchar(255) not null,\r\n" + + " updated_on timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP \r\n" + ")"; + PreparedStatement ps = connection.prepareStatement(statement); + ps.executeUpdate(); + connection.close(); + log.info("Table database_versions created."); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/Fixtures.java b/src/main/java/cloudgene/mapred/database/util/Fixtures.java index 5ade48c6..c31099c7 100644 --- a/src/main/java/cloudgene/mapred/database/util/Fixtures.java +++ b/src/main/java/cloudgene/mapred/database/util/Fixtures.java @@ -8,7 +8,6 @@ import cloudgene.mapred.database.TemplateDao; import cloudgene.mapred.database.UserDao; import cloudgene.mapred.util.HashUtil; -import genepi.db.Database; public class Fixtures { diff --git a/src/main/java/cloudgene/mapred/database/util/GroupedListHandler.java b/src/main/java/cloudgene/mapred/database/util/GroupedListHandler.java new file mode 100644 index 00000000..cea1f679 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/GroupedListHandler.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Vector; + +import org.apache.commons.dbutils.ResultSetHandler; + +public class GroupedListHandler implements ResultSetHandler { + + private IRowMapMapper mapper; + + public GroupedListHandler(IRowMapMapper rowMapper) { + this.mapper = rowMapper; + } + + public Map> toBeanList(ResultSet rs) + throws SQLException { + Map> result = new HashMap>(); + + int row = 0; + while (rs.next()) { + Object key = mapper.getRowKey(rs, row); + Object value = mapper.getRowValue(rs, row); + if (value != null) { + + List list = result.get(key); + if (list == null) { + list = new Vector(); + result.put(key, list); + } + list.add(value); + } + row++; + } + + return result; + } + + @Override + public Object handle(ResultSet rs) throws SQLException { + return toBeanList(rs); + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/IRowMapMapper.java b/src/main/java/cloudgene/mapred/database/util/IRowMapMapper.java new file mode 100644 index 00000000..13049953 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/IRowMapMapper.java @@ -0,0 +1,29 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public interface IRowMapMapper { + + public Object getRowKey(ResultSet rs, int row) throws SQLException; + + public Object getRowValue(ResultSet rs, int row) throws SQLException; + +} diff --git a/src/main/java/cloudgene/mapred/database/util/IRowMapper.java b/src/main/java/cloudgene/mapred/database/util/IRowMapper.java new file mode 100644 index 00000000..11a8e97b --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/IRowMapper.java @@ -0,0 +1,25 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public interface IRowMapper { + public Object mapRow(ResultSet rs, int row) throws SQLException; +} diff --git a/src/main/java/cloudgene/mapred/database/util/IUpdateListener.java b/src/main/java/cloudgene/mapred/database/util/IUpdateListener.java new file mode 100644 index 00000000..3a8d1ec0 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/IUpdateListener.java @@ -0,0 +1,9 @@ +package cloudgene.mapred.database.util; + +public interface IUpdateListener { + + public void beforeUpdate(Database database); + + public void afterUpdate(Database database); + +} diff --git a/src/main/java/cloudgene/mapred/database/util/JdbcDataAccessObject.java b/src/main/java/cloudgene/mapred/database/util/JdbcDataAccessObject.java new file mode 100644 index 00000000..5be510f2 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/JdbcDataAccessObject.java @@ -0,0 +1,180 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.dbutils.QueryRunner; +import org.apache.commons.dbutils.ResultSetHandler; + +public abstract class JdbcDataAccessObject { + + protected QueryRunner runner = null; + + protected Database database; + + public JdbcDataAccessObject(Database database) { + this.database = database; + runner = new QueryRunner(database.getDataSource()); + } + + /* + * protected Connection getConnection() { return database.getConnection(); } + */ + + public Object queryForObject(String sql, Object[] params, IRowMapper mapper) + throws SQLException { + return runner.query(sql, new ObjectHandler(mapper), params); + } + + @SuppressWarnings("rawtypes") + public List query(String sql, Object[] params, IRowMapper mapper) + throws SQLException { + return (List) runner.query(sql, new ListHandler(mapper), params); + } + + public Object queryForObject(String sql, IRowMapper mapper) + throws SQLException { + return runner.query(sql, new ObjectHandler(mapper)); + } + + @SuppressWarnings("rawtypes") + public Map queryForMap(String sql, IRowMapMapper mapper) + throws SQLException { + return (Map) runner.query(sql, new MapHandler(mapper)); + } + + @SuppressWarnings("rawtypes") + public Map queryForMap(String sql, Object[] params, IRowMapMapper mapper) + throws SQLException { + return (Map) runner.query(sql, new MapHandler(mapper), params); + } + + @SuppressWarnings("rawtypes") + public Map queryForGroupedList(String sql, Object[] params, + IRowMapMapper mapper) throws SQLException { + return (Map) runner.query(sql, new GroupedListHandler(mapper), params); + } + + @SuppressWarnings("rawtypes") + public Map queryForGroupedList(String sql, IRowMapMapper mapper) + throws SQLException { + return (Map) runner.query(sql, new GroupedListHandler(mapper)); + } + + @SuppressWarnings("rawtypes") + public List query(String sql, IRowMapper mapper) throws SQLException { + return (List) runner.query(sql, new ListHandler(mapper)); + } + + public int update(String sql, Object[] params) throws SQLException { + return runner.update(sql.toString(), params); + } + + public int update(String sql) throws SQLException { + return runner.update(sql.toString()); + } + + public int insert(String sql, Object[] params) throws SQLException { + + Connection connection = database.getDataSource().getConnection(); + + try { + PreparedStatement statement = connection.prepareStatement(sql, PreparedStatement.RETURN_GENERATED_KEYS); + + runner.fillStatement(statement, params); + + statement.executeUpdate(); + + ResultSet rs = statement.getGeneratedKeys(); + rs.beforeFirst(); + rs.next(); + int id = rs.getInt(1); + connection.close(); + return id; + } catch (Exception e) { + throw e; + } finally { + connection.close(); + } + } + + + public int[] batch(String sql, Object[][] params) throws SQLException { + + return runner.batch(sql.toString(), params); + } + + // DBUtils 1.6 method + public List batchGeneratedKeys(String sql, Object[][] params) + throws SQLException { + + ResultSetHandler> handler = new ResultSetHandler>() { + + public List handle(ResultSet rs) throws SQLException { + List identifiers = new ArrayList(); + + while (rs.next()) { + identifiers.add(rs.getInt(1)); + } + + return identifiers; + } + }; + + return runner.insertBatch(sql, handler, params); + } + + public boolean callProcedure(String sql, Object[] params) throws SQLException { + + Connection connection = null; + + try { + connection = database.getDataSource().getConnection(); + + CallableStatement cstmt = connection.prepareCall(sql); + runner.fillStatement(cstmt, params); + boolean state = cstmt.execute(); + cstmt.close(); + connection.close(); + return state; + } catch (SQLException e) { + throw e; + } finally { + connection.close(); + } + } + + + public static class IntegerMapper implements IRowMapper { + + @Override + public Object mapRow(ResultSet rs, int row) throws SQLException { + return rs.getInt(1); + } + + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/ListHandler.java b/src/main/java/cloudgene/mapred/database/util/ListHandler.java new file mode 100644 index 00000000..3be71165 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/ListHandler.java @@ -0,0 +1,55 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Vector; + +import org.apache.commons.dbutils.ResultSetHandler; + +public class ListHandler implements ResultSetHandler { + + private IRowMapper mapper; + + public ListHandler(IRowMapper rowMapper) { + this.mapper = rowMapper; + } + + public List toBeanList(ResultSet rs) throws SQLException { + List result = new Vector(); + + int row = 0; + while (rs.next()) { + Object value = mapper.mapRow(rs, row); + if (value != null) { + result.add(value); + } + row++; + } + + return result; + } + + @Override + public Object handle(ResultSet rs) throws SQLException { + return toBeanList(rs); + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/MapHandler.java b/src/main/java/cloudgene/mapred/database/util/MapHandler.java new file mode 100644 index 00000000..c515da36 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/MapHandler.java @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.dbutils.ResultSetHandler; + +public class MapHandler implements ResultSetHandler { + + private IRowMapMapper mapper; + + public MapHandler(IRowMapMapper rowMapper) { + this.mapper = rowMapper; + } + + public Map toBeanList(ResultSet rs) throws SQLException { + Map result = new HashMap(); + + int row = 0; + while (rs.next()) { + Object key = mapper.getRowKey(rs, row); + Object value = mapper.getRowValue(rs, row); + if (value != null) { + result.put(key, value); + } + row++; + } + + return result; + } + + @Override + public Object handle(ResultSet rs) throws SQLException { + return toBeanList(rs); + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/ObjectHandler.java b/src/main/java/cloudgene/mapred/database/util/ObjectHandler.java new file mode 100644 index 00000000..5da60aef --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/ObjectHandler.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.apache.commons.dbutils.ResultSetHandler; + +public class ObjectHandler implements ResultSetHandler { + + private IRowMapper mapper; + + public ObjectHandler(IRowMapper rowMapper) { + this.mapper = rowMapper; + } + + public Object toBean(ResultSet rs) throws SQLException { + + if (!rs.next()) { + return null; + } else { + return mapper.mapRow(rs, 0); + } + } + + @Override + public Object handle(ResultSet rs) throws SQLException { + return toBean(rs); + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/h2/H2Connector.java b/src/main/java/cloudgene/mapred/database/util/h2/H2Connector.java new file mode 100644 index 00000000..b607196c --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/h2/H2Connector.java @@ -0,0 +1,173 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util.h2; + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbutils.DbUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cloudgene.mapred.database.util.DatabaseConnector; +import genepi.io.FileUtil; + +public class H2Connector implements DatabaseConnector { + + protected static final Logger log = LoggerFactory.getLogger(H2Connector.class); + + private BasicDataSource dataSource; + + private String path; + private String user; + private String password; + private boolean multiuser = false; + + public H2Connector(String path, String user, String password, boolean multiuser) { + this.path = path; + this.user = user; + this.password = password; + this.multiuser = multiuser; + } + + public boolean createBackup(String folder) { + + File file = new File(path + ".h2.db"); + File file2 = new File(path + ".mv.db"); + + boolean exists = file.exists() || file2.exists(); + + if (exists) { + + FileUtil.copyDirectory(file.getParent(), folder); + + } + + log.info("Created backup file " + folder); + + return true; + + } + + public void connect() throws SQLException { + + log.debug("Establishing connection to " + user + "@" + path); + + if (DbUtils.loadDriver("org.h2.Driver")) { + try { + dataSource = new BasicDataSource(); + + dataSource.setDriverClassName("org.h2.Driver"); + + String newPath = path; + if (!path.startsWith("/")) { + newPath = "./" + path; + } else { + newPath = path; + } + + if (multiuser) { + dataSource.setUrl("jdbc:h2:" + newPath + ";AUTO_SERVER=TRUE;MODE=MySQL"); + } else { + dataSource.setUrl("jdbc:h2:" + newPath + ";MODE=MySQL"); + } + dataSource.setUsername(user); + dataSource.setPassword(password); + // dataSource.setMaxActive(1000); + // dataSource.setMaxWait(10000); + dataSource.setMaxIdle(10000); + dataSource.setDefaultAutoCommit(true); + + } catch (Exception e) { + e.printStackTrace(); + } + + } else { + log.error("H2 Driver Class not found"); + } + + } + + public void disconnect() throws SQLException { + dataSource.close(); + + } + + public void executeSQL(InputStream is) throws SQLException, IOException, URISyntaxException { + + String sqlContent = readFileAsString(is); + if (!sqlContent.isEmpty()) { + Connection connection = dataSource.getConnection(); + PreparedStatement ps = connection.prepareStatement(sqlContent); + ps.executeUpdate(); + connection.close(); + } + } + + public static String readFileAsString(InputStream is) throws java.io.IOException, URISyntaxException { + + DataInputStream in = new DataInputStream(is); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + StringBuilder builder = new StringBuilder(); + while ((strLine = br.readLine()) != null) { + builder.append("\n"); + builder.append(strLine); + } + + in.close(); + + return builder.toString(); + } + + public BasicDataSource getDataSource() { + return dataSource; + } + + @Override + public String getSchema() { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean existsTable(String table) throws SQLException { + Connection connection = dataSource.getConnection(); + DatabaseMetaData meta = connection.getMetaData(); + ResultSet res = meta.getTables(null, null, table.toUpperCase(), new String[] { "TABLE" }); + boolean exists = res.next(); + res.close(); + connection.close(); + if (!exists) { + log.warn("Table '" + table + "' not found'"); + } + return exists; + } + +} diff --git a/src/main/java/cloudgene/mapred/database/util/mysql/MySqlConnector.java b/src/main/java/cloudgene/mapred/database/util/mysql/MySqlConnector.java new file mode 100644 index 00000000..d1713e29 --- /dev/null +++ b/src/main/java/cloudgene/mapred/database/util/mysql/MySqlConnector.java @@ -0,0 +1,141 @@ +/******************************************************************************* + * Copyright (C) 2009-2016 Lukas Forer and Sebastian Schönherr + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + ******************************************************************************/ + +package cloudgene.mapred.database.util.mysql; + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbutils.DbUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cloudgene.mapred.database.util.AbstractDatabaseConnector; + +public class MySqlConnector extends AbstractDatabaseConnector { + + private static final Logger log = LoggerFactory.getLogger(MySqlConnector.class); + + private BasicDataSource dataSource; + + private String host; + private String port; + private String database; + private String user; + private String password; + + public MySqlConnector(String host, String port, String database, String user, String password) { + this.host = host; + this.port = port; + this.database = database; + this.user = user; + this.password = password; + } + + @Override + public void connect() throws SQLException { + + log.debug("Establishing connection to " + user + "@" + host + ":" + port); + + if (DbUtils.loadDriver("com.mysql.jdbc.Driver")) { + try { + dataSource = createDataSource(); + + dataSource.setDriverClassName("com.mysql.jdbc.Driver"); + dataSource.setUrl("jdbc:mysql://" + host + "/" + database + + "?autoReconnect=true&allowMultiQueries=true&rewriteBatchedStatements=true"); + dataSource.setUsername(user); + dataSource.setPassword(password); + + } catch (Exception e) { + e.printStackTrace(); + } + + } else { + System.out.println("MySQL Driver class not found."); + } + + } + + @Override + public void disconnect() throws SQLException { + dataSource.close(); + + } + + @Override + public BasicDataSource getDataSource() { + return dataSource; + } + + public void executeSQL(InputStream is) throws SQLException, IOException, URISyntaxException { + + String sqlContent = readFileAsString(is); + if (!sqlContent.isEmpty()) { + Connection connection = dataSource.getConnection(); + PreparedStatement ps = connection.prepareStatement(sqlContent); + ps.executeUpdate(); + connection.close(); + } + } + + public static String readFileAsString(InputStream is) throws java.io.IOException, URISyntaxException { + + DataInputStream in = new DataInputStream(is); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + String strLine; + StringBuilder builder = new StringBuilder(); + while ((strLine = br.readLine()) != null) { + builder.append("\n"); + builder.append(strLine); + } + + in.close(); + + return builder.toString(); + } + + @Override + public String getSchema() { + return database; + } + + @Override + public boolean existsTable(String table) throws SQLException { + Connection connection = dataSource.getConnection(); + DatabaseMetaData meta = connection.getMetaData(); + ResultSet res = meta.getTables(null, null, table, new String[] { "TABLE" }); + boolean exists = res.next(); + res.close(); + connection.close(); + if (!exists) { + log.warn("Table '" + table + "' not found'"); + } + return exists; + } + +} diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java index b4701232..5506598c 100644 --- a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java @@ -12,7 +12,7 @@ import cloudgene.mapred.database.MessageDao; import cloudgene.mapred.database.ParameterDao; import cloudgene.mapred.database.StepDao; -import genepi.db.Database; +import cloudgene.mapred.database.util.Database; public class PersistentWorkflowEngine extends WorkflowEngine { diff --git a/src/main/java/cloudgene/mapred/server/Application.java b/src/main/java/cloudgene/mapred/server/Application.java index 16b7b95e..f465f019 100644 --- a/src/main/java/cloudgene/mapred/server/Application.java +++ b/src/main/java/cloudgene/mapred/server/Application.java @@ -11,16 +11,16 @@ import cloudgene.mapred.database.TemplateDao; import cloudgene.mapred.database.updates.BcryptHashUpdate; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.DatabaseConnector; import cloudgene.mapred.database.util.DatabaseConnectorFactory; +import cloudgene.mapred.database.util.DatabaseUpdater; import cloudgene.mapred.database.util.Fixtures; import cloudgene.mapred.jobs.PersistentWorkflowEngine; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.plugins.PluginManager; import cloudgene.mapred.util.Config; import cloudgene.mapred.util.Settings; -import genepi.db.Database; -import genepi.db.DatabaseConnector; -import genepi.db.DatabaseUpdater; import genepi.io.FileUtil; import io.micronaut.context.annotation.Context; @@ -63,11 +63,11 @@ public Application() throws Exception { database.connect(connector); - log.info("Establish connection successful"); + log.info("Establish connection to database successful"); } catch (SQLException e) { - log.error("Establish connection failed", e); + log.error("Establish connection to database failed", e); System.exit(1); } diff --git a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java index 9e8a26b3..0158ef98 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java @@ -7,13 +7,13 @@ import cloudgene.mapred.core.Template; import cloudgene.mapred.database.JobDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.db.Database; import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import jakarta.inject.Inject; diff --git a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java index dc1518de..073175a0 100644 --- a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java +++ b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java @@ -6,11 +6,11 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.CounterHistoryDao; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.MailUtil; -import genepi.db.Database; import genepi.hadoop.HadoopUtil; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Inject; diff --git a/src/main/java/cloudgene/mapred/util/PublicUser.java b/src/main/java/cloudgene/mapred/util/PublicUser.java index 50609aa8..d227b00d 100644 --- a/src/main/java/cloudgene/mapred/util/PublicUser.java +++ b/src/main/java/cloudgene/mapred/util/PublicUser.java @@ -2,7 +2,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; -import genepi.db.Database; +import cloudgene.mapred.database.util.Database; public class PublicUser { diff --git a/src/test/java/cloudgene/mapred/TestApplication.java b/src/test/java/cloudgene/mapred/TestApplication.java index 0c4e3ce1..5fceeab9 100644 --- a/src/test/java/cloudgene/mapred/TestApplication.java +++ b/src/test/java/cloudgene/mapred/TestApplication.java @@ -11,12 +11,12 @@ import cloudgene.mapred.apps.Application; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.Config; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.Settings; import cloudgene.mapred.util.TestCluster; import cloudgene.mapred.util.TestMailServer; -import genepi.db.Database; import genepi.io.FileUtil; import io.micronaut.context.annotation.Context; diff --git a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java index 5381beb1..c77a3f7a 100644 --- a/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/admin/ChangeGroupTest.java @@ -12,10 +12,10 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.CloudgeneClientRestAssured; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.TestMailServer; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import io.restassured.http.Header; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java index 914d146b..8c7f6456 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ActivateUserTest.java @@ -18,8 +18,8 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.TestMailServer; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import jakarta.inject.Inject; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java index 8512305a..3d9a8927 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ApiTokensTest.java @@ -18,10 +18,10 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.util.CloudgeneClientRestAssured; import cloudgene.mapred.util.HashUtil; -import genepi.db.Database; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java index 0b3fda0d..227d3d5e 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/LoginUserTest.java @@ -17,9 +17,9 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.server.auth.DatabaseAuthenticationProvider; import cloudgene.mapred.util.HashUtil; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import io.restassured.response.Response; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java index 87232703..e77ab4ae 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/ResetPasswordTest.java @@ -18,9 +18,9 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.TestMailServer; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import jakarta.inject.Inject; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java index 5b8c13dc..80577af9 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/UpdatePasswordTest.java @@ -13,9 +13,9 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.TestMailServer; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import jakarta.inject.Inject; diff --git a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java index 9e1d82e7..62be9981 100644 --- a/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/users/UserProfileTest.java @@ -15,9 +15,9 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.util.CloudgeneClientRestAssured; import cloudgene.mapred.util.HashUtil; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; import io.restassured.http.Header; diff --git a/src/test/java/cloudgene/mapred/database/DatabaseUpdateTest.java b/src/test/java/cloudgene/mapred/database/DatabaseUpdateTest.java index ce8ae004..2b67f45d 100644 --- a/src/test/java/cloudgene/mapred/database/DatabaseUpdateTest.java +++ b/src/test/java/cloudgene/mapred/database/DatabaseUpdateTest.java @@ -7,9 +7,9 @@ import org.junit.jupiter.api.Test; import cloudgene.mapred.TestApplication; +import cloudgene.mapred.database.util.Database; +import cloudgene.mapred.database.util.DatabaseUpdater; import cloudgene.mapred.server.Application; -import genepi.db.Database; -import genepi.db.DatabaseUpdater; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; diff --git a/src/test/java/cloudgene/mapred/database/JobDaoTest.java b/src/test/java/cloudgene/mapred/database/JobDaoTest.java index 2e69e0f9..93df5d59 100644 --- a/src/test/java/cloudgene/mapred/database/JobDaoTest.java +++ b/src/test/java/cloudgene/mapred/database/JobDaoTest.java @@ -9,9 +9,9 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; +import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneJob; -import genepi.db.Database; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject;