[v3.x] Migrate genepi-libs and optimize job list (#101)

* Move all database classes from genepi-db to cloudgene * Migrate logging of db-utils to slf4j * Remove unused dependency to jakarta.annotation-api * Optimize job list
genepi · May 24, 2023 · fa925c3 · fa925c3
1 parent 3b874d1
commit fa925c3
Show file tree

Hide file tree

Showing 44 changed files with 1,513 additions and 75 deletions.
diff --git a/pom.xml b/pom.xml
@@ -106,18 +106,6 @@
 			</exclusions>
 		</dependency>
 
-		<dependency>
-			<groupId>genepi</groupId>
-			<artifactId>genepi-db</artifactId>
-			<version>1.2.1</version>
-			<exclusions>
-				<exclusion>
-					<groupId>log4j</groupId>
-					<artifactId>log4j</artifactId>
-				</exclusion>
-			</exclusions>
-		</dependency>
-
 		<dependency>
 			<groupId>genepi</groupId>
 			<artifactId>genepi-hadoop</artifactId>
@@ -138,6 +126,18 @@
 			</exclusions>
 		</dependency>
 
+		<dependency>
+			<groupId>commons-dbutils</groupId>
+			<artifactId>commons-dbutils</artifactId>
+			<version>1.7</version>
+		</dependency>
+
+		<dependency>
+			<groupId>commons-dbcp</groupId>
+			<artifactId>commons-dbcp</artifactId>
+			<version>1.4</version>
+		</dependency>
+
 		<dependency>
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>
@@ -383,12 +383,6 @@
 			<artifactId>netty-resolver-dns-native-macos</artifactId>
 			<scope>runtime</scope>
 		</dependency>
-
-		<dependency>
-			<groupId>jakarta.annotation</groupId>
-			<artifactId>jakarta.annotation-api</artifactId>
-			<scope>compile</scope>
-		</dependency>
 
 		<dependency>
 			<groupId>ch.qos.logback</groupId>

diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java
@@ -18,11 +18,11 @@
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 
 import cloudgene.mapred.core.User;
+import cloudgene.mapred.database.util.DatabaseUpdater;
 import cloudgene.mapred.util.GitHubException;
 import cloudgene.mapred.util.GitHubUtil;
 import cloudgene.mapred.util.GitHubUtil.Repository;
 import cloudgene.mapred.wdl.WdlApp;
-import genepi.db.DatabaseUpdater;
 import genepi.hadoop.S3Util;
 import genepi.io.FileUtil;
 import net.lingala.zip4j.ZipFile;

diff --git a/src/main/java/cloudgene/mapred/database/CounterDao.java b/src/main/java/cloudgene/mapred/database/CounterDao.java
@@ -8,10 +8,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.AbstractJob;
-import genepi.db.Database;
-import genepi.db.IRowMapMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class CounterDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java b/src/main/java/cloudgene/mapred/database/CounterHistoryDao.java
@@ -14,8 +14,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import genepi.db.Database;
-import genepi.db.JdbcDataAccessObject;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 
 public class CounterHistoryDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/DownloadDao.java b/src/main/java/cloudgene/mapred/database/DownloadDao.java
@@ -8,11 +8,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.Download;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class DownloadDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java
@@ -10,14 +10,14 @@
 
 import cloudgene.mapred.core.User;
 import cloudgene.mapred.database.UserDao.UserMapper;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.CloudgeneJob;
 import cloudgene.mapred.jobs.CloudgeneParameterInput;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.CloudgeneStep;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class JobDao extends JdbcDataAccessObject {
 
@@ -267,14 +267,19 @@ public List<AbstractJob> findAllNotRetiredJobs() {
 		sql.append("select * ");
 		sql.append("from job ");
 		sql.append("join `user` on job.user_id = `user`.id ");
-		sql.append("where state != ? AND state != ? ");
+		sql.append("where state  not in (?,?,?,?,?,?,?) ");
 		sql.append("order by job.id desc ");
 
 		List<AbstractJob> result = new Vector<AbstractJob>();
 
-		Object[] params = new Object[2];
-		params[0] = AbstractJob.STATE_RETIRED;
-		params[1] = AbstractJob.STATE_DELETED;
+		Object[] params = new Object[7];
+		params[0] = AbstractJob.STATE_WAITING;
+		params[1] = AbstractJob.STATE_RUNNING;
+		params[2] = AbstractJob.STATE_EXPORTING;
+		params[3] = AbstractJob.STATE_RETIRED;
+		params[4] = AbstractJob.STATE_DELETED;		
+		params[5] = AbstractJob.STATE_RETIRED;
+		params[6] = AbstractJob.STATE_DELETED;
 
 		try {
 

diff --git a/src/main/java/cloudgene/mapred/database/MessageDao.java b/src/main/java/cloudgene/mapred/database/MessageDao.java
@@ -8,11 +8,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneStep;
 import cloudgene.mapred.jobs.Message;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class MessageDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java
@@ -8,15 +8,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.AbstractJob;
 import cloudgene.mapred.jobs.CloudgeneParameterInput;
 import cloudgene.mapred.jobs.CloudgeneParameterOutput;
 import cloudgene.mapred.jobs.Download;
 import cloudgene.mapred.wdl.WdlParameterInputType;
 import cloudgene.mapred.wdl.WdlParameterOutputType;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class ParameterDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/StepDao.java b/src/main/java/cloudgene/mapred/database/StepDao.java
@@ -8,13 +8,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.jobs.CloudgeneJob;
 import cloudgene.mapred.jobs.CloudgeneStep;
 import cloudgene.mapred.jobs.Message;
 import cloudgene.mapred.steps.EmptyStep;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class StepDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/TemplateDao.java b/src/main/java/cloudgene/mapred/database/TemplateDao.java
@@ -9,9 +9,9 @@
 import org.slf4j.LoggerFactory;
 
 import cloudgene.mapred.core.Template;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 
 public class TemplateDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/UserDao.java b/src/main/java/cloudgene/mapred/database/UserDao.java
@@ -9,10 +9,10 @@
 import org.slf4j.LoggerFactory;
 
 import cloudgene.mapred.core.User;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IRowMapper;
+import cloudgene.mapred.database.util.JdbcDataAccessObject;
 import cloudgene.mapred.util.PublicUser;
-import genepi.db.Database;
-import genepi.db.IRowMapper;
-import genepi.db.JdbcDataAccessObject;
 
 public class UserDao extends JdbcDataAccessObject {
 

diff --git a/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java b/src/main/java/cloudgene/mapred/database/updates/BcryptHashUpdate.java
@@ -8,8 +8,8 @@
 
 import cloudgene.mapred.core.User;
 import cloudgene.mapred.database.UserDao;
-import genepi.db.Database;
-import genepi.db.IUpdateListener;
+import cloudgene.mapred.database.util.Database;
+import cloudgene.mapred.database.util.IUpdateListener;
 
 public class BcryptHashUpdate implements IUpdateListener {
 

diff --git a/src/main/java/cloudgene/mapred/database/util/AbstractDatabaseConnector.java b/src/main/java/cloudgene/mapred/database/util/AbstractDatabaseConnector.java
@@ -0,0 +1,81 @@
+package cloudgene.mapred.database.util;
+
+import org.apache.commons.dbcp.BasicDataSource;
+
+public abstract class AbstractDatabaseConnector implements DatabaseConnector {
+
+	private int maxActive = 10;
+
+	private int maxWait = 10000;
+
+	private boolean defaultAutoCommit = true;
+
+	private boolean testWhileIdle = true;
+
+	private int minEvictableIdleTimeMillis = 1800000;
+
+	private int timeBetweenEvictionRunsMillis = 1800000;
+
+	protected BasicDataSource createDataSource() {
+
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setMaxActive(maxActive);
+		dataSource.setMaxWait(maxWait);
+		dataSource.setMaxIdle(maxActive);
+		dataSource.setDefaultAutoCommit(defaultAutoCommit);
+		dataSource.setTestWhileIdle(testWhileIdle);
+		dataSource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
+		dataSource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
+		return dataSource;
+
+	}
+
+	public int getMaxActive() {
+		return maxActive;
+	}
+
+	public void setMaxActive(int maxActive) {
+		this.maxActive = maxActive;
+	}
+
+	public int getMaxWait() {
+		return maxWait;
+	}
+
+	public void setMaxWait(int maxWait) {
+		this.maxWait = maxWait;
+	}
+
+	public boolean isDefaultAutoCommit() {
+		return defaultAutoCommit;
+	}
+
+	public void setDefaultAutoCommit(boolean defaultAutoCommit) {
+		this.defaultAutoCommit = defaultAutoCommit;
+	}
+
+	public boolean isTestWhileIdle() {
+		return testWhileIdle;
+	}
+
+	public void setTestWhileIdle(boolean testWhileIdle) {
+		this.testWhileIdle = testWhileIdle;
+	}
+
+	public int getMinEvictableIdleTimeMillis() {
+		return minEvictableIdleTimeMillis;
+	}
+
+	public void setMinEvictableIdleTimeMillis(int minEvictableIdleTimeMillis) {
+		this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis;
+	}
+
+	public int getTimeBetweenEvictionRunsMillis() {
+		return timeBetweenEvictionRunsMillis;
+	}
+
+	public void setTimeBetweenEvictionRunsMillis(int timeBetweenEvictionRunsMillis) {
+		this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis;
+	}
+
+}