From 71918f07024b3e91fc96545eac5ace7563a34285 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Tue, 24 Dec 2024 19:41:28 +0300 Subject: [PATCH 1/4] Store LDAPv3 database in SQL JDBC database --- .github/workflows/build.yml | 33 +- README.md | 2 +- .../config/JDBCBackendConfiguration.xml | 74 +++ opendj-server-legacy/pom.xml | 31 +- .../resource/schema/02-config.ldif | 8 +- .../opends/server/backends/jdbc/Backend.java | 31 + .../opends/server/backends/jdbc/Storage.java | 545 ++++++++++++++++++ .../server/backends/jdbc/package-info.java | 18 + .../backends/jdbc/EncryptedTestCase.java | 81 +++ .../opends/server/backends/jdbc/TestCase.java | 79 +++ .../server/replication/GenerationIdTest.java | 3 +- pom.xml | 16 - 12 files changed, 874 insertions(+), 47 deletions(-) create mode 100644 opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/JDBCBackendConfiguration.xml create mode 100644 opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Backend.java create mode 100644 opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java create mode 100644 opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/package-info.java create mode 100644 opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/EncryptedTestCase.java create mode 100644 opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c88f41497d..9ddccc777b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,12 +41,7 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-m2-repository-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2-repository - - name: Run docker cassandra - if: runner.os == 'Linux' - run: | - docker run --rm -it -d -p 9042:9042 --name cassandra cassandra - timeout 5m bash -c 'until docker logs cassandra | grep -q "Created default superuser role"; do sleep 5; done' - - name: Set Integration Test Environment + - name: Set Integration Test Environment id: failsafe if: runner.os != 'Windows' run: | @@ -71,17 +66,37 @@ jobs: opendj-server-legacy/target/package/opendj/bin/start-ds opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example2,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000 opendj-server-legacy/target/package/opendj/bin/stop-ds - rm -rf opendj-server-legacy/target/package/opendj/config opendj-server-legacy/target/package/opendj/db opendj-server-legacy/target/package/opendj/changelogDb opendj-server-legacy/target/package/opendj/logs + rm -rf opendj-server-legacy/target/package/opendj/{config,db,changelogDb,logs} - name: Test LDAP in Cassandra if: runner.os == 'Linux' run: | + docker run --rm -it -d -p 9042:9042 --name cassandra cassandra + timeout 5m bash -c 'until docker logs cassandra | grep -q "Created default superuser role"; do sleep 5; done' export OPENDJ_JAVA_ARGS="-server -Xmx512m -Ddatastax-java-driver.basic.contact-points.0=localhost:9042 -Ddatastax-java-driver.basic.load-balancing-policy.local-datacenter=datacenter1" opendj-server-legacy/target/package/opendj/setup --backendType cas -h localhost -p 1389 --ldapsPort 1636 --adminConnectorPort 4444 --enableStartTLS --generateSelfSignedCertificate --rootUserDN "cn=Directory Manager" --rootUserPassword password --baseDN dc=example,dc=com --sampleData 5000 --cli --acceptLicense --no-prompt + opendj-server-legacy/target/package/opendj/bin/dsconfig create-backend -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type cas --set base-dn:dc=example,dc=com --set db-directory:keyspace_name --set enabled:true --no-prompt --trustAll + opendj-server-legacy/target/package/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example,dc=com opendj-server-legacy/target/package/opendj/config/MakeLDIF/example.template + opendj-server-legacy/target/package/opendj/bin/import-ldif --ldifFile /tmp/test.ldif --backendID=userRoot -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --trustAll opendj-server-legacy/target/package/opendj/bin/status --bindDN "cn=Directory Manager" --bindPassword password opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 - opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 5000 + opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000 + opendj-server-legacy/target/package/opendj/bin/stop-ds + rm -rf opendj-server-legacy/target/package/opendj/{config,db,changelogDb,logs} + - name: Test LDAP in Postgres + if: runner.os == 'Linux' + run: | + docker run --rm -it -d -p 5432:5432 -e POSTGRES_DB=database_name -e POSTGRES_PASSWORD=password --name postgres postgres + timeout 5m bash -c 'until docker logs postgres | grep -q "database system is ready to accept connections"; do sleep 5; done' + export OPENDJ_JAVA_ARGS="-server -Xmx512m" + opendj-server-legacy/target/package/opendj/setup -h localhost -p 1389 --ldapsPort 1636 --adminConnectorPort 4444 --enableStartTLS --generateSelfSignedCertificate --rootUserDN "cn=Directory Manager" --rootUserPassword password --cli --acceptLicense --no-prompt + opendj-server-legacy/target/package/opendj/bin/dsconfig create-backend -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type jdbc --set base-dn:dc=example,dc=com --set db-directory:jdbc:postgresql://localhost:5432/database_name?user=postgres\&password=password --set enabled:true --no-prompt --trustAll + opendj-server-legacy/target/package/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example,dc=com opendj-server-legacy/target/package/opendj/config/MakeLDIF/example.template + opendj-server-legacy/target/package/opendj/bin/import-ldif --ldifFile /tmp/test.ldif --backendID=userRoot -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --trustAll + opendj-server-legacy/target/package/opendj/bin/status --bindDN "cn=Directory Manager" --bindPassword password + opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 + opendj-server-legacy/target/package/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "ou=people,dc=example,dc=com" --searchScope sub "(uid=user.*)" dn | grep ^dn: | wc -l | grep -q 10000 opendj-server-legacy/target/package/opendj/bin/stop-ds - rm -rf opendj-server-legacy/target/package/opendj/config opendj-server-legacy/target/package/opendj/db opendj-server-legacy/target/package/opendj/changelogDb opendj-server-legacy/target/package/opendj/logs + rm -rf opendj-server-legacy/target/package/opendj/{config,db,changelogDb,logs} - name: Test on Windows if: runner.os == 'Windows' run: | diff --git a/README.md b/README.md index 410a6cd68a..d017252184 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ OpenDJ is an [LDAPv3](http://tools.ietf.org/html/rfc4510) compliant directory service, which has been developed for the Java platform, providing a high performance, highly available, and secure store for the identities managed by your organization. Its easy installation process, combined with the power of the Java platform makes OpenDJ -the simplest, fastest directory to deploy and manage and allow [store LDAPv3 database in Cassandra/Scylla cluster](https://github.com/OpenIdentityPlatform/OpenDJ/wiki/How-To#store-ldap-catalog-data-in-cassandra-nosql-cluster). +the simplest, fastest directory to deploy and manage and allow store LDAPv3 database in [SQL JDBC database](https://github.com/OpenIdentityPlatform/OpenDJ/wiki/How-To#store-ldap-catalog-data-in-jdbc-databse) or [NoSQL Cassandra/Scylla cluster](https://github.com/OpenIdentityPlatform/OpenDJ/wiki/How-To#store-ldap-catalog-data-in-cassandra-nosql-cluster). An open source, lightweight, embeddable directory that can easily share real-time customer, device, and user identity data across enterprise, cloud, social, and mobile environments. * Massive data scale and high availability provide developers with ultra-lightweight ways to access identity data diff --git a/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/JDBCBackendConfiguration.xml b/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/JDBCBackendConfiguration.xml new file mode 100644 index 0000000000..4f832d27b0 --- /dev/null +++ b/opendj-maven-plugin/src/main/resources/config/xml/org/forgerock/opendj/server/config/JDBCBackendConfiguration.xml @@ -0,0 +1,74 @@ + + + + + A stores application data in JDBC source. + + + It is the traditional "directory server" backend and is similar to + the backends provided by the Sun Java System Directory Server. The + + stores the entries in an encoded form and also provides indexes that + can be used to quickly locate target entries based on different + kinds of criteria. + + + + ds-cfg-jdbc-backend + ds-cfg-pluggable-backend + + + + + + + org.opends.server.backends.jdbc.Backend + + + + + + + Default this to the db/backend-id + + Specifies the connection string + + + jdbc:postgresql://localhost/test + + + + + + + jdbc:postgresql://localhost/test + + + + + + + + ds-cfg-db-directory + + + + diff --git a/opendj-server-legacy/pom.xml b/opendj-server-legacy/pom.xml index 97008f70a5..8d75ab8352 100644 --- a/opendj-server-legacy/pom.xml +++ b/opendj-server-legacy/pom.xml @@ -258,6 +258,11 @@ + + org.postgresql + postgresql + 42.7.4 + org.testng testng @@ -267,7 +272,13 @@ org.testcontainers cassandra - 1.19.0 + 1.20.4 + test + + + org.testcontainers + postgresql + 1.20.4 test @@ -1082,24 +1093,6 @@ - - - org.openidentityplatform.maven.plugins - javadoc-updater-maven-plugin - 1.0.0 - - - site - - fixjavadoc - - - ${project.build.directory}/site/javadoc - - - - - org.apache.maven.plugins diff --git a/opendj-server-legacy/resource/schema/02-config.ldif b/opendj-server-legacy/resource/schema/02-config.ldif index 71bb719136..697a1aed15 100644 --- a/opendj-server-legacy/resource/schema/02-config.ldif +++ b/opendj-server-legacy/resource/schema/02-config.ldif @@ -15,7 +15,7 @@ # Portions Copyright 2011 profiq, s.r.o. # Portions Copyright 2012 Manuel Gaupp # Portions copyright 2015 Edan Idzerda -# Portions copyright 2023 3A Systems LLC +# Portions copyright 2023-2024 3A Systems LLC # This file contains the attribute type and objectclass definitions for use # with the Directory Server configuration. @@ -6013,6 +6013,12 @@ objectClasses: ( 1.3.6.1.4.1.60142.2.1.2.1 STRUCTURAL MUST ds-cfg-db-directory X-ORIGIN 'OpenDJ Directory Server' ) +objectClasses: ( 1.3.6.1.4.1.60142.2.1.2.2 + NAME 'ds-cfg-jdbc-backend' + SUP ds-cfg-pluggable-backend + STRUCTURAL + MUST ds-cfg-db-directory + X-ORIGIN 'OpenDJ Directory Server' ) objectClasses: ( 1.3.6.1.4.1.36733.2.1.2.27 NAME 'ds-task-reset-change-number' SUP ds-task diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Backend.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Backend.java new file mode 100644 index 0000000000..ce3f3f4678 --- /dev/null +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Backend.java @@ -0,0 +1,31 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions Copyright [year] [name of copyright owner]". + * + * Copyright 2024 3A Systems, LLC. + */ +package org.opends.server.backends.jdbc; + +import org.forgerock.opendj.config.server.ConfigException; +import org.forgerock.opendj.server.config.server.JDBCBackendCfg; +import org.opends.server.backends.pluggable.BackendImpl; +import org.opends.server.core.ServerContext; + +public class Backend extends BackendImpl{ + + @Override + protected Storage configureStorage(JDBCBackendCfg cfg, ServerContext serverContext) throws ConfigException + { + return new Storage(cfg, serverContext); + } + +} diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java new file mode 100644 index 0000000000..c91fffa790 --- /dev/null +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java @@ -0,0 +1,545 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions Copyright [year] [name of copyright owner]". + * + * Copyright 2024 3A Systems, LLC. + */ +package org.opends.server.backends.jdbc; + +import org.forgerock.i18n.LocalizableMessage; +import org.forgerock.i18n.slf4j.LocalizedLogger; +import org.forgerock.opendj.config.server.ConfigChangeResult; +import org.forgerock.opendj.config.server.ConfigException; +import org.forgerock.opendj.config.server.ConfigurationChangeListener; +import org.forgerock.opendj.ldap.ByteSequence; +import org.forgerock.opendj.ldap.ByteString; +import org.forgerock.opendj.server.config.server.JDBCBackendCfg; +import org.opends.server.backends.pluggable.spi.*; +import org.opends.server.core.ServerContext; +import org.opends.server.types.BackupConfig; +import org.opends.server.types.BackupDirectory; +import org.opends.server.types.DirectoryException; +import org.opends.server.types.RestoreConfig; +import org.opends.server.util.BackupManager; + +import java.sql.*; +import java.util.*; + +import static org.opends.server.backends.pluggable.spi.StorageUtils.addErrorMessage; +import static org.opends.server.util.StaticUtils.stackTraceToSingleLineString; + +public class Storage implements org.opends.server.backends.pluggable.spi.Storage, ConfigurationChangeListener{ + + private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); + + private JDBCBackendCfg config; + + public Storage(JDBCBackendCfg cfg, ServerContext serverContext) { + this.config = cfg; + cfg.addJDBCChangeListener(this); + } + + //config + @Override + public boolean isConfigurationChangeAcceptable(JDBCBackendCfg configuration,List unacceptableReasons) { + return true; + } + + @Override + public ConfigChangeResult applyConfigurationChange(JDBCBackendCfg cfg) { + final ConfigChangeResult ccr = new ConfigChangeResult(); + try + { + this.config = cfg; + } + catch (Exception e) + { + addErrorMessage(ccr, LocalizableMessage.raw(stackTraceToSingleLineString(e))); + } + return ccr; + } + + ResultSet executeResultSet(PreparedStatement statement) throws SQLException { + if (logger.isTraceEnabled()) { + logger.trace(LocalizableMessage.raw("jdbc: %s",statement)); + } + return statement.executeQuery(); + } + + boolean execute(PreparedStatement statement) throws SQLException { + if (logger.isTraceEnabled()) { + logger.trace(LocalizableMessage.raw("jdbc: %s",statement)); + } + return statement.execute(); + } + + Connection con; + @Override + public void open(AccessMode accessMode) throws Exception { + con=DriverManager.getConnection(config.getDBDirectory()); + con.setAutoCommit(false); + con.setReadOnly(!AccessMode.READ_WRITE.equals(accessMode)); + storageStatus = StorageStatus.working(); + } + + private StorageStatus storageStatus = StorageStatus.lockedDown(LocalizableMessage.raw("closed")); + @Override + public StorageStatus getStorageStatus() { + return storageStatus; + } + + @Override + public void close() { + storageStatus = StorageStatus.lockedDown(LocalizableMessage.raw("closed")); + try { + if (con != null && !con.isClosed()) { + con.close(); + } + } catch (SQLException e) { + logger.error(LocalizableMessage.raw("close(): %s",e),e); + } + con=null; + } + + String getTableName(TreeName treeName) { + return "\"OpenDJ"+treeName.toString()+"\""; + } + + @Override + public void removeStorageFiles() throws StorageRuntimeException { + final boolean isOpen=getStorageStatus().isWorking(); + if (!isOpen) { + try { + open(AccessMode.READ_WRITE); + }catch (Exception e) { + throw new StorageRuntimeException(e); + } + } + try { + for (TreeName treeName : listTrees()) { + final PreparedStatement statement=con.prepareStatement("drop table "+getTableName(treeName)); + execute(statement); + } + }catch (Throwable e) { + throw new StorageRuntimeException(e); + } + if (!isOpen) { + close(); + } + } + + //operation + @Override + public T read(ReadOperation readOperation) throws Exception { + return readOperation.run(new ReadableTransactionImpl()); + } + + @Override + public void write(WriteOperation writeOperation) throws Exception { + try { + writeOperation.run(new WriteableTransactionTransactionImpl()); + con.commit(); + } catch (Exception e) { + try { + con.rollback(); + } catch (SQLException ex) {} + throw e; + } + } + + private class ReadableTransactionImpl implements ReadableTransaction { + @Override + public ByteString read(TreeName treeName, ByteSequence key) { + try { + final PreparedStatement statement=con.prepareStatement("select v from "+getTableName(treeName)+" where k=?"); + statement.setBytes(1,key.toByteArray()); + try(ResultSet rc=executeResultSet(statement)) { + return rc.next() ? ByteString.wrap(rc.getBytes("v")) : null; + } + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public Cursor openCursor(TreeName treeName) { + return new CursorImpl(treeName); + } + + @Override + public long getRecordCount(TreeName treeName) { + try { + final PreparedStatement statement=con.prepareStatement("select count(*) from "+getTableName(treeName)); + try(ResultSet rc=executeResultSet(statement)) { + return rc.next() ? rc.getLong(1) : 0; + } + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + private final class WriteableTransactionTransactionImpl extends ReadableTransactionImpl implements WriteableTransaction { + + public WriteableTransactionTransactionImpl() { + super(); + try { + if (con.isReadOnly()) { + throw new ReadOnlyStorageException(); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void openTree(TreeName treeName, boolean createOnDemand) { + if (createOnDemand) { + try { + final PreparedStatement statement=con.prepareStatement("create table if not exists "+getTableName(treeName)+" (k bytea primary key,v bytea)"); + execute(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + + public void clearTree(TreeName treeName) { + try { + final PreparedStatement statement=con.prepareStatement("truncate table "+getTableName(treeName)); + execute(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void deleteTree(TreeName treeName) { + try { + final PreparedStatement statement=con.prepareStatement("drop table "+getTableName(treeName)); + execute(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void put(TreeName treeName, ByteSequence key, ByteSequence value) { + try { + delete(treeName,key); + final PreparedStatement statement=con.prepareStatement("insert into "+getTableName(treeName)+" (k,v) values(?,?) "); + statement.setBytes(1,key.toByteArray()); + statement.setBytes(2,value.toByteArray()); + execute(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f) { + final ByteString oldValue=read(treeName,key); + final ByteSequence newValue=f.computeNewValue(oldValue); + if (Objects.equals(newValue, oldValue)) + { + return false; + } + if (newValue == null) + { + delete(treeName, key); + return true; + } + put(treeName,key,newValue); + return true; + } + + @Override + public boolean delete(TreeName treeName, ByteSequence key) { + try { + final PreparedStatement statement=con.prepareStatement("delete from "+getTableName(treeName)+" where k=?"); + statement.setBytes(1,key.toByteArray()); + execute(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + return true; + } + } + + private final class CursorImpl implements Cursor { + final TreeName treeName; + //final WriteableTransactionTransactionImpl tx; + + ResultSet rc; + + public CursorImpl(TreeName treeName) { + this.treeName=treeName; + //this.tx=tx; + try { + final PreparedStatement statement=con.prepareStatement("select k,v from "+getTableName(treeName)+" order by k", + ResultSet.TYPE_SCROLL_SENSITIVE, + ResultSet.CONCUR_UPDATABLE); + rc=executeResultSet(statement); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean next() { + try { + return rc.next(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isDefined() { + try{ + return rc.getRow()>0; + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public ByteString getKey() throws NoSuchElementException { + if (!isDefined()) { + throw new NoSuchElementException(); + } + try{ + return ByteString.wrap(rc.getBytes("k")); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public ByteString getValue() throws NoSuchElementException { + if (!isDefined()) { + throw new NoSuchElementException(); + } + try{ + return ByteString.wrap(rc.getBytes("v")); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void delete() throws NoSuchElementException, UnsupportedOperationException { + if (!isDefined()) { + throw new NoSuchElementException(); + } + try{ + rc.deleteRow(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + if (rc!=null) { + try{ + rc.close(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + rc = null; + } + } + + + @Override + public boolean positionToKeyOrNext(ByteSequence key) { + if (!isDefined() || key.compareTo(getKey())<0) { //restart iterator + try{ + rc.first(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + try{ + if (!isDefined()){ + return false; + } + do { + if (key.compareTo(getKey())<=0) { + return true; + } + }while(rc.next()); + }catch (SQLException e) { + throw new RuntimeException(e); + } + return false; + } + + @Override + public boolean positionToKey(ByteSequence key) { + if (!isDefined() || key.compareTo(getKey())<0) { //restart iterator + try{ + rc.first(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + if (!isDefined()){ + return false; + } + if (isDefined() && key.compareTo(getKey())==0) { + return true; + } + try{ + do { + if (key.compareTo(getKey())==0) { + return true; + } + }while(rc.next()); + }catch (SQLException e) { + throw new RuntimeException(e); + } + return false; + } + + + @Override + public boolean positionToLastKey() { + try{ + return rc.last(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean positionToIndex(int index) { + try{ + rc.first(); + }catch (SQLException e) { + throw new RuntimeException(e); + } + if (!isDefined()){ + return false; + } + int ct=0; + try{ + do { + if (ct==index) { + return true; + } + ct++; + }while(rc.next()); + }catch (SQLException e) { + throw new RuntimeException(e); + } + return false; + } + } + + @Override + public Set listTrees() { + final Set res=new HashSet<>(); + try(ResultSet rs = con.getMetaData().getTables(null, null, "OpenDJ%", new String[]{"TABLE"})) { + while (rs.next()) { + res.add(TreeName.valueOf(rs.getString("TABLE_NAME").substring(6))); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return res; + } + + + private final class ImporterImpl implements Importer { + final WriteableTransactionTransactionImpl tx; + + final Boolean isOpen; + + public ImporterImpl() { + isOpen=getStorageStatus().isWorking(); + if (!isOpen) { + try { + open(AccessMode.READ_WRITE); + }catch (Exception e) { + throw new StorageRuntimeException(e); + } + } + tx=new WriteableTransactionTransactionImpl(); + } + + @Override + public void close() { + try { + con.commit(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + if (!isOpen) { + Storage.this.close(); + } + } + + @Override + public void clearTree(TreeName name) { + tx.clearTree(name); + } + + @Override + public void put(TreeName treeName, ByteSequence key, ByteSequence value) { + tx.put(treeName, key, value); + } + + @Override + public ByteString read(TreeName treeName, ByteSequence key) { + return tx.read(treeName, key); + } + + @Override + public SequentialCursor openCursor(TreeName treeName) { + return tx.openCursor(treeName); + } + } + + //import + @Override + public Importer startImport() throws ConfigException, StorageRuntimeException { + return new ImporterImpl(); + } + + //backup + @Override + public boolean supportsBackupAndRestore() { + return false; + } + + @Override + public void createBackup(BackupConfig backupConfig) throws DirectoryException + { + // TODO backup over snapshot or SQL export + //new BackupManager(config.getBackendId()).createBackup(this, backupConfig); + } + + @Override + public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException + { + new BackupManager(config.getBackendId()).removeBackup(backupDirectory, backupID); + } + + @Override + public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException + { + // TODO restore over snapshot or SQL export + //new BackupManager(config.getBackendId()).restoreBackup(this, restoreConfig); + } + +} diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/package-info.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/package-info.java new file mode 100644 index 0000000000..d29cef4160 --- /dev/null +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/package-info.java @@ -0,0 +1,18 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions Copyright [year] [name of copyright owner]". + * + * Copyright 2024 3A Systems LLC. + */ +@org.opends.server.types.PublicAPI( + stability=org.opends.server.types.StabilityLevel.PRIVATE) +package org.opends.server.backends.jdbc; diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/EncryptedTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/EncryptedTestCase.java new file mode 100644 index 0000000000..6b0279c475 --- /dev/null +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/EncryptedTestCase.java @@ -0,0 +1,81 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions Copyright [year] [name of copyright owner]". + * + * Copyright 2023-2024 3A Systems, LLC. + */ +package org.opends.server.backends.jdbc; + +import org.forgerock.opendj.server.config.server.JDBCBackendCfg; +import org.opends.server.backends.pluggable.PluggableBackendImplTestCase; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DriverManager; + +import static org.forgerock.opendj.config.ConfigurationMock.mockCfg; +import static org.mockito.Mockito.when; + +@Test +public class EncryptedTestCase extends PluggableBackendImplTestCase { + PostgreSQLContainer container; + + @BeforeClass + @Override + public void setUp() throws Exception { + if(DockerClientFactory.instance().isDockerAvailable()) { + container = new PostgreSQLContainer<>("postgres:latest") + .withExposedPorts(5432) + .withUsername("postgres") + .withPassword("password") + .withDatabaseName("database_name"); + container.start(); + } + try(Connection con= DriverManager.getConnection(createBackendCfg().getDBDirectory())){ + } catch (Exception e) { + throw new SkipException("run before test: docker run --rm -it -p 5432:5432 -e POSTGRES_DB=database_name -e POSTGRES_PASSWORD=password --name postgres postgres"); + } + super.setUp(); + } + + @Override + protected Backend createBackend() { + return new Backend(); + } + + @Override + protected JDBCBackendCfg createBackendCfg() { + JDBCBackendCfg backendCfg = mockCfg(JDBCBackendCfg.class); + when(backendCfg.getBackendId()).thenReturn("EncPsqlTestCase"+System.currentTimeMillis()); + when(backendCfg.getDBDirectory()).thenReturn("jdbc:postgresql://localhost:"+ ((container==null)?"5432":container.getMappedPort(5432))+"/database_name?user=postgres&password=password"); + + when(backendCfg.isConfidentialityEnabled()).thenReturn(true); + when(backendCfg.getCipherKeyLength()).thenReturn(128); + when(backendCfg.getCipherTransformation()).thenReturn("AES/CBC/PKCS5Padding"); + return backendCfg; + } + + + @AfterClass + @Override + public void cleanUp() throws Exception { + super.cleanUp(); + if(container != null) { + container.close(); + } + } +} diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java new file mode 100644 index 0000000000..0b01008e7b --- /dev/null +++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java @@ -0,0 +1,79 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions Copyright [year] [name of copyright owner]". + * + * Copyright 2024 3A Systems, LLC. + */ +package org.opends.server.backends.jdbc; + +import org.forgerock.opendj.server.config.server.JDBCBackendCfg; +import org.opends.server.backends.pluggable.PluggableBackendImplTestCase; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DriverManager; + +import static org.forgerock.opendj.config.ConfigurationMock.mockCfg; +import static org.mockito.Mockito.when; + +//docker run --rm -it -p 5432:5432 -e POSTGRES_PASSWORD=password --name postgres postgres + +@Test +public class TestCase extends PluggableBackendImplTestCase { + + PostgreSQLContainer container; + + @BeforeClass + @Override + public void setUp() throws Exception { + if(DockerClientFactory.instance().isDockerAvailable()) { + container = new PostgreSQLContainer<>("postgres:latest") + .withExposedPorts(5432) + .withUsername("postgres") + .withPassword("password") + .withDatabaseName("database_name"); + container.start(); + } + try(Connection con= DriverManager.getConnection(createBackendCfg().getDBDirectory())){ + } catch (Exception e) { + throw new SkipException("run before test: docker run --rm -it -p 5432:5432 -e POSTGRES_DB=database_name -e POSTGRES_PASSWORD=password --name postgres postgres"); + } + super.setUp(); + } + + @Override + protected Backend createBackend() { + return new Backend(); + } + + @Override + protected JDBCBackendCfg createBackendCfg() { + JDBCBackendCfg backendCfg = mockCfg(JDBCBackendCfg.class); + when(backendCfg.getBackendId()).thenReturn("PsqlTestCase"); + when(backendCfg.getDBDirectory()).thenReturn("jdbc:postgresql://localhost:"+ ((container==null)?"5432":container.getMappedPort(5432))+"/database_name?user=postgres&password=password"); + return backendCfg; + } + + @AfterClass + @Override + public void cleanUp() throws Exception { + super.cleanUp(); + if(container != null) { + container.close(); + } + } +} diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/GenerationIdTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/GenerationIdTest.java index b5fa90941d..2d1c828124 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/replication/GenerationIdTest.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/GenerationIdTest.java @@ -13,6 +13,7 @@ * * Copyright 2006-2010 Sun Microsystems, Inc. * Portions Copyright 2011-2016 ForgeRock AS. + * Portions Copyright 2023-2024 3A Systems, LLC. */ package org.opends.server.replication; @@ -990,7 +991,7 @@ public void testMultiRS() throws Exception private void waitForStableGenerationId(final long expectedGenId) throws Exception { TestTimer timer = new TestTimer.Builder() - .maxSleep(20, SECONDS) + .maxSleep(30, SECONDS) .sleepTimes(100, MILLISECONDS) .toTimer(); timer.repeatUntilSuccess(new CallableVoid() diff --git a/pom.xml b/pom.xml index a7b14efe89..b529fee480 100644 --- a/pom.xml +++ b/pom.xml @@ -286,22 +286,6 @@ ${project.groupId}.${project.artifactId} - org.openidentityplatform.maven.plugins - javadoc-updater-maven-plugin - 1.0.0 - - - site - - fixjavadoc - - - ${project.reporting.outputDirectory} - - - - - org.apache.maven.plugins maven-source-plugin From 8593015f4b6908acaa6bccc482aa064cfdd65097 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Tue, 24 Dec 2024 21:11:01 +0300 Subject: [PATCH 2/4] Store LDAPv3 database in SQL JDBC database --- .../src/main/java/org/opends/server/backends/jdbc/Storage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java index c91fffa790..67b6959fce 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/Storage.java @@ -519,7 +519,7 @@ public Importer startImport() throws ConfigException, StorageRuntimeException { //backup @Override public boolean supportsBackupAndRestore() { - return false; + return true; } @Override From 0e113a47252d4024be279e92bd252be3728d2974 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Wed, 25 Dec 2024 11:52:40 +0300 Subject: [PATCH 3/4] FIX mono install --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ddccc777b..1f9e8caa67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,8 +24,9 @@ jobs: sudo wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/ubuntu/dists/$(lsb_release -c -s)/winehq-$(lsb_release -c -s).sources sudo apt-get update sudo apt install --install-recommends winehq-stable || sudo apt install --install-recommends winehq-staging - sudo mkdir -p /opt/wine/mono && sudo wget "https://dl.winehq.org/wine/wine-mono/8.0.0/wine-mono-8.0.0-x86.tar.xz" -P /opt/wine/mono && sudo tar -xf /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz -C /opt/wine/mono && sudo rm /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz wine --version + sudo wget "https://dl.winehq.org/wine/wine-mono/8.0.0/wine-mono-8.0.0-x86.msi" -O /tmp/wine-mono.msi + wine msiexec /i /tmp/wine-mono.msi - uses: actions/checkout@v4 with: fetch-depth: 0 From f82050c908b1129c6f731f23753028efcd42b07d Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Wed, 25 Dec 2024 13:18:55 +0300 Subject: [PATCH 4/4] FIX mono install --- .github/workflows/build.yml | 4 ++-- .github/workflows/deploy.yml | 3 ++- .github/workflows/release.yml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f9e8caa67..dec517870c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: sudo apt-get update sudo apt install --install-recommends winehq-stable || sudo apt install --install-recommends winehq-staging wine --version - sudo wget "https://dl.winehq.org/wine/wine-mono/8.0.0/wine-mono-8.0.0-x86.msi" -O /tmp/wine-mono.msi + version="9.4.0"; sudo wget "https://dl.winehq.org/wine/wine-mono/$version/wine-mono-$version-x86.msi" -O /tmp/wine-mono.msi wine msiexec /i /tmp/wine-mono.msi - uses: actions/checkout@v4 with: @@ -74,7 +74,7 @@ jobs: docker run --rm -it -d -p 9042:9042 --name cassandra cassandra timeout 5m bash -c 'until docker logs cassandra | grep -q "Created default superuser role"; do sleep 5; done' export OPENDJ_JAVA_ARGS="-server -Xmx512m -Ddatastax-java-driver.basic.contact-points.0=localhost:9042 -Ddatastax-java-driver.basic.load-balancing-policy.local-datacenter=datacenter1" - opendj-server-legacy/target/package/opendj/setup --backendType cas -h localhost -p 1389 --ldapsPort 1636 --adminConnectorPort 4444 --enableStartTLS --generateSelfSignedCertificate --rootUserDN "cn=Directory Manager" --rootUserPassword password --baseDN dc=example,dc=com --sampleData 5000 --cli --acceptLicense --no-prompt + opendj-server-legacy/target/package/opendj/setup -h localhost -p 1389 --ldapsPort 1636 --adminConnectorPort 4444 --enableStartTLS --generateSelfSignedCertificate --rootUserDN "cn=Directory Manager" --rootUserPassword password --cli --acceptLicense --no-prompt opendj-server-legacy/target/package/opendj/bin/dsconfig create-backend -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --backend-name=userRoot --type cas --set base-dn:dc=example,dc=com --set db-directory:keyspace_name --set enabled:true --no-prompt --trustAll opendj-server-legacy/target/package/opendj/bin/makeldif -o /tmp/test.ldif -c suffix=dc=example,dc=com opendj-server-legacy/target/package/opendj/config/MakeLDIF/example.template opendj-server-legacy/target/package/opendj/bin/import-ldif --ldifFile /tmp/test.ldif --backendID=userRoot -h localhost -p 4444 --bindDN "cn=Directory Manager" --bindPassword password --trustAll diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6abb4e47e1..d5caea8f1e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,8 +24,9 @@ jobs: sudo wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/ubuntu/dists/$(lsb_release -c -s)/winehq-$(lsb_release -c -s).sources sudo apt-get update sudo apt install --install-recommends winehq-stable || sudo apt install --install-recommends winehq-staging - sudo mkdir -p /opt/wine/mono && sudo wget "https://dl.winehq.org/wine/wine-mono/8.0.0/wine-mono-8.0.0-x86.tar.xz" -P /opt/wine/mono && sudo tar -xf /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz -C /opt/wine/mono && sudo rm /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz wine --version + version="9.4.0"; sudo wget "https://dl.winehq.org/wine/wine-mono/$version/wine-mono-$version-x86.msi" -O /tmp/wine-mono.msi + wine msiexec /i /tmp/wine-mono.msi - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 034e8ae9c9..f1d9e94f80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,8 +28,9 @@ jobs: sudo wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/ubuntu/dists/$(lsb_release -c -s)/winehq-$(lsb_release -c -s).sources sudo apt-get update sudo apt install --install-recommends winehq-stable || sudo apt install --install-recommends winehq-staging - sudo mkdir -p /opt/wine/mono && sudo wget "https://dl.winehq.org/wine/wine-mono/8.0.0/wine-mono-8.0.0-x86.tar.xz" -P /opt/wine/mono && sudo tar -xf /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz -C /opt/wine/mono && sudo rm /opt/wine/mono/wine-mono-8.0.0-x86.tar.xz wine --version + version="9.4.0"; sudo wget "https://dl.winehq.org/wine/wine-mono/$version/wine-mono-$version-x86.msi" -O /tmp/wine-mono.msi + wine msiexec /i /tmp/wine-mono.msi - uses: actions/checkout@v4 with: fetch-depth: 0