Skip to content

Commit

Permalink
Fix Intersmash#37: fix postgres database provisioner
Browse files Browse the repository at this point in the history
  • Loading branch information
@tommaso-borgato
tommaso-borgato committed May 30, 2023
1 parent d0393fb commit d3a6a4d
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 41 deletions.
70 changes: 40 additions & 30 deletions ...boss/intersmash/testsuite/provision/openshift/KeycloakQuarkusOperatorProvisionerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,14 @@
import org.keycloak.k8s.v2alpha1.keycloakspec.db.UsernameSecret;
import org.slf4j.event.Level;

import cz.xtf.core.config.OpenShiftConfig;
import cz.xtf.core.openshift.OpenShiftWaiters;
import cz.xtf.core.openshift.OpenShifts;
import cz.xtf.core.waiting.SimpleWaiter;
import cz.xtf.junit5.annotations.CleanBeforeAll;
import io.fabric8.kubernetes.api.model.DeletionPropagation;
import io.fabric8.kubernetes.api.model.KubernetesResourceList;
import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation;
import io.fabric8.kubernetes.client.dsl.Resource;
import lombok.extern.slf4j.Slf4j;

/**
Expand Down Expand Up @@ -142,23 +144,26 @@ public void cleanup() throws IOException {
@AfterAll
public static void removeOperatorGroup() {
OpenShifts.adminBinary().execute("delete", "operatorgroup", "--all");
KEYCLOAK_OPERATOR_PROVISIONER.dismiss();
if (!Objects.isNull(KEYCLOAK_OPERATOR_PROVISIONER))
KEYCLOAK_OPERATOR_PROVISIONER.dismiss();
POSTGRESQL_IMAGE_PROVISIONER.undeploy();
POSTGRESQL_IMAGE_PROVISIONER.postUndeploy();
}

@AfterEach
public void customResourcesCleanup() {

// delete keycloaks
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().list().getItems().stream()
.map(resource -> resource.getMetadata().getName()).forEach(name -> KEYCLOAK_OPERATOR_PROVISIONER
.keycloakClient().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete());
// delete realms
KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().list().getItems().stream()
.map(resource -> resource.getMetadata().getName()).forEach(name -> KEYCLOAK_OPERATOR_PROVISIONER
.keycloakRealmImportClient().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND)
.delete());
if (!Objects.isNull(KEYCLOAK_OPERATOR_PROVISIONER)) {
// delete keycloaks
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().list().getItems().stream()
.map(resource -> resource.getMetadata().getName()).forEach(name -> KEYCLOAK_OPERATOR_PROVISIONER
.keycloakClient().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete());
// delete realms
KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().list().getItems()
.stream()
.map(resource -> resource.getMetadata().getName()).forEach(name -> KEYCLOAK_OPERATOR_PROVISIONER
.keycloakRealmImportClient().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND)
.delete());
}
}

/**
Expand Down Expand Up @@ -202,7 +207,8 @@ public void exampleSso() {

verifyKeycloak(keycloak, true);
} finally {
KEYCLOAK_OPERATOR_PROVISIONER.unsubscribe();
if (!Objects.isNull(KEYCLOAK_OPERATOR_PROVISIONER))
KEYCLOAK_OPERATOR_PROVISIONER.unsubscribe();
}
}

Expand Down Expand Up @@ -257,6 +263,7 @@ public void exampleSsoWithDatabase() {
passwordSecret.setName(POSTGRESQL_IMAGE_PROVISIONER.getSecretName());
passwordSecret.setKey(PostgreSQLImageOpenShiftProvisioner.POSTGRESQL_PASSWORD_KEY);
db.setPasswordSecret(passwordSecret);
db.setDatabase(POSTGRESQL_IMAGE_PROVISIONER.getApplication().getDbName());
spec.setDb(db);

realmName = "saml-basic-auth";
Expand Down Expand Up @@ -288,7 +295,8 @@ public void exampleSsoWithDatabase() {

verifyKeycloak(keycloak, realmImport, true);
} finally {
KEYCLOAK_OPERATOR_PROVISIONER.unsubscribe();
if (!Objects.isNull(KEYCLOAK_OPERATOR_PROVISIONER))
KEYCLOAK_OPERATOR_PROVISIONER.unsubscribe();
POSTGRESQL_IMAGE_PROVISIONER.undeploy();
POSTGRESQL_IMAGE_PROVISIONER.postUndeploy();
}
Expand All @@ -299,61 +307,63 @@ private void verifyKeycloak(Keycloak keycloak, boolean waitForPods) {
}

private void verifyKeycloak(Keycloak keycloak, KeycloakRealmImport realmImport, boolean waitForPods) {
NonNamespaceOperation<Keycloak, KubernetesResourceList<Keycloak>, Resource<Keycloak>> keycloakClient = KEYCLOAK_OPERATOR_PROVISIONER
.keycloakClient();
NonNamespaceOperation<KeycloakRealmImport, KubernetesResourceList<KeycloakRealmImport>, Resource<KeycloakRealmImport>> keycloakRealmImportClient = KEYCLOAK_OPERATOR_PROVISIONER
.keycloakRealmImportClient();
// create and verify that object exists
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().inNamespace(OpenShiftConfig.namespace()).createOrReplace(keycloak);
keycloakClient.createOrReplace(keycloak);
KEYCLOAK_OPERATOR_PROVISIONER.waitFor(keycloak);
// two pods expected keycloak-0 and keycloak-postgresql-*, keycloak-0 won't start unless keycloak-postgresql-* is ready
if (waitForPods) {
OpenShiftWaiters.get(OpenShifts.master(), () -> false)
.areExactlyNPodsReady(keycloak.getSpec().getInstances().intValue(), "app", keycloak.getKind().toLowerCase())
.level(Level.DEBUG).waitFor();
log.debug(KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getStatus().toString());
}
Assertions.assertEquals(keycloak.getSpec().getHostname().getHostname(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getHostname().getHostname());
keycloakClient.withName(name).get().getSpec().getHostname().getHostname());
if (!Objects.isNull(keycloak.getSpec().getDb())) {
Assertions.assertEquals(keycloak.getSpec().getDb().getHost(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getHost());
keycloakClient.withName(name).get().getSpec().getDb().getHost());
Assertions.assertEquals(keycloak.getSpec().getDb().getDatabase(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getDatabase());
keycloakClient.withName(name).get().getSpec().getDb().getDatabase());
Assertions.assertEquals(keycloak.getSpec().getDb().getUsernameSecret().getName(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getUsernameSecret()
keycloakClient.withName(name).get().getSpec().getDb().getUsernameSecret()
.getName());
Assertions.assertEquals(keycloak.getSpec().getDb().getUsernameSecret().getKey(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getUsernameSecret()
keycloakClient.withName(name).get().getSpec().getDb().getUsernameSecret()
.getKey());
Assertions.assertEquals(keycloak.getSpec().getDb().getPasswordSecret().getName(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getPasswordSecret()
keycloakClient.withName(name).get().getSpec().getDb().getPasswordSecret()
.getName());
Assertions.assertEquals(keycloak.getSpec().getDb().getPasswordSecret().getKey(),
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).get().getSpec().getDb().getPasswordSecret()
keycloakClient.withName(name).get().getSpec().getDb().getPasswordSecret()
.getKey());
}

// import new realm
if (realmImport != null) {
KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().inNamespace(OpenShiftConfig.namespace())
.createOrReplace(realmImport);
keycloakRealmImportClient.createOrReplace(realmImport);
KEYCLOAK_OPERATOR_PROVISIONER.waitFor(realmImport);
KEYCLOAK_OPERATOR_PROVISIONER.waitFor(keycloak);

Assertions.assertEquals(
KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().withName(realmImport.getMetadata().getName())
keycloakRealmImportClient.withName(realmImport.getMetadata().getName())
.get().getSpec().getRealm().getRealm(),
realmName);

KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().withName(realmImport.getMetadata().getName())
keycloakRealmImportClient.withName(realmImport.getMetadata().getName())
.withPropagationPolicy(DeletionPropagation.FOREGROUND)
.delete();
new SimpleWaiter(() -> KEYCLOAK_OPERATOR_PROVISIONER.keycloakRealmImportClient().list().getItems()
new SimpleWaiter(() -> keycloakRealmImportClient.list().getItems()
.stream()
.noneMatch(ri -> realmImport.getMetadata().getName().equalsIgnoreCase(ri.getMetadata().getName())));
}

// delete and verify that object was removed
KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND)
keycloakClient.withName(name).withPropagationPolicy(DeletionPropagation.FOREGROUND)
.delete();
new SimpleWaiter(() -> KEYCLOAK_OPERATOR_PROVISIONER.keycloakClient().list().getItems().size() == 0).level(Level.DEBUG)
new SimpleWaiter(() -> keycloakClient.list().getItems().size() == 0).level(Level.DEBUG)
.waitFor();
if (waitForPods) {
OpenShiftWaiters.get(OpenShifts.master(), () -> false)
Expand Down
21 changes: 11 additions & 10 deletions ...va/org/jboss/intersmash/tools/provision/openshift/KeycloakQuarkusOperatorProvisioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.MixedOperation;
import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation;
import io.fabric8.kubernetes.client.dsl.Resource;
import lombok.NonNull;

Expand Down Expand Up @@ -191,7 +192,7 @@ public void waitFor(KeycloakRealmImport realmImport) {
.count() == 1;
}
return false;
});
}).reason("Wait for KeycloakRealmImport resource to be imported").level(Level.DEBUG).waitFor();
}

private void waitForKeycloakResourceReadiness() {
Expand All @@ -214,12 +215,12 @@ private void waitForKeycloakResourceReadiness() {
* @return A concrete {@link Resource} instance representing the {@link org.jboss.intersmash.tools.provision.openshift.operator.keycloak.keycloak.Keycloak} resource definition
*/
public Resource<Keycloak> keycloak() {
return keycloakClient().inNamespace(OpenShiftConfig.namespace())
return keycloakClient()
.withName(getApplication().getKeycloak().getMetadata().getName());
}

public List<KeycloakRealmImport> keycloakRealmImports() {
return keycloakRealmImportClient().inNamespace(OpenShiftConfig.namespace()).list().getItems()
return keycloakRealmImportClient().list().getItems()
.stream().filter(
realm -> getApplication().getKeycloakRealmImports().stream().map(
ri -> ri.getMetadata().getName())
Expand All @@ -245,15 +246,15 @@ private StatefulSet getStatefulSet() {
public void undeploy() {
keycloakRealmImports()
.forEach(
keycloakRealm -> keycloakRealmImportClient().inNamespace(OpenShiftConfig.namespace())
keycloakRealm -> keycloakRealmImportClient()
.withName(keycloakRealm.getMetadata().getName())
.withPropagationPolicy(DeletionPropagation.FOREGROUND)
.delete());
new SimpleWaiter(
() -> keycloakRealmImportClient().inNamespace(OpenShiftConfig.namespace()).list().getItems().size() == 0)
() -> keycloakRealmImportClient().list().getItems().size() == 0)
.reason("Wait for all keycloakRealmImports instances to be deleted.").level(Level.DEBUG).waitFor();
keycloak().withPropagationPolicy(DeletionPropagation.FOREGROUND).delete();
new SimpleWaiter(() -> keycloakClient().inNamespace(OpenShiftConfig.namespace()).list().getItems().size() == 0)
new SimpleWaiter(() -> keycloakClient().list().getItems().size() == 0)
.reason("Wait for Keycloak instances to be deleted.").level(Level.DEBUG).waitFor();

// wait for 0 pods
Expand Down Expand Up @@ -325,19 +326,19 @@ public URL getURL() {
}
}

public MixedOperation<Keycloak, KubernetesResourceList<Keycloak>, Resource<Keycloak>> keycloakClient() {
public NonNamespaceOperation<Keycloak, KubernetesResourceList<Keycloak>, Resource<Keycloak>> keycloakClient() {
try (KubernetesClient kubernetesClient = new DefaultKubernetesClient()) {
MixedOperation<Keycloak, KubernetesResourceList<Keycloak>, Resource<Keycloak>> keycloakClient = kubernetesClient
.resources(Keycloak.class);
return keycloakClient;
return keycloakClient.inNamespace(OpenShiftConfig.namespace());
}
}

public MixedOperation<KeycloakRealmImport, KubernetesResourceList<KeycloakRealmImport>, Resource<KeycloakRealmImport>> keycloakRealmImportClient() {
public NonNamespaceOperation<KeycloakRealmImport, KubernetesResourceList<KeycloakRealmImport>, Resource<KeycloakRealmImport>> keycloakRealmImportClient() {
try (KubernetesClient kubernetesClient = new DefaultKubernetesClient()) {
MixedOperation<KeycloakRealmImport, KubernetesResourceList<KeycloakRealmImport>, Resource<KeycloakRealmImport>> keycloakRealmImportClient = kubernetesClient
.resources(KeycloakRealmImport.class);
return keycloakRealmImportClient;
return keycloakRealmImportClient.inNamespace(OpenShiftConfig.namespace());
}
}
}
4 changes: 3 additions & 1 deletion ...a/org/jboss/intersmash/tools/provision/openshift/PostgreSQLImageOpenShiftProvisioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public Map<String, String> getImageVariables() {
vars.put("POSTGRESQL_MAX_CONNECTIONS", "100");
vars.put("POSTGRESQL_SHARED_BUFFERS", "16MB");
vars.put("POSTGRESQL_MAX_PREPARED_TRANSACTIONS", "90");
vars.put("POSTGRESQL_DATABASE", dbApplication.getDbName());
// Temporary workaround for https://github.com/sclorg/postgresql-container/issues/297
// Increase the "set_passwords.sh" timeout from the default 60s to 300s to give the
// PostgreSQL server chance properly to start under high OCP cluster load
Expand All @@ -93,8 +94,9 @@ public void customizeApplicationBuilder(ApplicationBuilder appBuilder) {
.addData(POSTGRESQL_PASSWORD_KEY, dbApplication.getPassword().getBytes())
.addData(POSTGRESQL_ADMIN_PASSWORD_KEY,
dbApplication.getAdminPassword().getBytes());
// the secret is also used to configure POSTGRESQL_USER, POSTGRESQL_PASSWORD, POSTGRESQL_ADMIN_PASSWORD
appBuilder.deploymentConfig().podTemplate().container().configFromConfigMap(
dbApplication.getName(),
getSecretName(),
(String t) -> t.replace("-", "_").toUpperCase(),
POSTGRESQL_USER_KEY,
POSTGRESQL_PASSWORD_KEY,
Expand Down

0 comments on commit d3a6a4d

Please sign in to comment.