Skip to content

Commit

Permalink
Fix K8s/OpenShift deployment when using management interface+https
Browse files Browse the repository at this point in the history
Note that these changes will move the management ssl configuration to built-time configuration. 

Moreover, the logic to select the HTTPS schema mimics to the logic in VertxHttpRecorder.initializeMainHttpServer method.

Fix quarkusio#32225
  • Loading branch information
Sgitario committed Apr 26, 2023
1 parent 4fb86db commit a908208
Show file tree
Hide file tree
Showing 12 changed files with 183 additions and 20 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,22 @@
public final class KubernetesProbePortNameBuildItem extends SimpleBuildItem {

private final String name;
private final String schema;

public KubernetesProbePortNameBuildItem(String name) {
this(name, null);
}

public KubernetesProbePortNameBuildItem(String name, String schema) {
this.name = name;
this.schema = schema;
}

public String getName() {
return name;
}

public String getSchema() {
return schema;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -957,7 +958,14 @@ public static DecoratorBuildItem createProbeHttpPortDecorator(String name, Strin
Integer port = probeConfig.httpActionPort
.orElse(ports.stream().filter(p -> httpPortName.equals(p.getName()))
.map(KubernetesPortBuildItem::getPort).findFirst().orElse(DEFAULT_HTTP_PORT));
return new DecoratorBuildItem(target, new ApplyHttpGetActionPortDecorator(name, name, port, probeKind));

String schema = probeConfig.httpActionSchema
.or(() -> portName.filter(p -> httpPortName.equals(p.getName()))
.map(KubernetesProbePortNameBuildItem::getSchema)
.filter(Objects::nonNull))
.orElseGet(() -> port != null && (port == 443 || port == 8443) ? "HTTPS" : "HTTP");

return new DecoratorBuildItem(target, new ApplyHttpGetActionPortDecorator(name, name, port, probeKind, schema));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public class ProbeConfig {
@ConfigItem
Optional<String> httpActionPath;

/**
* The schema of the {@literal HTTP get} action. Can be either "HTTP" or "HTTPS".
*/
@ConfigItem
Optional<String> httpActionSchema;

/**
* The command to use for the probe.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import io.quarkus.vertx.http.deployment.webjar.WebJarBuildItem;
import io.quarkus.vertx.http.deployment.webjar.WebJarResourcesFilter;
import io.quarkus.vertx.http.deployment.webjar.WebJarResultsBuildItem;
import io.quarkus.vertx.http.runtime.CertificateConfig;
import io.quarkus.vertx.http.runtime.management.ManagementInterfaceBuildTimeConfig;
import io.smallrye.health.SmallRyeHealthReporter;
import io.smallrye.health.api.HealthGroup;
Expand Down Expand Up @@ -102,6 +103,10 @@ class SmallRyeHealthProcessor {
private static final String BRANDING_FAVICON_GENERAL = BRANDING_DIR + "favicon.ico";
private static final String BRANDING_FAVICON_MODULE = BRANDING_DIR + "smallrye-health-ui.ico";

// For Kubernetes exposing
private static final String SCHEMA_HTTP = "HTTP";
private static final String SCHEMA_HTTPS = "HTTPS";

static class OpenAPIIncluded implements BooleanSupplier {
HealthBuildTimeConfig config;

Expand Down Expand Up @@ -344,7 +349,8 @@ public void kubernetes(NonApplicationRootPathBuildItem nonApplicationRootPathBui

if (managementInterfaceBuildTimeConfig.enabled) {
// Switch to the "management" port
port.produce(new KubernetesProbePortNameBuildItem("management"));
port.produce(new KubernetesProbePortNameBuildItem("management",
selectSchemaForManagement(managementInterfaceBuildTimeConfig)));
}

livenessPathItemProducer.produce(
Expand Down Expand Up @@ -466,4 +472,16 @@ public String updateApiUrl(String original, String healthPath) {
private static boolean shouldInclude(LaunchModeBuildItem launchMode, SmallRyeHealthConfig healthConfig) {
return launchMode.getLaunchMode().isDevOrTest() || healthConfig.ui.alwaysInclude;
}

private static String selectSchemaForManagement(ManagementInterfaceBuildTimeConfig managementHttpConfig) {
CertificateConfig certificateConfig = managementHttpConfig.ssl.certificate;
if (certificateConfig.keyStoreFile.isPresent()
|| certificateConfig.trustStoreFile.isPresent()
|| (certificateConfig.files.isPresent() && certificateConfig.files.get().isEmpty())
|| (certificateConfig.keyFiles.isPresent() && certificateConfig.keyFiles.get().isEmpty())) {
return SCHEMA_HTTPS;
}

return SCHEMA_HTTP;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.quarkus.runtime.annotations.ConfigGroup;
import io.quarkus.runtime.annotations.ConfigItem;
import io.quarkus.runtime.annotations.DefaultConverter;
import io.vertx.core.http.ClientAuth;

/**
* Shared configuration for setting up server-side SSL.
Expand All @@ -17,6 +18,13 @@ public class ServerSslConfig {
*/
public CertificateConfig certificate;

/**
* Configures the engine to require/request client authentication.
* NONE, REQUEST, REQUIRED
*/
@ConfigItem(defaultValue = "NONE")
public ClientAuth clientAuth;

/**
* The cipher suites to use. If none is given, a reasonable default is selected.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import io.quarkus.runtime.annotations.ConfigItem;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.vertx.core.http.ClientAuth;
import io.quarkus.vertx.http.runtime.ServerSslConfig;

/**
* Management interface configuration.
Expand All @@ -27,11 +27,9 @@ public class ManagementInterfaceBuildTimeConfig {
public ManagementAuthConfig auth;

/**
* Configures the engine to require/request client authentication.
* NONE, REQUEST, REQUIRED
* The SSL config
*/
@ConfigItem(name = "ssl.client-auth", defaultValue = "NONE")
public ClientAuth tlsClientAuth;
public ServerSslConfig ssl;

/**
* A common root path for management endpoints. Various extension-provided management endpoints such as metrics
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import io.quarkus.vertx.http.runtime.HeaderConfig;
import io.quarkus.vertx.http.runtime.ProxyConfig;
import io.quarkus.vertx.http.runtime.ServerLimitsConfig;
import io.quarkus.vertx.http.runtime.ServerSslConfig;

/**
* Configures the management interface.
Expand Down Expand Up @@ -53,11 +52,6 @@ public class ManagementInterfaceConfiguration {
@ConfigItem(defaultValue = "true")
public boolean hostEnabled;

/**
* The SSL config
*/
public ServerSslConfig ssl;

/**
* When set to {@code true}, the HTTP server automatically sends `100 CONTINUE`
* response when the request expects it (with the `Expect: 100-Continue` header).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public static HttpServerOptions createSslOptionsForManagementInterface(Managemen
return null;
}

ServerSslConfig sslConfig = httpConfiguration.ssl;
ServerSslConfig sslConfig = buildTimeConfig.ssl;

final Optional<Path> certFile = sslConfig.certificate.file;
final Optional<Path> keyFile = sslConfig.certificate.keyFile;
Expand Down Expand Up @@ -233,7 +233,7 @@ public static HttpServerOptions createSslOptionsForManagementInterface(Managemen
int sslPort = httpConfiguration.determinePort(launchMode);
// -2 instead of -1 (see http) to have vert.x assign two different random ports if both http and https shall be random
serverOptions.setPort(sslPort == 0 ? -2 : sslPort);
serverOptions.setClientAuth(buildTimeConfig.tlsClientAuth);
serverOptions.setClientAuth(buildTimeConfig.ssl.clientAuth);

applyCommonOptionsForManagementInterface(serverOptions, buildTimeConfig, httpConfiguration, websocketSubProtocols);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@

public class KubernetesWithHealthUsingManagementInterfaceTest {

private static final String NAME = "kubernetes-with-health-and-management";

@RegisterExtension
static final QuarkusProdModeTest config = new QuarkusProdModeTest()
.withApplicationRoot((jar) -> jar.addClasses(GreetingResource.class))
.setApplicationName("health")
.setApplicationName(NAME)
.setApplicationVersion("0.1-SNAPSHOT")
.setRun(true)
.setLogFileName("k8s.log")
.withConfigurationResource("kubernetes-with-health-and-management.properties")
.withConfigurationResource(NAME + ".properties")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-smallrye-health", Version.getVersion())));

Expand Down Expand Up @@ -64,7 +66,7 @@ public void assertGeneratedResources() throws IOException {
.deserializeAsList(kubernetesDir.resolve("kubernetes.yml"));
assertThat(kubernetesList.get(0)).isInstanceOfSatisfying(Deployment.class, d -> {
assertThat(d.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo("health");
assertThat(m.getName()).isEqualTo(NAME);
});

assertThat(d.getSpec()).satisfies(deploymentSpec -> {
Expand All @@ -76,14 +78,16 @@ public void assertGeneratedResources() throws IOException {
assertProbePath(p, "/q/health/ready");

assertNotNull(p.getHttpGet());
assertEquals(p.getHttpGet().getPort().getIntVal(), 9000);
assertEquals("HTTP", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
assertThat(container.getLivenessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(20);
assertProbePath(p, "/liveness");

assertNotNull(p.getHttpGet());
assertEquals(p.getHttpGet().getPort().getIntVal(), 9000);
assertEquals("HTTP", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
});
});
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package io.quarkus.it.kubernetes;

import static io.restassured.RestAssured.given;
import static org.assertj.core.api.Assertions.assertThat;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.Probe;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.quarkus.builder.Version;
import io.quarkus.kubernetes.spi.CustomProjectRootBuildItem;
import io.quarkus.maven.dependency.Dependency;
import io.quarkus.test.LogFile;
import io.quarkus.test.ProdBuildResults;
import io.quarkus.test.ProdModeTestResults;
import io.quarkus.test.QuarkusProdModeTest;

public class KubernetesWithHealthUsingSecuredManagementInterfaceTest {

private static final String NAME = "kubernetes-with-health-and-secured-management";

@RegisterExtension
static final QuarkusProdModeTest config = new QuarkusProdModeTest()
.withApplicationRoot((jar) -> jar.addClasses(GreetingResource.class))
.setApplicationName(NAME)
.setApplicationVersion("0.1-SNAPSHOT")
.addCustomResourceEntry(Path.of("src", "main", "resources", "META-INF", "server.keystore"),
"manifests/" + NAME + "/server.keystore")
.setLogFileName("k8s.log")
.setRun(true)
.withConfigurationResource(NAME + ".properties")
.setForcedDependencies(List.of(
Dependency.of("io.quarkus", "quarkus-smallrye-health", Version.getVersion())))
.addBuildChainCustomizerEntries(
new QuarkusProdModeTest.BuildChainCustomizerEntry(
OpenshiftWithCommonResourcesTest.CustomProjectRootBuildItemProducerProdMode.class,
Collections.singletonList(CustomProjectRootBuildItem.class), Collections.emptyList()));

@ProdBuildResults
private ProdModeTestResults prodModeTestResults;

@LogFile
private Path logfile;

@Test
public void assertApplicationRuns() {
assertThat(logfile).isRegularFile().hasFileName("k8s.log");
TestUtil.assertLogFileContents(logfile, "kubernetes", "health");
given()
.when().get("/greeting")
.then()
.statusCode(200)
.body(is("hello"));
}

@Test
public void assertGeneratedResources() throws IOException {
final Path kubernetesDir = prodModeTestResults.getBuildDir().resolve("kubernetes");
assertThat(kubernetesDir)
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.json"))
.isDirectoryContaining(p -> p.getFileName().endsWith("kubernetes.yml"));
List<HasMetadata> kubernetesList = DeserializationUtil
.deserializeAsList(kubernetesDir.resolve("kubernetes.yml"));
assertThat(kubernetesList.get(0)).isInstanceOfSatisfying(Deployment.class, d -> {
assertThat(d.getMetadata()).satisfies(m -> {
assertThat(m.getName()).isEqualTo(NAME);
});

assertThat(d.getSpec()).satisfies(deploymentSpec -> {
assertThat(deploymentSpec.getTemplate()).satisfies(t -> {
assertThat(t.getSpec()).satisfies(podSpec -> {
assertThat(podSpec.getContainers()).singleElement().satisfies(container -> {
assertThat(container.getReadinessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(5);
assertProbePath(p, "/q/health/ready");

assertNotNull(p.getHttpGet());
assertEquals("HTTPS", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
assertThat(container.getLivenessProbe()).isNotNull().satisfies(p -> {
assertThat(p.getInitialDelaySeconds()).isEqualTo(20);
assertProbePath(p, "/liveness");

assertNotNull(p.getHttpGet());
assertEquals("HTTPS", p.getHttpGet().getScheme());
assertEquals(9000, p.getHttpGet().getPort().getIntVal());
});
});
});
});
});
});
}

private void assertProbePath(Probe p, String expectedPath) {
assertThat(p.getHttpGet()).satisfies(h -> {
assertThat(h.getPath()).isEqualTo(expectedPath);
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
quarkus.http.port=9090
quarkus.management.enabled=true
quarkus.management.ssl.certificate.key-store-file=META-INF/server.keystore
quarkus.management.ssl.certificate.key-store-password=password
quarkus.kubernetes.liveness-probe.initial-delay=20s
quarkus.smallrye-health.liveness-path=/liveness
Binary file not shown.

0 comments on commit a908208

Please sign in to comment.