Generalize the ingress configuration to support single-host in single…

…-user mode on kubernetes (#14134)

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties

@@ -478,14 +478,41 @@ che.infra.kubernetes.pvc.wait_bound=true
 che.infra.kubernetes.installer_server_min_port=10000
 che.infra.kubernetes.installer_server_max_port=20000
 
-# Defines annotations for ingresses which are used for servers exposing. Value depends on ingress controller.
-# For example for nginx ingress controller 0.9.0-beta17 the following value is recommended:
-# {"ingress.kubernetes.io/rewrite-target": "/","ingress.kubernetes.io/ssl-redirect": "false",\
-#     "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}
+# Defines annotations for ingresses which are used for servers exposing. Value depends on the kind of ingress
+# controller.
 #
 # OpenShift infrastructure ignores this property because it uses Routes instead of ingresses.
+#
+# Note that for a single-host deployment strategy to work, a controller supporting URL rewriting has to be
+# used (so that URLs can point to different servers while the servers don't need to support changing the app root).
+# The che.infra.kubernetes.ingress.path.rewrite_transform property defines how the path of the ingress should be
+# transformed to support the URL rewriting and this property defines the set of annotations on the ingress itself
+# that instruct the chosen ingress controller to actually do the URL rewriting, potentially building on the path
+# transformation (if required by the chosen ingress controller).
+#
+# For example for nginx ingress controller 0.22.0 and later the following value is recommended:
+# {"ingress.kubernetes.io/rewrite-target": "/$1","ingress.kubernetes.io/ssl-redirect": "false",\
+#     "ingress.kubernetes.io/proxy-connect-timeout": "3600","ingress.kubernetes.io/proxy-read-timeout": "3600"}
+# and the che.infra.kubernetes.ingress.path.rewrite_transform should be set to "%s(.*)"
+#
+# For nginx ingress controller older than 0.22.0, the rewrite-target should be set to merely "/" and the path transform
+# to "%s" (see the the che.infra.kubernetes.ingress.path.rewrite_transform property).
+#
+# Please consult the nginx ingress controller documentation for the explanation of how the ingress controller uses
+# the regular expression present in the ingress path and how it achieves the URL rewriting.
 che.infra.kubernetes.ingress.annotations_json=NULL
 
+# Defines a "recipe" on how to declare the path of the ingress that should expose a server.
+# The "%s" represents the base public URL of the server and is guaranteed to end with a forward slash. This property
+# must be a valid input to the String.format() method and contain exactly one reference to "%s".
+#
+# Please see the description of the che.infra.kubernetes.ingress.annotations_json property to see how these two
+# properties interplay when specifying the ingress annotations and path.
+#
+# If not defined, this property defaults to "%s" (without the quotes) which means that the path is not transformed in
+# any way for use with the ingress controller.
+che.infra.kubernetes.ingress.path_transform=NULL
+
 # Defines security context for pods that will be created by Kubernetes Infra
 #
 # This is ignored by OpenShift infra

deploy/kubernetes/helm/che/templates/configmap.yaml

@@ -68,10 +68,11 @@ data:
   JAVA_OPTS: "-XX:MaxRAMFraction=2 -XX:+UseParallelGC -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=20 -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -Dsun.zip.disableMemoryMapping=true -Xms20m "
   CHE_WORKSPACE_AUTO_START: "false"
 {{- if .Values.global.tls.enabled }}
-  CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"kubernetes.io/ingress.class": "nginx", "kubernetes.io/tls-acme": "true", "{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/rewrite-target": "/","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect": "true","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-connect-timeout": "3600","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-read-timeout": "3600"}'
+  CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"kubernetes.io/ingress.class": "nginx", "kubernetes.io/tls-acme": "true", "{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/rewrite-target": "/$1","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect": "true","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-connect-timeout": "3600","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-read-timeout": "3600"}'
 {{- else }}
-  CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"kubernetes.io/ingress.class": "nginx", "{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/rewrite-target": "/","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect": "false","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-connect-timeout": "3600","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-read-timeout": "3600"}'
+  CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON: '{"kubernetes.io/ingress.class": "nginx", "{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/rewrite-target": "/$1","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/ssl-redirect": "false","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-connect-timeout": "3600","{{ .Values.global.ingressAnnotationsPrefix }}ingress.kubernetes.io/proxy-read-timeout": "3600"}'
 {{- end }}
+  CHE_INFRA_KUBERNETES_INGRESS_PATH__TRANSFORM: '%s(.*)'
   CHE_INFRA_KUBERNETES_SERVER__STRATEGY: {{ .Values.global.serverStrategy }}
   CHE_LOGGER_CONFIG: {{ .Values.global.log.loggerConfig | quote}}
   CHE_LOGS_APPENDERS_IMPL: {{ .Values.global.log.appenderName }}

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java

@@ -17,9 +17,9 @@
 import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY;
 import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.PerWorkspacePVCStrategy.PER_WORKSPACE_STRATEGY;
 import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.UniqueWorkspacePVCStrategy.UNIQUE_STRATEGY;
-import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.DefaultHostIngressExternalServerExposer.DEFAULT_HOST_STRATEGY;
-import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.MultiHostIngressExternalServerExposer.MULTI_HOST_STRATEGY;
-import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.SingleHostIngressExternalServerExposer.SINGLE_HOST_STRATEGY;
+import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.DefaultHostIngressServiceExposureStrategy.DEFAULT_HOST_STRATEGY;
+import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.MultiHostIngressServiceExposureStrategy.MULTI_HOST_STRATEGY;
+import static org.eclipse.che.workspace.infrastructure.kubernetes.server.external.SingleHostIngressServiceExposureStrategy.SINGLE_HOST_STRATEGY;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.TypeLiteral;
@@ -60,11 +60,11 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.env.LogsRootEnvVariableProvider;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.server.ServersConverter;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.IngressAnnotationsProvider;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.DefaultHostIngressExternalServerExposer;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposerStrategy;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposerStrategyProvider;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.MultiHostIngressExternalServerExposer;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.SingleHostIngressExternalServerExposer;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.DefaultHostIngressServiceExposureStrategy;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.IngressServiceExposureStrategy;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.IngressServiceExposureStrategyProvider;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.MultiHostIngressServiceExposureStrategy;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.SingleHostIngressServiceExposureStrategy;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.DefaultSecureServersFactory;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.SecureServerExposerFactory;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.SecureServerExposerFactoryProvider;
@@ -117,23 +117,19 @@ protected void configure() {
         .addBinding()
         .to(KubernetesClientTermination.class);
 
-    MapBinder<String, ExternalServerExposerStrategy<KubernetesEnvironment>> ingressStrategies =
-        MapBinder.newMapBinder(
-            binder(),
-            new TypeLiteral<String>() {},
-            new TypeLiteral<ExternalServerExposerStrategy<KubernetesEnvironment>>() {});
+    MapBinder<String, IngressServiceExposureStrategy> ingressStrategies =
+        MapBinder.newMapBinder(binder(), String.class, IngressServiceExposureStrategy.class);
     ingressStrategies
         .addBinding(MULTI_HOST_STRATEGY)
-        .to(MultiHostIngressExternalServerExposer.class);
+        .to(MultiHostIngressServiceExposureStrategy.class);
     ingressStrategies
         .addBinding(SINGLE_HOST_STRATEGY)
-        .to(SingleHostIngressExternalServerExposer.class);
+        .to(SingleHostIngressServiceExposureStrategy.class);
     ingressStrategies
         .addBinding(DEFAULT_HOST_STRATEGY)
-        .to(DefaultHostIngressExternalServerExposer.class);
-    bind(new TypeLiteral<ExternalServerExposerStrategy<KubernetesEnvironment>>() {})
-        .toProvider(
-            new TypeLiteral<ExternalServerExposerStrategyProvider<KubernetesEnvironment>>() {});
+        .to(DefaultHostIngressServiceExposureStrategy.class);
+    bind(IngressServiceExposureStrategy.class)
+        .toProvider(IngressServiceExposureStrategyProvider.class);
 
     bind(ServersConverter.class).to(new TypeLiteral<ServersConverter<KubernetesEnvironment>>() {});
 

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java

@@ -85,6 +85,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent;
 import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.IngressPathTransformInverter;
 import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool;
 import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher;
 import org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory;
@@ -118,6 +119,7 @@ public class KubernetesInternalRuntime<E extends KubernetesEnvironment>
   private final Set<InternalEnvironmentProvisioner> internalEnvironmentProvisioners;
   private final KubernetesEnvironmentProvisioner<E> kubernetesEnvironmentProvisioner;
   private final SidecarToolingProvisioner<E> toolingProvisioner;
+  private final IngressPathTransformInverter ingressPathTransformInverter;
   private final RuntimeHangingDetector runtimeHangingDetector;
   protected final Tracer tracer;
 
@@ -140,6 +142,7 @@ public KubernetesInternalRuntime(
       Set<InternalEnvironmentProvisioner> internalEnvironmentProvisioners,
       KubernetesEnvironmentProvisioner<E> kubernetesEnvironmentProvisioner,
       SidecarToolingProvisioner<E> toolingProvisioner,
+      IngressPathTransformInverter ingressPathTransformInverter,
       RuntimeHangingDetector runtimeHangingDetector,
       Tracer tracer,
       @Assisted KubernetesRuntimeContext<E> context,
@@ -161,6 +164,7 @@ public KubernetesInternalRuntime(
     this.toolingProvisioner = toolingProvisioner;
     this.kubernetesEnvironmentProvisioner = kubernetesEnvironmentProvisioner;
     this.internalEnvironmentProvisioners = internalEnvironmentProvisioners;
+    this.ingressPathTransformInverter = ingressPathTransformInverter;
     this.runtimeHangingDetector = runtimeHangingDetector;
     this.startSynchronizer = startSynchronizerFactory.create(context.getIdentity());
     this.tracer = tracer;
@@ -645,7 +649,7 @@ protected void startMachines() throws InfrastructureException {
     listenEvents();
 
     final KubernetesServerResolver serverResolver =
-        new KubernetesServerResolver(createdServices, readyIngresses);
+        new KubernetesServerResolver(ingressPathTransformInverter, createdServices, readyIngresses);
 
     doStartMachine(serverResolver);
   }

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/server/ServersConverter.java

@@ -27,7 +27,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ConfigurationProvisioner;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerExposer;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposerStrategy;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposer;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.SecureServerExposer;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.SecureServerExposerFactoryProvider;
 
@@ -44,14 +44,14 @@
 public class ServersConverter<T extends KubernetesEnvironment>
     implements ConfigurationProvisioner<T> {
 
-  private final ExternalServerExposerStrategy<T> externalServerExposerStrategy;
+  private final ExternalServerExposer<T> externalServerExposer;
   private final SecureServerExposerFactoryProvider<T> secureServerExposerFactoryProvider;
 
   @Inject
   public ServersConverter(
-      ExternalServerExposerStrategy<T> externalServerExposerStrategy,
+      ExternalServerExposer<T> externalServerExposer,
       SecureServerExposerFactoryProvider<T> secureServerExposerFactoryProvider) {
-    this.externalServerExposerStrategy = externalServerExposerStrategy;
+    this.externalServerExposer = externalServerExposer;
     this.secureServerExposerFactoryProvider = secureServerExposerFactoryProvider;
   }
 
@@ -72,7 +72,7 @@ public void provision(T k8sEnv, RuntimeIdentity identity) throws InfrastructureE
         if (!machineConfig.getServers().isEmpty()) {
           KubernetesServerExposer kubernetesServerExposer =
               new KubernetesServerExposer<>(
-                  externalServerExposerStrategy,
+                  externalServerExposer,
                   secureServerExposer,
                   machineName,
                   podConfig,

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerExposer.java

@@ -39,7 +39,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.UniqueNamesProvisioner;
-import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposerStrategy;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.ExternalServerExposer;
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.secure.SecureServerExposer;
 
 /**
@@ -103,15 +103,15 @@ public class KubernetesServerExposer<T extends KubernetesEnvironment> {
   public static final int SERVER_UNIQUE_PART_SIZE = 8;
   public static final String SERVER_PREFIX = "server";
 
-  private final ExternalServerExposerStrategy<T> externalServerExposer;
+  private final ExternalServerExposer<T> externalServerExposer;
   private final SecureServerExposer<T> secureServerExposer;
   private final String machineName;
   private final Container container;
   private final PodData pod;
   private final T k8sEnv;
 
   public KubernetesServerExposer(
-      ExternalServerExposerStrategy<T> externalServerExposer,
+      ExternalServerExposer<T> externalServerExposer,
       SecureServerExposer<T> secureServerExposer,
       String machineName,
       PodData pod,

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/server/KubernetesServerResolver.java

@@ -24,6 +24,7 @@
 import org.eclipse.che.api.workspace.server.model.impl.ServerImpl;
 import org.eclipse.che.commons.annotation.Nullable;
 import org.eclipse.che.workspace.infrastructure.kubernetes.Annotations;
+import org.eclipse.che.workspace.infrastructure.kubernetes.server.external.IngressPathTransformInverter;
 
 /**
  * Helps to resolve {@link ServerImpl servers} by machine name according to specified {@link Ingress
@@ -40,8 +41,13 @@
 public class KubernetesServerResolver {
   private final Multimap<String, Service> services;
   private final Multimap<String, Ingress> ingresses;
+  private IngressPathTransformInverter pathTransformInverter;
 
-  public KubernetesServerResolver(List<Service> services, List<Ingress> ingresses) {
+  public KubernetesServerResolver(
+      IngressPathTransformInverter pathTransformInverter,
+      List<Service> services,
+      List<Ingress> ingresses) {
+    this.pathTransformInverter = pathTransformInverter;
     this.services = ArrayListMultimap.create();
     for (Service service : services) {
       String machineName =
@@ -109,7 +115,10 @@ private void fillIngressServers(Ingress ingress, Map<String, ServerImpl> servers
         .forEach(
             (name, config) -> {
               String path =
-                  buildPath(ingressRule.getHttp().getPaths().get(0).getPath(), config.getPath());
+                  buildPath(
+                      pathTransformInverter.undoPathTransformation(
+                          ingressRule.getHttp().getPaths().get(0).getPath()),
+                      config.getPath());
               servers.put(
                   name,
                   new RuntimeServerBuilder()
@@ -137,6 +146,11 @@ private String buildPath(String fragment1, @Nullable String fragment2) {
       }
     }
 
+    // always end server URLs with a slash, so that they can be safely sub-path'd..
+    if (sb.charAt(sb.length() - 1) != '/') {
+      sb.append('/');
+    }
+
     return sb.toString();
   }
 }