Skip to content

Commit

Permalink
High availability support for gateway Additional Registration - sing…
Browse files Browse the repository at this point in the history
…le commit

Signed-off-by: alexandr cumarav <[email protected]>
  • Loading branch information
cumarav committed Oct 18, 2023
1 parent b2492fd commit c06a065
Show file tree
Hide file tree
Showing 18 changed files with 693 additions and 53 deletions.
10 changes: 4 additions & 6 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
APIML_SECURITY_AUTH_JWT_CUSTOMAUTHHEADER: customJwtHeader
APIML_SECURITY_AUTH_PASSTICKET_CUSTOMUSERHEADER: customUserHeader
APIML_SECURITY_AUTH_PASSTICKET_CUSTOMAUTHHEADER: customPassticketHeader
APIML_SERVICE_CENTRALREGISTRYURLS: https://discovery-service-2:10011/eureka
ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka
mock-services:
image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }}
metrics-service:
Expand Down Expand Up @@ -254,10 +254,6 @@ jobs:
APIML_SERVICE_APIMLID: central-apiml
APIML_CLOUDGATEWAY_REGISTRY_ENABLED: true
APIML_SECURITY_X509_REGISTRY_ALLOWEDUSERS: USER,UNKNOWNUSER
discoverable-client:
image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }}
mock-services:
image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }}

# Second group of services represents domain apiml instance which registers it's gateway in central's discovery service
discovery-service-2:
Expand All @@ -274,7 +270,9 @@ jobs:
APIML_SERVICE_HOSTNAME: gateway-service-2
APIML_SERVICE_PORT: 10037
APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10031/eureka/
APIML_SERVICE_CENTRALREGISTRYURLS: https://discovery-service:10011/eureka
ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka
ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_GATEWAYURL: /
ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_SERVICEURL: /

steps:
- uses: actions/checkout@v3
Expand Down
13 changes: 11 additions & 2 deletions cloud-gateway-service/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ apiml:
nonStrictVerifySslCertificatesOfServices: true
cloudGateway:
registry:
enabled: false
enabled: true
metadata-key-allow-list: zos.sysname,zos.system,zos.sysplex,zos.cpcName,zos.zosName,zos.lpar

server:
Expand Down Expand Up @@ -62,10 +62,19 @@ logging:
management:
endpoint:
gateway:
enabled: true
enabled: false
endpoints:
web:
base-path: /application
exposure:
include: health,gateway
---
spring:
config:
activate:
on-profile: debug

logging:
level:
org.zowe.apiml: DEBUG

2 changes: 1 addition & 1 deletion cloud-gateway-service/src/main/resources/logback.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,4 @@
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>
</configuration>
2 changes: 1 addition & 1 deletion config/local/gateway-service.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ apiml:
hostname: localhost
ipAddress: 127.0.0.1
port: 10010
centralRegistryUrls: https://localhost:10021/eureka/,https://localhost:10031/eureka/
additionalRegistration: # List of additional Apiml Discovery Services metadata to register with
discoveryServiceUrls: https://localhost:10011/eureka/

security:
Expand Down
3 changes: 0 additions & 3 deletions gateway-package/src/main/resources/bin/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
# - ZWE_configs_apiml_security_oidc_identityMapperUrl
# - ZWE_configs_apiml_security_oidc_identityMapperUser
# - ZWE_configs_apiml_service_allowEncodedSlashes - Allows encoded slashes on on URLs through gateway
# - ZWE_configs_apiml_service_centralRegistryUrls - List of additional Discovery Services URLs to register with
# - ZWE_configs_apiml_service_corsEnabled
# - ZWE_configs_certificate_keystore_alias - The alias of the key within the keystore
# - ZWE_configs_certificate_keystore_file - The keystore to use for SSL certificates
Expand Down Expand Up @@ -210,9 +209,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${GATEWAY_CODE} java \
-Dapiml.service.hostname=${ZWE_haInstance_hostname:-localhost} \
-Dapiml.service.port=${ZWE_configs_port:-7554} \
-Dapiml.service.discoveryServiceUrls=${ZWE_DISCOVERY_SERVICES_LIST:-"https://${ZWE_haInstance_hostname:-localhost}:${ZWE_components_discovery_port:-7553}/eureka/"} \
-Dapiml.service.centralRegistryUrls=${ZWE_configs_apiml_service_centralRegistryUrls:-} \
-Dapiml.service.allowEncodedSlashes=${ZWE_configs_apiml_service_allowEncodedSlashes:-true} \
-Dapiml.service.centralRegistryUrls=${ZWE_configs_apiml_service_centralRegistryUrls:-} \
-Dapiml.service.corsEnabled=${ZWE_configs_apiml_service_corsEnabled:-false} \
-Dapiml.service.externalUrl="${httpProtocol}://${ZWE_zowe_externalDomains_0}:${ZWE_zowe_externalPort}" \
-Dapiml.service.apimlId=${ZWE_configs_apimlId:-} \
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* This program and the accompanying materials are made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v20.html
*
* SPDX-License-Identifier: EPL-2.0
*
* Copyright Contributors to the Zowe Project.
*/

package org.zowe.apiml.gateway.config;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.util.List;

@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class AdditionalRegistration {

private String discoveryServiceUrls;
private List<Route> routes;

@Data
@AllArgsConstructor
@NoArgsConstructor
static class Route {
private String gatewayUrl;
private String serviceUrl;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* This program and the accompanying materials are made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v20.html
*
* SPDX-License-Identifier: EPL-2.0
*
* Copyright Contributors to the Zowe Project.
*/

package org.zowe.apiml.gateway.config;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.type.AnnotatedTypeMetadata;
import org.springframework.web.context.support.StandardServletEnvironment;

import java.util.List;
import java.util.stream.Collectors;

@Slf4j
@RequiredArgsConstructor
public class AdditionalRegistrationCondition implements Condition {

@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
String dcUrls = context.getEnvironment().getProperty("apiml.service.additionalRegistration[0].discoveryServiceUrls");
List<String> additionalKeys = ((StandardServletEnvironment) context.getEnvironment()).getSystemEnvironment()
.entrySet().stream().map(e -> e.getKey().toUpperCase()).filter(key -> key.startsWith(AdditionalRegistrationConfig.COMMON_PREFIX))
.collect(Collectors.toList());
boolean isAdditionalRegistrationsDetected = dcUrls != null || !additionalKeys.isEmpty();
log.debug("isAdditionalRegistrationsDetected: {}", isAdditionalRegistrationsDetected);
return isAdditionalRegistrationsDetected;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
* This program and the accompanying materials are made available under the terms of the
* Eclipse Public License v2.0 which accompanies this distribution, and is available at
* https://www.eclipse.org/legal/epl-v20.html
*
* SPDX-License-Identifier: EPL-2.0
*
* Copyright Contributors to the Zowe Project.
*/

package org.zowe.apiml.gateway.config;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.util.CollectionUtils;

import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

@Slf4j
@Configuration
@RequiredArgsConstructor
public class AdditionalRegistrationConfig {

public static final String DISCOVERY_SERVICE_URLS_KEY = "DISCOVERYSERVICEURLS";
public static final String GATEWAY_URL_KEY = "GATEWAYURL";
public static final String COMMON_PREFIX = "ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_";
public static final String SERVICE_URL_KEY = "SERVICEURL";
private static final int EXPECTED_ROUTE_PART_INDEX = 2;

@Bean
public List<AdditionalRegistration> additionalRegistration(StandardEnvironment environment) {
List<AdditionalRegistration> additionalRegistrations = extractAdditionalRegistrations(System.getenv());
log.debug("Parsed {} additional regs, \t first: {}", additionalRegistrations.size(), additionalRegistrations.stream().findFirst().orElse(null));
return additionalRegistrations;
}

static List<AdditionalRegistration> extractAdditionalRegistrations(Map<String, String> allProperties) {

if (CollectionUtils.isEmpty(allProperties)) {
return Collections.emptyList();
}

Map<String, String> additionalProperties = allProperties.entrySet().stream().filter(entry -> entry.getKey().startsWith(COMMON_PREFIX))
.filter(entry -> StringUtils.isNotBlank(entry.getValue()))
.map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey().replace(COMMON_PREFIX, ""), entry.getValue()))
.collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue, (previous, current) -> current, TreeMap::new));

for (Map.Entry<String, String> prp : additionalProperties.entrySet()) {
log.debug("Additional property: {}={}", prp.getKey(), prp.getValue());
}

Integer listSize = additionalProperties.keySet().stream().map(key -> parsePropertyName(key).getKey())
.max(Integer::compareTo).map(maxIndex -> maxIndex + 1).orElse(0);

List<AdditionalRegistration> additionalRegistrations = IntStream.range(0, listSize)
.mapToObj(index -> AdditionalRegistration.builder().routes(new ArrayList<>()).build())
.collect(Collectors.toList());

mapProperties(additionalRegistrations, additionalProperties);
for (AdditionalRegistration registration : additionalRegistrations) {
List<AdditionalRegistration.Route> definedRoutes = registration.getRoutes().stream().filter(route -> !AdditionalRegistrationConfig.isRouteDefined(route)).collect(Collectors.toList());
registration.setRoutes(definedRoutes);
}

return additionalRegistrations.stream()
.filter(registration -> StringUtils.isNotBlank(registration.getDiscoveryServiceUrls()))
.collect(Collectors.toList());
}

private static void mapProperties(List<AdditionalRegistration> additionalRegistrations, Map<String, String> properties) {
for (Map.Entry<String, String> entry : properties.entrySet()) {
String propertyKey = entry.getKey();
Pair<Integer, String> property = parsePropertyName(propertyKey);
if (property.getKey() == -1) {
continue;
}
final String propertyValue = entry.getValue();
switch (StringUtils.upperCase(property.getValue())) {
case DISCOVERY_SERVICE_URLS_KEY:
additionalRegistrations.get(property.getKey()).setDiscoveryServiceUrls(propertyValue);
break;
case GATEWAY_URL_KEY:
setRouteProperty(additionalRegistrations.get(property.getKey()), parseRouteIndex(propertyKey), (AdditionalRegistration.Route route) -> route.setGatewayUrl(propertyValue));
break;
case SERVICE_URL_KEY:
setRouteProperty(additionalRegistrations.get(property.getKey()), parseRouteIndex(propertyKey), (AdditionalRegistration.Route route) -> route.setServiceUrl(propertyValue));
break;
default:
break;
}
}
}

private static void setRouteProperty(AdditionalRegistration registration, int routeIndex, Consumer<AdditionalRegistration.Route> setter) {
if (routeIndex > -1) {
if (registration.getRoutes().size() <= routeIndex) {
registration.getRoutes().add(new AdditionalRegistration.Route());
setRouteProperty(registration, routeIndex, setter);
}
setter.accept(registration.getRoutes().get(routeIndex));
}
}

private static int parseRouteIndex(String propertyName) {
String[] parts = StringUtils.split(propertyName, "_.");
if (parts.length > EXPECTED_ROUTE_PART_INDEX && StringUtils.isNumeric(parts[EXPECTED_ROUTE_PART_INDEX])) {
return Integer.parseInt(parts[EXPECTED_ROUTE_PART_INDEX]);
}
return -1;
}

private static Pair<Integer, String> parsePropertyName(String fullPropertyName) {
String[] parts = StringUtils.split(fullPropertyName, "_");
if (StringUtils.isNumeric(parts[0])) {
return Pair.of(Integer.parseInt(parts[0]), parts[parts.length - 1]);
}
return Pair.of(-1, null);
}

private static boolean isRouteDefined(AdditionalRegistration.Route route) {
return route == null || StringUtils.isBlank(route.getGatewayUrl()) && StringUtils.isBlank(route.getServiceUrl());
}
}
Loading

0 comments on commit c06a065

Please sign in to comment.