diff --git a/components/api/src/main/java/com/hotels/styx/api/HttpHeader.java b/components/api/src/main/java/com/hotels/styx/api/HttpHeader.java index 11c0017b6d..0478dbec1c 100644 --- a/components/api/src/main/java/com/hotels/styx/api/HttpHeader.java +++ b/components/api/src/main/java/com/hotels/styx/api/HttpHeader.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/components/api/src/main/java/com/hotels/styx/api/HttpRequest.java b/components/api/src/main/java/com/hotels/styx/api/HttpRequest.java index e8943708d2..5eec5d61c7 100644 --- a/components/api/src/main/java/com/hotels/styx/api/HttpRequest.java +++ b/components/api/src/main/java/com/hotels/styx/api/HttpRequest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ import java.util.Set; import java.util.function.Predicate; -import static com.google.common.base.Objects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.hotels.styx.api.HttpHeaderNames.CONNECTION; import static com.hotels.styx.api.HttpHeaderNames.CONTENT_LENGTH; @@ -346,13 +345,10 @@ public Optional cookie(String name) { @Override public String toString() { - return toStringHelper(this) - .add("version", version) - .add("method", method) - .add("uri", url) - .add("headers", headers) - .add("id", id) - .toString(); + return "{version=" + version + + ", method=" + method + + ", uri=" + url + + ", id=" + id + "}"; } /** diff --git a/components/api/src/main/java/com/hotels/styx/api/HttpResponse.java b/components/api/src/main/java/com/hotels/styx/api/HttpResponse.java index fd17ac49ae..ff6e0f443d 100644 --- a/components/api/src/main/java/com/hotels/styx/api/HttpResponse.java +++ b/components/api/src/main/java/com/hotels/styx/api/HttpResponse.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import java.util.Set; import java.util.function.Predicate; -import static com.google.common.base.Objects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.hotels.styx.api.HttpHeaderNames.CONTENT_LENGTH; import static com.hotels.styx.api.HttpHeaderNames.SET_COOKIE; @@ -211,11 +210,8 @@ public Optional cookie(String name) { @Override public String toString() { - return toStringHelper(this) - .add("version", version) - .add("status", status) - .add("headers", headers) - .toString(); + return "{version=" + version + + ", status=" + status + "}"; } @Override diff --git a/components/api/src/main/java/com/hotels/styx/api/LiveHttpRequest.java b/components/api/src/main/java/com/hotels/styx/api/LiveHttpRequest.java index 8a8979f513..828ff11fd9 100644 --- a/components/api/src/main/java/com/hotels/styx/api/LiveHttpRequest.java +++ b/components/api/src/main/java/com/hotels/styx/api/LiveHttpRequest.java @@ -27,7 +27,6 @@ import java.util.function.Function; import java.util.function.Predicate; -import static com.google.common.base.Objects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.hotels.styx.api.HttpHeaderNames.CONNECTION; import static com.hotels.styx.api.HttpHeaderNames.CONTENT_LENGTH; @@ -385,13 +384,10 @@ public Optional cookie(String name) { @Override public String toString() { - return toStringHelper(this) - .add("version", version) - .add("method", method) - .add("uri", url) - .add("headers", headers) - .add("id", id) - .toString(); + return "{version=" + version + + ", method=" + method + + ", uri=" + url + + ", id=" + id + "}"; } private interface BuilderTransformer { diff --git a/components/api/src/main/java/com/hotels/styx/api/LiveHttpResponse.java b/components/api/src/main/java/com/hotels/styx/api/LiveHttpResponse.java index 63c4de0825..367a4b4eb0 100644 --- a/components/api/src/main/java/com/hotels/styx/api/LiveHttpResponse.java +++ b/components/api/src/main/java/com/hotels/styx/api/LiveHttpResponse.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,6 @@ import java.util.function.Function; import java.util.function.Predicate; -import static com.google.common.base.Objects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.hotels.styx.api.HttpHeaderNames.CONTENT_LENGTH; import static com.hotels.styx.api.HttpHeaderNames.SET_COOKIE; @@ -232,11 +231,8 @@ public Optional cookie(String name) { @Override public String toString() { - return toStringHelper(this) - .add("version", version) - .add("status", status) - .add("headers", headers) - .toString(); + return "{version=" + version + + ", status=" + status + "}"; } @Override diff --git a/components/api/src/test/java/com/hotels/styx/api/HttpHeaderTest.java b/components/api/src/test/java/com/hotels/styx/api/HttpHeaderTest.java index be4cbb414e..17ebcfea34 100644 --- a/components/api/src/test/java/com/hotels/styx/api/HttpHeaderTest.java +++ b/components/api/src/test/java/com/hotels/styx/api/HttpHeaderTest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/components/api/src/test/java/com/hotels/styx/api/HttpRequestTest.java b/components/api/src/test/java/com/hotels/styx/api/HttpRequestTest.java index f8a4a1ae28..1c31f56732 100644 --- a/components/api/src/test/java/com/hotels/styx/api/HttpRequestTest.java +++ b/components/api/src/test/java/com/hotels/styx/api/HttpRequestTest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,9 +23,6 @@ import java.util.Optional; -import static com.hotels.styx.api.HttpRequest.get; -import static com.hotels.styx.api.HttpRequest.patch; -import static com.hotels.styx.api.HttpRequest.put; import static com.hotels.styx.api.HttpHeader.header; import static com.hotels.styx.api.HttpHeaderNames.CONTENT_LENGTH; import static com.hotels.styx.api.HttpHeaderNames.COOKIE; @@ -33,6 +30,9 @@ import static com.hotels.styx.api.HttpMethod.DELETE; import static com.hotels.styx.api.HttpMethod.GET; import static com.hotels.styx.api.HttpMethod.POST; +import static com.hotels.styx.api.HttpRequest.get; +import static com.hotels.styx.api.HttpRequest.patch; +import static com.hotels.styx.api.HttpRequest.put; import static com.hotels.styx.api.HttpVersion.HTTP_1_0; import static com.hotels.styx.api.HttpVersion.HTTP_1_1; import static com.hotels.styx.api.RequestCookie.requestCookie; @@ -131,10 +131,8 @@ public void canUseBuilderToSetRequestProperties() { .cookies(requestCookie("cfoo", "bar")) .build(); - assertThat(request.toString(), is("HttpRequest{version=HTTP/1.1, method=PATCH, uri=https://hotels.com, " + - "headers=[headerName=a, Cookie=cfoo=bar, Host=hotels.com], id=id}")); - - assertThat(request.headers("headerName"), is(singletonList("a"))); + assertThat(request.toString(), is("{version=HTTP/1.1, method=PATCH, uri=https://hotels.com, id=id}")); + assertThat(request.headers().toString(), is("[headerName=a, Cookie=cfoo=bar, Host=hotels.com]")); } @Test diff --git a/components/api/src/test/java/com/hotels/styx/api/LiveHttpRequestTest.java b/components/api/src/test/java/com/hotels/styx/api/LiveHttpRequestTest.java index 7a381fc76c..827dcb8b2a 100644 --- a/components/api/src/test/java/com/hotels/styx/api/LiveHttpRequestTest.java +++ b/components/api/src/test/java/com/hotels/styx/api/LiveHttpRequestTest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -137,9 +137,8 @@ public void canUseBuilderToSetRequestProperties() { .cookies(requestCookie("cfoo", "bar")) .build(); - assertThat(request.toString(), is("LiveHttpRequest{version=HTTP/1.0, method=PATCH, uri=https://hotels.com, headers=[headerName=a, Cookie=cfoo=bar, Host=hotels.com], id=id}")); - - assertThat(request.headers("headerName"), is(singletonList("a"))); + assertThat(request.toString(), is("{version=HTTP/1.0, method=PATCH, uri=https://hotels.com, id=id}")); + assertThat(request.headers().toString(), is("[headerName=a, Cookie=cfoo=bar, Host=hotels.com]")); } @Test diff --git a/components/client/src/main/java/com/hotels/styx/client/HttpRequestOperationFactory.java b/components/client/src/main/java/com/hotels/styx/client/HttpRequestOperationFactory.java index 5c7c306d0f..386dab31ae 100644 --- a/components/client/src/main/java/com/hotels/styx/client/HttpRequestOperationFactory.java +++ b/components/client/src/main/java/com/hotels/styx/client/HttpRequestOperationFactory.java @@ -19,6 +19,10 @@ import com.hotels.styx.api.metrics.codahale.CodaHaleMetricRegistry; import com.hotels.styx.client.OriginStatsFactory.CachingOriginStatsFactory; import com.hotels.styx.client.netty.connectionpool.HttpRequestOperation; +import com.hotels.styx.common.format.DefaultHttpMessageFormatter; +import com.hotels.styx.common.format.HttpMessageFormatter; + +import static java.util.Objects.requireNonNull; /** * A Factory for creating an HttpRequestOperation from an LiveHttpRequest. @@ -41,6 +45,7 @@ class Builder { boolean flowControlEnabled; boolean requestLoggingEnabled; boolean longFormat; + HttpMessageFormatter httpMessageFormatter = new DefaultHttpMessageFormatter(); public static Builder httpRequestOperationFactoryBuilder() { return new Builder(); @@ -71,13 +76,19 @@ public Builder longFormat(boolean longFormat) { return this; } + public Builder httpMessageFormatter(HttpMessageFormatter httpMessageFormatter) { + this.httpMessageFormatter = requireNonNull(httpMessageFormatter); + return this; + } + public HttpRequestOperationFactory build() { return request -> new HttpRequestOperation( request, originStatsFactory, responseTimeoutMillis, requestLoggingEnabled, - longFormat); + longFormat, + httpMessageFormatter); } } diff --git a/components/client/src/main/java/com/hotels/styx/client/StyxBackendServiceClient.java b/components/client/src/main/java/com/hotels/styx/client/StyxBackendServiceClient.java index cd4a3aa948..45f9fb001e 100644 --- a/components/client/src/main/java/com/hotels/styx/client/StyxBackendServiceClient.java +++ b/components/client/src/main/java/com/hotels/styx/client/StyxBackendServiceClient.java @@ -255,9 +255,9 @@ private static String hosts(Iterable origins) { } } - private static void logError(LiveHttpRequest rewrittenRequest, Throwable throwable) { + private static void logError(LiveHttpRequest request, Throwable throwable) { LOGGER.error("Error Handling request={} exceptionClass={} exceptionMessage=\"{}\"", - new Object[]{rewrittenRequest, throwable.getClass().getName(), throwable.getMessage()}); + new Object[]{request, throwable.getClass().getName(), throwable.getMessage()}); } private LiveHttpResponse removeUnexpectedResponseBody(LiveHttpRequest request, LiveHttpResponse response) { diff --git a/components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java b/components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java index ae93667066..61ed65a842 100644 --- a/components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java +++ b/components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java @@ -24,7 +24,6 @@ import com.hotels.styx.api.Url; import com.hotels.styx.api.extension.Origin; import com.hotels.styx.api.extension.service.TlsSettings; -import com.hotels.styx.client.netty.connectionpool.HttpRequestOperation; import com.hotels.styx.client.netty.connectionpool.NettyConnectionFactory; import com.hotels.styx.client.ssl.SslContextFactory; import io.netty.handler.ssl.SslContext; @@ -40,6 +39,7 @@ import static com.hotels.styx.api.HttpHeaderNames.USER_AGENT; import static com.hotels.styx.api.extension.Origin.newOriginBuilder; import static com.hotels.styx.client.HttpConfig.newHttpConfigBuilder; +import static com.hotels.styx.client.HttpRequestOperationFactory.Builder.httpRequestOperationFactoryBuilder; import static java.util.Objects.requireNonNull; /** @@ -316,15 +316,13 @@ Builder copy() { * @return a new instance */ public StyxHttpClient build() { + NettyConnectionFactory connectionFactory = new NettyConnectionFactory.Builder() .httpConfig(newHttpConfigBuilder().setMaxHeadersSize(maxHeaderSize).build()) .tlsSettings(tlsSettings) - .httpRequestOperationFactory(request -> new HttpRequestOperation( - request, - null, - responseTimeout, - false, - false)) + .httpRequestOperationFactory(httpRequestOperationFactoryBuilder() + .responseTimeoutMillis(responseTimeout) + .build()) .build(); return new StyxHttpClient(connectionFactory, this.copy()); diff --git a/components/client/src/main/java/com/hotels/styx/client/netty/connectionpool/HttpRequestOperation.java b/components/client/src/main/java/com/hotels/styx/client/netty/connectionpool/HttpRequestOperation.java index cf97fd1310..31a7e9509f 100644 --- a/components/client/src/main/java/com/hotels/styx/client/netty/connectionpool/HttpRequestOperation.java +++ b/components/client/src/main/java/com/hotels/styx/client/netty/connectionpool/HttpRequestOperation.java @@ -18,14 +18,16 @@ import com.google.common.annotations.VisibleForTesting; import com.hotels.styx.api.Buffers; import com.hotels.styx.api.HttpMethod; +import com.hotels.styx.api.HttpVersion; import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.api.LiveHttpResponse; -import com.hotels.styx.api.HttpVersion; import com.hotels.styx.api.Requests; import com.hotels.styx.api.exceptions.TransportLostException; import com.hotels.styx.api.extension.Origin; import com.hotels.styx.client.Operation; import com.hotels.styx.client.OriginStatsFactory; +import com.hotels.styx.common.format.HttpMessageFormatter; +import com.hotels.styx.common.format.SanitisedHttpMessageFormatter; import com.hotels.styx.common.logging.HttpRequestMessageLogger; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -79,8 +81,8 @@ public class HttpRequestOperation implements Operation COOKIE_HEADER_NAMES = Arrays.asList("cookie", "set-cookie"); + + private final List headersToHide; + private final List cookiesToHide; + + public SanitisedHttpHeaderFormatter(List headersToHide, List cookiesToHide) { + this.headersToHide = requireNonNull(headersToHide); + this.cookiesToHide = requireNonNull(cookiesToHide); + } + + public String format(HttpHeaders headers) { + return StreamSupport.stream(headers.spliterator(), false) + .map(this::hideOrFormatHeader) + .collect(Collectors.joining(", ")); + } + + private String hideOrFormatHeader(HttpHeader header) { + return shouldHideHeader(header) + ? header.name() + "=****" + : formatHeaderAsCookieIfNecessary(header); + } + + private boolean shouldHideHeader(HttpHeader header) { + return headersToHide.stream() + .anyMatch(h -> h.equalsIgnoreCase(header.name())); + } + + private String formatHeaderAsCookieIfNecessary(HttpHeader header) { + return isHeaderACookie(header) + ? formatCookieHeader(header) + : header.toString(); + } + + private boolean isHeaderACookie(HttpHeader header) { + return COOKIE_HEADER_NAMES.contains(header.name().toLowerCase()); + } + + private String formatCookieHeader(HttpHeader header) { + String cookies = RequestCookie.decode(header.value()).stream() + .map(this::hideOrFormatCookie) + .collect(Collectors.joining(";")); + + return header.name() + "=" + cookies; + } + + private String hideOrFormatCookie(RequestCookie cookie) { + return shouldHideCookie(cookie) + ? cookie.name() + "=****" + : cookie.toString(); + } + + private boolean shouldHideCookie(RequestCookie cookie) { + return cookiesToHide.contains(cookie.name()); + } + +} diff --git a/components/common/src/main/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatter.java b/components/common/src/main/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatter.java new file mode 100644 index 0000000000..eb37e45fa4 --- /dev/null +++ b/components/common/src/main/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatter.java @@ -0,0 +1,96 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.common.format; + +import com.hotels.styx.api.HttpHeaders; +import com.hotels.styx.api.HttpMethod; +import com.hotels.styx.api.HttpRequest; +import com.hotels.styx.api.HttpResponse; +import com.hotels.styx.api.HttpResponseStatus; +import com.hotels.styx.api.HttpVersion; +import com.hotels.styx.api.LiveHttpRequest; +import com.hotels.styx.api.LiveHttpResponse; +import com.hotels.styx.api.Url; + +import static java.util.Objects.requireNonNull; + +/** + * Formats requests and responses so that the headers are sanitised using the provided {@link SanitisedHttpHeaderFormatter}. + */ +public class SanitisedHttpMessageFormatter implements HttpMessageFormatter { + + private static final String NULL = "null"; + private final SanitisedHttpHeaderFormatter sanitisedHttpHeaderFormatter; + + public SanitisedHttpMessageFormatter(SanitisedHttpHeaderFormatter sanitisedHttpHeaderFormatter) { + this.sanitisedHttpHeaderFormatter = requireNonNull(sanitisedHttpHeaderFormatter); + } + + @Override + public String formatRequest(HttpRequest request) { + return request == null ? NULL + : formatRequest( + request.version(), + request.method(), + request.url(), + request.id(), + request.headers()); + } + + @Override + public String formatRequest(LiveHttpRequest request) { + return request == null ? NULL + : formatRequest( + request.version(), + request.method(), + request.url(), + request.id(), + request.headers()); + } + + @Override + public String formatResponse(HttpResponse response) { + return response == null ? NULL + : formatResponse( + response.version(), + response.status(), + response.headers()); + } + + @Override + public String formatResponse(LiveHttpResponse response) { + return response == null ? NULL + : formatResponse( + response.version(), + response.status(), + response.headers()); + } + + private String formatRequest(HttpVersion version, HttpMethod method, Url url, Object id, HttpHeaders headers) { + return "{version=" + version + + ", method=" + method + + ", uri=" + url + + ", headers=[" + sanitisedHttpHeaderFormatter.format(headers) + + "], id=" + id + "}"; + } + + private String formatResponse(HttpVersion version, HttpResponseStatus status, HttpHeaders headers) { + return "{version=" + version + + ", status=" + status + + ", headers=[" + sanitisedHttpHeaderFormatter.format(headers) + "]}"; + } + +} diff --git a/components/common/src/main/java/com/hotels/styx/common/logging/HttpRequestMessageLogger.java b/components/common/src/main/java/com/hotels/styx/common/logging/HttpRequestMessageLogger.java index 12f20f7924..7134231761 100644 --- a/components/common/src/main/java/com/hotels/styx/common/logging/HttpRequestMessageLogger.java +++ b/components/common/src/main/java/com/hotels/styx/common/logging/HttpRequestMessageLogger.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,35 +18,40 @@ import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.api.LiveHttpResponse; import com.hotels.styx.api.extension.Origin; +import com.hotels.styx.common.format.HttpMessageFormatter; import org.slf4j.Logger; +import static java.util.Objects.requireNonNull; import static org.slf4j.LoggerFactory.getLogger; /** * Logs client side requests and responses when enabled. Disabled by default. */ public class HttpRequestMessageLogger { + private final Logger logger; private final boolean longFormatEnabled; + private final HttpMessageFormatter httpMessageFormatter; - public HttpRequestMessageLogger(String name, boolean longFormatEnabled) { + public HttpRequestMessageLogger(String name, boolean longFormatEnabled, HttpMessageFormatter httpMessageFormatter) { this.longFormatEnabled = longFormatEnabled; - logger = getLogger(name); + this.httpMessageFormatter = requireNonNull(httpMessageFormatter); + this.logger = getLogger(name); } public void logRequest(LiveHttpRequest request, Origin origin) { if (request == null) { - logger.warn("requestId=N/A, request=null, origin={}", origin); + logger.warn("requestId=N/A, origin={}, request=null", origin); } else { - logger.info("requestId={}, request={}", new Object[] {request.id(), information(request, origin, longFormatEnabled)}); + logger.info("requestId={}, origin={}, request={}", new Object[] {request.id(), origin, requestAsString(request)}); } } public void logRequest(LiveHttpRequest request, Origin origin, boolean secure) { if (request == null) { - logger.warn("requestId=N/A, request=null, origin={}", origin); + logger.warn("requestId=N/A, origin={}, request=null", origin); } else { - logger.info("requestId={}, secure={}, request={}", new Object[] {request.id(), secure, information(request, origin, longFormatEnabled)}); + logger.info("requestId={}, secure={}, origin={}, request={}", new Object[] {request.id(), secure, origin, requestAsString(request)}); } } @@ -54,7 +59,7 @@ public void logResponse(LiveHttpRequest request, LiveHttpResponse response) { if (response == null) { logger.warn("requestId={}, response=null", id(request)); } else { - logger.info("requestId={}, response={}", id(request), information(response, longFormatEnabled)); + logger.info("requestId={}, response={}", id(request), responseAsString(response)); } } @@ -62,57 +67,20 @@ public void logResponse(LiveHttpRequest request, LiveHttpResponse response, bool if (response == null) { logger.warn("requestId={}, response=null", id(request)); } else { - logger.info("requestId={}, secure={}, response={}", new Object[] {id(request), secure, information(response, longFormatEnabled)}); + logger.info("requestId={}, secure={}, response={}", new Object[] {id(request), secure, responseAsString(response)}); } } - private static Object id(LiveHttpRequest request) { - return request != null ? request.id() : null; + private String requestAsString(LiveHttpRequest request) { + return longFormatEnabled ? httpMessageFormatter.formatRequest(request) : request.toString(); } - private static Info information(LiveHttpResponse response, boolean longFormatEnabled) { - Info info = new Info().add("status", response.status()); - - if (longFormatEnabled) { - info.add("headers", response.headers()); - } - return info; + private String responseAsString(LiveHttpResponse response) { + return longFormatEnabled ? httpMessageFormatter.formatResponse(response) : response.toString(); } - private static Info information(LiveHttpRequest request, Origin origin, boolean longFormatEnabled) { - Info info = new Info() - .add("method", request.method()) - .add("uri", request.url()) - .add("origin", origin != null ? origin.hostAndPortString() : "N/A"); - - if (longFormatEnabled) { - info.add("headers", request.headers()); - } - return info; + private static Object id(LiveHttpRequest request) { + return request != null ? request.id() : null; } - private static class Info { - private final StringBuilder sb = new StringBuilder(); - - public Info add(String variable, Object value) { - if (sb.length() > 0) { - sb.append(", "); - } - - sb.append(variable).append("="); - - if (value instanceof String) { - sb.append('"').append(value).append('"'); - } else { - sb.append(value); - } - - return this; - } - - @Override - public String toString() { - return "{" + sb + "}"; - } - } } diff --git a/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpHeaderFormatterTest.java b/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpHeaderFormatterTest.java new file mode 100644 index 0000000000..4f927fe5d1 --- /dev/null +++ b/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpHeaderFormatterTest.java @@ -0,0 +1,49 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.common.format; + +import com.hotels.styx.api.HttpHeaders; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class SanitisedHttpHeaderFormatterTest { + + @Test + public void formatShouldFormatRequest() { + + HttpHeaders headers = new HttpHeaders.Builder() + .add("header1", "a") + .add("header2", "b") + .add("header3", "c") + .add("header4", "d") + .add("COOKIE", "cookie1=e;cookie2=f;") + .add("SET-COOKIE", "cookie3=g;cookie4=h;") + .build(); + + List headersToHide = Arrays.asList("HEADER1", "HEADER3"); + List cookiesToHide = Arrays.asList("cookie2", "cookie4"); + String formattedHeaders = new SanitisedHttpHeaderFormatter(headersToHide, cookiesToHide).format(headers); + + assertThat(formattedHeaders, + is("header1=****, header2=b, header3=****, header4=d, COOKIE=cookie1=e;cookie2=****, SET-COOKIE=cookie3=g;cookie4=****")); + } + +} \ No newline at end of file diff --git a/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatterTest.java b/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatterTest.java new file mode 100644 index 0000000000..81ca869835 --- /dev/null +++ b/components/common/src/test/java/com/hotels/styx/common/format/SanitisedHttpMessageFormatterTest.java @@ -0,0 +1,88 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.common.format; + +import com.hotels.styx.api.HttpRequest; +import com.hotels.styx.api.HttpResponse; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.hotels.styx.api.HttpRequest.get; +import static com.hotels.styx.api.HttpVersion.HTTP_1_1; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; + +public class SanitisedHttpMessageFormatterTest { + + private static final HttpRequest httpRequest = get("/") + .version(HTTP_1_1) + .header("HeaderName", "HeaderValue") + .build(); + + private static final HttpResponse httpResponse = new HttpResponse.Builder() + .version(HTTP_1_1) + .header("HeaderName", "HeaderValue") + .build(); + + private static final String FORMATTED_HEADERS = "headers"; + private static final String HTTP_REQUEST_PATTERN = "\\{version=HTTP/1.1, method=GET, uri=/, headers=\\[" + FORMATTED_HEADERS + "\\], id=[a-zA-Z0-9-]*}"; + private static final String HTTP_RESPONSE_PATTERN = "\\{version=HTTP/1.1, status=200 OK, headers=\\[" + FORMATTED_HEADERS + "\\]}"; + + @Mock + private SanitisedHttpHeaderFormatter sanitisedHttpHeaderFormatter; + + private SanitisedHttpMessageFormatter sanitisedHttpMessageFormatter; + + @BeforeClass + public void setup() { + MockitoAnnotations.initMocks(this); + sanitisedHttpMessageFormatter = new SanitisedHttpMessageFormatter(sanitisedHttpHeaderFormatter); + when(sanitisedHttpHeaderFormatter.format(any())).thenReturn(FORMATTED_HEADERS); + } + + @Test + public void shouldFormatHttpRequest() { + String formattedRequest = sanitisedHttpMessageFormatter.formatRequest(httpRequest); + assertMatchesRegex(formattedRequest, HTTP_REQUEST_PATTERN); + } + + @Test + public void shouldFormatLiveHttpRequest() { + String formattedRequest = sanitisedHttpMessageFormatter.formatRequest(httpRequest.stream()); + assertMatchesRegex(formattedRequest, HTTP_REQUEST_PATTERN); + } + + @Test + public void shouldFormatHttpResponse() { + String formattedResponse = sanitisedHttpMessageFormatter.formatResponse(httpResponse); + assertMatchesRegex(formattedResponse, HTTP_RESPONSE_PATTERN); + } + + @Test + public void shouldFormatLiveHttpResponse() { + String formattedResponse = sanitisedHttpMessageFormatter.formatResponse(httpResponse.stream()); + assertMatchesRegex(formattedResponse, HTTP_RESPONSE_PATTERN); + } + + private void assertMatchesRegex(String actual, String expected) { + assertTrue(actual.matches(expected), + "\n\nPattern to match: " + expected + "\nActual result: " + actual + "\n\n"); + } + +} \ No newline at end of file diff --git a/components/proxy/src/main/java/com/hotels/styx/BuiltInInterceptors.java b/components/proxy/src/main/java/com/hotels/styx/BuiltInInterceptors.java index 3ad115f5eb..57b5791962 100644 --- a/components/proxy/src/main/java/com/hotels/styx/BuiltInInterceptors.java +++ b/components/proxy/src/main/java/com/hotels/styx/BuiltInInterceptors.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.hotels.styx.api.HttpInterceptor; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.proxy.interceptors.ConfigurationContextResolverInterceptor; import com.hotels.styx.proxy.interceptors.HopByHopHeadersRemovingInterceptor; import com.hotels.styx.proxy.interceptors.HttpMessageLoggingInterceptor; @@ -36,7 +37,7 @@ final class BuiltInInterceptors { private BuiltInInterceptors() { } - static List internalStyxInterceptors(StyxConfig config) { + static List internalStyxInterceptors(StyxConfig config, HttpMessageFormatter httpMessageFormatter) { ImmutableList.Builder builder = ImmutableList.builder(); boolean loggingEnabled = config.get("request-logging.inbound.enabled", Boolean.class) @@ -46,7 +47,7 @@ static List internalStyxInterceptors(StyxConfig config) { .orElse(false); if (loggingEnabled) { - builder.add(new HttpMessageLoggingInterceptor(longFormatEnabled)); + builder.add(new HttpMessageLoggingInterceptor(longFormatEnabled, httpMessageFormatter)); } builder.add(new TcpTunnelRequestRejector()) diff --git a/components/proxy/src/main/java/com/hotels/styx/Environment.java b/components/proxy/src/main/java/com/hotels/styx/Environment.java index 48156d511b..40465d9b3b 100644 --- a/components/proxy/src/main/java/com/hotels/styx/Environment.java +++ b/components/proxy/src/main/java/com/hotels/styx/Environment.java @@ -18,6 +18,8 @@ import com.google.common.eventbus.EventBus; import com.hotels.styx.api.MetricRegistry; import com.hotels.styx.api.metrics.codahale.CodaHaleMetricRegistry; +import com.hotels.styx.common.format.DefaultHttpMessageFormatter; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.configstore.ConfigStore; import com.hotels.styx.proxy.HttpErrorStatusCauseLogger; import com.hotels.styx.proxy.HttpErrorStatusMetrics; @@ -38,16 +40,20 @@ public final class Environment implements com.hotels.styx.api.Environment { private final StyxConfig configuration; private final HttpErrorStatusListener httpErrorStatusListener; private final ServerEnvironment serverEnvironment; + private final HttpMessageFormatter httpMessageFormatter; private Environment(Builder builder) { this.eventBus = firstNonNull(builder.eventBus, () -> new EventBus("Styx")); this.configStore = new ConfigStore(); - this.configuration = requireNonNull(builder.configuration); + this.configuration = builder.configuration; this.version = firstNonNull(builder.version, Version::newVersion); this.serverEnvironment = new ServerEnvironment(firstNonNull(builder.metricRegistry, CodaHaleMetricRegistry::new)); + this.httpMessageFormatter = builder.httpMessageFormatter; - this.httpErrorStatusListener = HttpErrorStatusListener.compose(new HttpErrorStatusCauseLogger(), new HttpErrorStatusMetrics(serverEnvironment.metricRegistry())); + this.httpErrorStatusListener = HttpErrorStatusListener.compose( + new HttpErrorStatusCauseLogger(httpMessageFormatter), + new HttpErrorStatusMetrics(serverEnvironment.metricRegistry())); } // prevent unnecessary construction of defaults @@ -95,6 +101,9 @@ public ServerEnvironment serverEnvironment() { return serverEnvironment; } + public HttpMessageFormatter httpMessageFormatter() { + return httpMessageFormatter; + } /** * Builder for {@link com.hotels.styx.Environment}. @@ -104,6 +113,7 @@ public static class Builder { private Version version; private EventBus eventBus; private StyxConfig configuration = StyxConfig.defaultConfig(); + private HttpMessageFormatter httpMessageFormatter = new DefaultHttpMessageFormatter(); public Builder configuration(StyxConfig configuration) { this.configuration = requireNonNull(configuration); @@ -125,6 +135,11 @@ public Builder eventBus(EventBus eventBus) { return this; } + public Builder httpMessageFormatter(HttpMessageFormatter httpMessageFormatter) { + this.httpMessageFormatter = requireNonNull(httpMessageFormatter); + return this; + } + public Environment build() { return new Environment(this); } diff --git a/components/proxy/src/main/java/com/hotels/styx/ServerConfigSchema.java b/components/proxy/src/main/java/com/hotels/styx/ServerConfigSchema.java index f22d8afbe8..b64995cbd7 100644 --- a/components/proxy/src/main/java/com/hotels/styx/ServerConfigSchema.java +++ b/components/proxy/src/main/java/com/hotels/styx/ServerConfigSchema.java @@ -111,7 +111,9 @@ final class ServerConfigSchema { optional("request-logging", object( optional("inbound", logFormatSchema), optional("outbound", logFormatSchema), - atLeastOne("inbound", "outbound") + atLeastOne("inbound", "outbound"), + optional("hideHeaders", list(string())), + optional("hideCookies", list(string())) )), optional("styxHeaders", object( optional("styxInfo", object( diff --git a/components/proxy/src/main/java/com/hotels/styx/StyxPipelineFactory.java b/components/proxy/src/main/java/com/hotels/styx/StyxPipelineFactory.java index f251a257b4..60ec87dd5c 100644 --- a/components/proxy/src/main/java/com/hotels/styx/StyxPipelineFactory.java +++ b/components/proxy/src/main/java/com/hotels/styx/StyxPipelineFactory.java @@ -81,7 +81,7 @@ public HttpHandler create(StyxServerComponents config) { boolean requestTracking = environment.configuration().get("requestTracking", Boolean.class).orElse(false); return new HttpInterceptorPipeline( - internalStyxInterceptors(environment.styxConfig()), + internalStyxInterceptors(environment.styxConfig(), environment.httpMessageFormatter()), configuredPipeline(builtinRoutingObjects), requestTracking); } diff --git a/components/proxy/src/main/java/com/hotels/styx/StyxServer.java b/components/proxy/src/main/java/com/hotels/styx/StyxServer.java index 6d1d41f3a9..9e36dec722 100644 --- a/components/proxy/src/main/java/com/hotels/styx/StyxServer.java +++ b/components/proxy/src/main/java/com/hotels/styx/StyxServer.java @@ -147,11 +147,11 @@ private static String readYaml(Resource resource) { throw new RuntimeException(e); } } - private final HttpServer proxyServer; - private final HttpServer adminServer; + private final HttpServer adminServer; private final ServiceManager serviceManager; + private final Stopwatch stopwatch; public StyxServer(StyxServerComponents config) { diff --git a/components/proxy/src/main/java/com/hotels/styx/proxy/BackendServicesRouter.java b/components/proxy/src/main/java/com/hotels/styx/proxy/BackendServicesRouter.java index fe04c6f3ec..6d12ff1012 100644 --- a/components/proxy/src/main/java/com/hotels/styx/proxy/BackendServicesRouter.java +++ b/components/proxy/src/main/java/com/hotels/styx/proxy/BackendServicesRouter.java @@ -33,7 +33,6 @@ import com.hotels.styx.client.OriginStatsFactory; import com.hotels.styx.client.OriginStatsFactory.CachingOriginStatsFactory; import com.hotels.styx.client.OriginsInventory; -import com.hotels.styx.client.StyxHeaderConfig; import com.hotels.styx.client.StyxHostHttpClient; import com.hotels.styx.client.StyxHttpClient; import com.hotels.styx.client.connectionpool.ConnectionPool; @@ -44,6 +43,7 @@ import com.hotels.styx.client.healthcheck.OriginHealthStatusMonitorFactory; import com.hotels.styx.client.healthcheck.UrlRequestHealthCheck; import com.hotels.styx.client.netty.connectionpool.NettyConnectionFactory; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.server.HttpRouter; import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; @@ -131,7 +131,8 @@ public void onChange(Registry.Changes changes) { requestLoggingEnabled, longFormat, originStatsFactory, - poolSettings.connectionExpirationSeconds()); + poolSettings.connectionExpirationSeconds(), + environment.httpMessageFormatter()); ConnectionPool.Factory connectionPoolFactory = new SimpleConnectionPoolFactory.Builder() .connectionFactory(connectionFactory) @@ -139,14 +140,7 @@ public void onChange(Registry.Changes changes) { .metricRegistry(originsMetrics) .build(); - StyxHttpClient healthCheckClient = healthCheckClient(backendService); - - OriginHealthStatusMonitor healthStatusMonitor = healthStatusMonitor(backendService, healthCheckClient); - - StyxHostHttpClient.Factory hostClientFactory = (ConnectionPool connectionPool) -> { - StyxHeaderConfig headerConfig = environment.styxConfig().styxHeaderConfig(); - return StyxHostHttpClient.create(connectionPool); - }; + OriginHealthStatusMonitor healthStatusMonitor = healthStatusMonitor(backendService); OriginsInventory inventory = new OriginsInventory.Builder(backendService.id()) .eventBus(environment.eventBus()) @@ -154,7 +148,7 @@ public void onChange(Registry.Changes changes) { .connectionPoolFactory(connectionPoolFactory) .originHealthMonitor(healthStatusMonitor) .initialOrigins(backendService.origins()) - .hostClientFactory(hostClientFactory) + .hostClientFactory(StyxHostHttpClient::create) .build(); pipeline = new ProxyToClientPipeline(newClientHandler(backendService, inventory, originStatsFactory), () -> { @@ -167,7 +161,7 @@ public void onChange(Registry.Changes changes) { }); } - private OriginHealthStatusMonitor healthStatusMonitor(BackendService backendService, StyxHttpClient healthCheckClient) { + private OriginHealthStatusMonitor healthStatusMonitor(BackendService backendService) { return new OriginHealthStatusMonitorFactory() .create(backendService.id(), backendService.healthCheckConfig(), @@ -175,7 +169,7 @@ private OriginHealthStatusMonitor healthStatusMonitor(BackendService backendServ backendService.id(), environment.metricRegistry(), backendService.healthCheckConfig()), - healthCheckClient); + healthCheckClient(backendService)); } private StyxHttpClient healthCheckClient(BackendService backendService) { @@ -196,7 +190,8 @@ private Connection.Factory connectionFactory( boolean requestLoggingEnabled, boolean longFormat, OriginStatsFactory originStatsFactory, - long connectionExpiration) { + long connectionExpiration, + HttpMessageFormatter httpMessageFormatter) { Connection.Factory factory = new NettyConnectionFactory.Builder() .nettyEventLoop(nettyEventLoopGroup, socketChannelClass) @@ -207,6 +202,7 @@ private Connection.Factory connectionFactory( .responseTimeoutMillis(responseTimeoutMillis) .requestLoggingEnabled(requestLoggingEnabled) .longFormat(longFormat) + .httpMessageFormatter(httpMessageFormatter) .build() ) .tlsSettings(tlsSettings) diff --git a/components/proxy/src/main/java/com/hotels/styx/proxy/HttpErrorStatusCauseLogger.java b/components/proxy/src/main/java/com/hotels/styx/proxy/HttpErrorStatusCauseLogger.java index 7ac255cfa7..ea2f4ce717 100644 --- a/components/proxy/src/main/java/com/hotels/styx/proxy/HttpErrorStatusCauseLogger.java +++ b/components/proxy/src/main/java/com/hotels/styx/proxy/HttpErrorStatusCauseLogger.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,21 +15,29 @@ */ package com.hotels.styx.proxy; +import com.hotels.styx.api.HttpResponseStatus; import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.api.LiveHttpResponse; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.server.HttpErrorStatusListener; -import com.hotels.styx.api.HttpResponseStatus; import org.slf4j.Logger; import java.net.InetSocketAddress; +import static java.util.Objects.requireNonNull; import static org.slf4j.LoggerFactory.getLogger; /** * Wrapper for {@link HttpErrorStatusListener} that also logs {@link Throwable}s. */ public class HttpErrorStatusCauseLogger implements HttpErrorStatusListener { + private static final Logger LOG = getLogger(HttpErrorStatusCauseLogger.class); + private final HttpMessageFormatter formatter; + + public HttpErrorStatusCauseLogger(HttpMessageFormatter formatter) { + this.formatter = requireNonNull(formatter); + } @Override public void proxyErrorOccurred(HttpResponseStatus status, Throwable cause) { @@ -44,7 +52,7 @@ public void proxyErrorOccurred(HttpResponseStatus status, Throwable cause) { @Override public void proxyErrorOccurred(LiveHttpRequest request, InetSocketAddress clientAddress, HttpResponseStatus status, Throwable cause) { if (status.code() == 500) { - LOG.error("Failure status=\"{}\" during request={}, clientAddress={}", new Object[]{status, request, clientAddress, cause}); + LOG.error("Failure status=\"{}\" during request={}, clientAddress={}", new Object[]{status, formatter.formatRequest(request), clientAddress, cause}); } else { proxyErrorOccurred(status, cause); } @@ -57,12 +65,12 @@ public void proxyErrorOccurred(Throwable cause) { @Override public void proxyWriteFailure(LiveHttpRequest request, LiveHttpResponse response, Throwable cause) { - LOG.error("Error writing response. request={}, response={}, cause={}", new Object[]{request, response, cause}); + LOG.error("Error writing response. request={}, response={}, cause={}", new Object[]{formatter.formatRequest(request), formatter.formatResponse(response), cause}); } @Override public void proxyingFailure(LiveHttpRequest request, LiveHttpResponse response, Throwable cause) { - LOG.error("Error proxying request. request={} response={} cause={}", new Object[]{request, response, cause}); + LOG.error("Error proxying request. request={} response={} cause={}", new Object[]{formatter.formatRequest(request), formatter.formatResponse(response), cause}); } private static String withoutStackTrace(Throwable cause) { diff --git a/components/proxy/src/main/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptor.java b/components/proxy/src/main/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptor.java index 449eb16988..c26b5f6632 100644 --- a/components/proxy/src/main/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptor.java +++ b/components/proxy/src/main/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptor.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,10 +15,11 @@ */ package com.hotels.styx.proxy.interceptors; -import com.hotels.styx.api.HttpInterceptor; -import com.hotels.styx.api.LiveHttpResponse; import com.hotels.styx.api.Eventual; +import com.hotels.styx.api.HttpInterceptor; import com.hotels.styx.api.LiveHttpRequest; +import com.hotels.styx.api.LiveHttpResponse; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.common.logging.HttpRequestMessageLogger; /** @@ -28,8 +29,8 @@ public class HttpMessageLoggingInterceptor implements HttpInterceptor { private final HttpRequestMessageLogger logger; - public HttpMessageLoggingInterceptor(boolean longFormatEnabled) { - this.logger = new HttpRequestMessageLogger("com.hotels.styx.http-messages.inbound", longFormatEnabled); + public HttpMessageLoggingInterceptor(boolean longFormatEnabled, HttpMessageFormatter httpMessageFormatter) { + this.logger = new HttpRequestMessageLogger("com.hotels.styx.http-messages.inbound", longFormatEnabled, httpMessageFormatter); } @Override diff --git a/components/proxy/src/main/java/com/hotels/styx/routing/handlers/StaticResponseHandler.java b/components/proxy/src/main/java/com/hotels/styx/routing/handlers/StaticResponseHandler.java index d87bae8ed4..c2b88cbd0a 100644 --- a/components/proxy/src/main/java/com/hotels/styx/routing/handlers/StaticResponseHandler.java +++ b/components/proxy/src/main/java/com/hotels/styx/routing/handlers/StaticResponseHandler.java @@ -19,6 +19,7 @@ import com.hotels.styx.api.Buffer; import com.hotels.styx.api.ByteStream; import com.hotels.styx.api.Eventual; +import com.hotels.styx.api.HttpHeaders; import com.hotels.styx.api.HttpInterceptor; import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.api.LiveHttpResponse; @@ -29,12 +30,14 @@ import com.hotels.styx.routing.config.StyxObjectDefinition; import reactor.core.publisher.Flux; +import java.util.Collections; import java.util.List; import static com.hotels.styx.api.HttpResponseStatus.statusWithCode; import static com.hotels.styx.api.LiveHttpResponse.response; import static com.hotels.styx.config.schema.SchemaDsl.field; import static com.hotels.styx.config.schema.SchemaDsl.integer; +import static com.hotels.styx.config.schema.SchemaDsl.list; import static com.hotels.styx.config.schema.SchemaDsl.object; import static com.hotels.styx.config.schema.SchemaDsl.optional; import static com.hotels.styx.config.schema.SchemaDsl.string; @@ -47,29 +50,52 @@ public class StaticResponseHandler implements RoutingObject { public static final Schema.FieldType SCHEMA = object( field("status", integer()), - optional("content", string())); + optional("content", string()), + optional("headers", list(object( + field("name", string()), + field("value", string()) + )))); private final int status; private final String text; + private final HttpHeaders headers; - public StaticResponseHandler(int status, String text) { + public StaticResponseHandler(int status, String text, HttpHeaders headers) { this.status = status; this.text = text; + this.headers = headers; } @Override public Eventual handle(LiveHttpRequest request, HttpInterceptor.Context context) { - return Eventual.of(response(statusWithCode(status)).body(new ByteStream(Flux.just(new Buffer(text, UTF_8)))).build()); + return Eventual.of(response(statusWithCode(status)) + .body(new ByteStream(Flux.just(new Buffer(text, UTF_8)))) + .headers(headers) + .build()); } private static class StaticResponseConfig { private final int status; private final String response; + private final List headers; public StaticResponseConfig(@JsonProperty("status") int status, - @JsonProperty("content") String content) { + @JsonProperty("content") String content, + @JsonProperty("headers") List headers) { this.status = status; this.response = content; + this.headers = headers; + } + } + + private static class HttpHeaderConfig { + private String name; + private String value; + + public HttpHeaderConfig(@JsonProperty("name") String name, + @JsonProperty("value") String value) { + this.name = name; + this.value = value; } } @@ -83,7 +109,20 @@ public RoutingObject build(List fullName, Context context, StyxObjectDef StaticResponseConfig config = new JsonNodeConfig(configBlock.config()) .as(StaticResponseConfig.class); - return new StaticResponseHandler(config.status, config.response); + HttpHeaders httpHeaders = buildHttpHeaders(config); + return new StaticResponseHandler(config.status, config.response, httpHeaders); + } + + private HttpHeaders buildHttpHeaders(StaticResponseConfig config) { + List headerConfig = config.headers == null + ? Collections.emptyList() + : config.headers; + + HttpHeaders.Builder headersBuilder = new HttpHeaders.Builder(); + for (HttpHeaderConfig header : headerConfig) { + headersBuilder.add(header.name, header.value); + } + return headersBuilder.build(); } } } diff --git a/components/proxy/src/main/java/com/hotels/styx/startup/StyxServerComponents.java b/components/proxy/src/main/java/com/hotels/styx/startup/StyxServerComponents.java index 42fa52af8a..36ee5e1283 100644 --- a/components/proxy/src/main/java/com/hotels/styx/startup/StyxServerComponents.java +++ b/components/proxy/src/main/java/com/hotels/styx/startup/StyxServerComponents.java @@ -31,6 +31,8 @@ import com.hotels.styx.api.metrics.codahale.CodaHaleMetricRegistry; import com.hotels.styx.api.plugins.spi.Plugin; import com.hotels.styx.client.netty.eventloop.PlatformAwareClientEventLoopGroupFactory; +import com.hotels.styx.common.format.SanitisedHttpHeaderFormatter; +import com.hotels.styx.common.format.SanitisedHttpMessageFormatter; import com.hotels.styx.infrastructure.configuration.yaml.JsonNodeConfig; import com.hotels.styx.proxy.plugin.NamedPlugin; import com.hotels.styx.routing.RoutingMetadataDecorator; @@ -62,6 +64,7 @@ import static com.hotels.styx.startup.ServicesLoader.SERVICES_FROM_CONFIG; import static com.hotels.styx.startup.StyxServerComponents.LoggingSetUp.DO_NOT_MODIFY; import static com.hotels.styx.startup.extensions.PluginLoadingForStartup.loadPlugins; +import static java.util.Collections.emptyList; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadExecutor; import static java.util.stream.Collectors.toList; @@ -205,12 +208,20 @@ public StartupConfig startupConfig() { return startupConfig; } - private static Environment newEnvironment(StyxConfig styxConfig, MetricRegistry metricRegistry) { + private static Environment newEnvironment(StyxConfig config, MetricRegistry metricRegistry) { + + SanitisedHttpHeaderFormatter headerFormatter = new SanitisedHttpHeaderFormatter( + config.get("request-logging.hideHeaders", List.class).orElse(emptyList()), + config.get("request-logging.hideCookies", List.class).orElse(emptyList())); + + SanitisedHttpMessageFormatter sanitisedHttpMessageFormatter = new SanitisedHttpMessageFormatter(headerFormatter); + return new Environment.Builder() - .configuration(styxConfig) + .configuration(config) .metricRegistry(metricRegistry) .buildInfo(readBuildInfo()) .eventBus(new AsyncEventBus("styx", newSingleThreadExecutor())) + .httpMessageFormatter(sanitisedHttpMessageFormatter) .build(); } diff --git a/components/proxy/src/test/java/com/hotels/styx/StyxConfigTest.java b/components/proxy/src/test/java/com/hotels/styx/StyxConfigTest.java index 3321fc3de4..28b337ad93 100644 --- a/components/proxy/src/test/java/com/hotels/styx/StyxConfigTest.java +++ b/components/proxy/src/test/java/com/hotels/styx/StyxConfigTest.java @@ -18,6 +18,9 @@ import com.hotels.styx.proxy.ProxyServerConfig; import org.testng.annotations.Test; +import java.util.Collections; +import java.util.List; + import static com.hotels.styx.support.matchers.IsOptional.isValue; import static java.lang.Runtime.getRuntime; import static org.hamcrest.MatcherAssert.assertThat; @@ -59,6 +62,27 @@ public void initializesFromConfigurationSource() { assertThat(styxConfig.get("metrics.reporting.prefix", String.class).get(), is("STYXHPT")); } + @Test + public void readsListsOfHeadersAndCookiesToHide() { + String yaml = + "request-logging:\n" + + " hideHeaders:\n" + + " - header1\n" + + " - header2\n" + + " hideCookies:\n" + + " - cookie1\n" + + " - cookie2\n"; + + StyxConfig styxConfig = StyxConfig.fromYaml(yaml, false); + List headersToHide = styxConfig.get("request-logging.hideHeaders", List.class).orElse(Collections.emptyList()); + List cookiesToHide = styxConfig.get("request-logging.hideCookies", List.class).orElse(Collections.emptyList()); + + assertThat(headersToHide.get(0), is("header1")); + assertThat(headersToHide.get(1), is("header2")); + assertThat(cookiesToHide.get(0), is("cookie1")); + assertThat(cookiesToHide.get(1), is("cookie2")); + } + @Test public void readsBossThreadsCountFromConfigurationSource() { String yaml = "" + diff --git a/components/proxy/src/test/java/com/hotels/styx/proxy/HttpErrorStatusCauseLoggerTest.java b/components/proxy/src/test/java/com/hotels/styx/proxy/HttpErrorStatusCauseLoggerTest.java index 85f7891eb1..7ac32cbc17 100644 --- a/components/proxy/src/test/java/com/hotels/styx/proxy/HttpErrorStatusCauseLoggerTest.java +++ b/components/proxy/src/test/java/com/hotels/styx/proxy/HttpErrorStatusCauseLoggerTest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,8 +16,12 @@ package com.hotels.styx.proxy; import com.hotels.styx.api.LiveHttpRequest; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.support.matchers.LoggingTestSupport; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -29,15 +33,29 @@ import static com.hotels.styx.support.matchers.LoggingEventMatcher.loggingEvent; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; public class HttpErrorStatusCauseLoggerTest { - HttpErrorStatusCauseLogger httpErrorStatusCauseLogger; - LoggingTestSupport loggingTestSupport; + + private static final String FORMATTED_REQUEST = "request"; + + private HttpErrorStatusCauseLogger httpErrorStatusCauseLogger; + private LoggingTestSupport loggingTestSupport; + + @Mock + private HttpMessageFormatter httpMessageFormatter; + + @BeforeClass + public void setup() { + MockitoAnnotations.initMocks(this); + when(httpMessageFormatter.formatRequest(any(LiveHttpRequest.class))).thenReturn(FORMATTED_REQUEST); + } @BeforeMethod public void setUp() { loggingTestSupport = new LoggingTestSupport(HttpErrorStatusCauseLogger.class); - httpErrorStatusCauseLogger = new HttpErrorStatusCauseLogger(); + httpErrorStatusCauseLogger = new HttpErrorStatusCauseLogger(httpMessageFormatter); } @AfterMethod @@ -81,7 +99,7 @@ public void logsInternalServerErrorWithRequest() { assertThat(loggingTestSupport.log(), hasItem( loggingEvent( ERROR, - "Failure status=\"500 Internal Server Error\" during request=LiveHttpRequest\\{version=HTTP/1.1, method=GET, uri=/foo, headers=\\[\\], id=.*\\}, clientAddress=localhost:80", + "Failure status=\"500 Internal Server Error\" during request=" + FORMATTED_REQUEST + ", clientAddress=localhost:80", "java.lang.Exception", "This is just a test"))); } diff --git a/components/proxy/src/test/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptorTest.java b/components/proxy/src/test/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptorTest.java index bfc6f5b085..a9d3214a63 100644 --- a/components/proxy/src/test/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptorTest.java +++ b/components/proxy/src/test/java/com/hotels/styx/proxy/interceptors/HttpMessageLoggingInterceptorTest.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,11 +17,16 @@ import com.hotels.styx.api.Eventual; import com.hotels.styx.api.HttpInterceptor; +import com.hotels.styx.api.HttpVersion; import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.api.LiveHttpResponse; +import com.hotels.styx.common.format.HttpMessageFormatter; import com.hotels.styx.server.HttpInterceptorContext; import com.hotels.styx.support.matchers.LoggingTestSupport; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import reactor.core.publisher.Mono; @@ -35,15 +40,31 @@ import static com.hotels.styx.support.matchers.LoggingEventMatcher.loggingEvent; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; public class HttpMessageLoggingInterceptorTest { + + private static final String FORMATTED_REQUEST = "request"; + private static final String FORMATTED_RESPONSE = "response"; + private LoggingTestSupport responseLogSupport; private HttpMessageLoggingInterceptor interceptor; + @Mock + private HttpMessageFormatter httpMessageFormatter; + + @BeforeClass + public void setup() { + MockitoAnnotations.initMocks(this); + when(httpMessageFormatter.formatRequest(any(LiveHttpRequest.class))).thenReturn(FORMATTED_REQUEST); + when(httpMessageFormatter.formatResponse(any(LiveHttpResponse.class))).thenReturn(FORMATTED_RESPONSE); + } + @BeforeMethod public void before() { responseLogSupport = new LoggingTestSupport("com.hotels.styx.http-messages.inbound"); - interceptor = new HttpMessageLoggingInterceptor(true); + interceptor = new HttpMessageLoggingInterceptor(true, httpMessageFormatter); } @AfterMethod @@ -54,6 +75,7 @@ public void after() { @Test public void logsRequestsAndResponses() { LiveHttpRequest request = get("/") + .version(HttpVersion.HTTP_1_1) .header("ReqHeader", "ReqHeaderValue") .cookies(requestCookie("ReqCookie", "ReqCookieValue")) .build(); @@ -64,17 +86,14 @@ public void logsRequestsAndResponses() { .cookies(responseCookie("RespCookie", "RespCookieValue").build()) ))); - String requestPattern = "request=\\{method=GET, uri=/, origin=\"N/A\", headers=\\[ReqHeader=ReqHeaderValue, Cookie=ReqCookie=ReqCookieValue\\]\\}"; - String responsePattern = "response=\\{status=200 OK, headers=\\[RespHeader=RespHeaderValue\\, Set-Cookie=RespCookie=RespCookieValue]\\}"; - assertThat(responseLogSupport.log(), contains( - loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + requestPattern), - loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + responsePattern))); + loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, origin=null, request=" + FORMATTED_REQUEST), + loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, response=" + FORMATTED_RESPONSE))); } @Test public void logsRequestsAndResponsesShort() { - interceptor = new HttpMessageLoggingInterceptor(false); + interceptor = new HttpMessageLoggingInterceptor(false, httpMessageFormatter); LiveHttpRequest request = get("/") .header("ReqHeader", "ReqHeaderValue") .cookies(requestCookie("ReqCookie", "ReqCookieValue")) @@ -86,11 +105,11 @@ public void logsRequestsAndResponsesShort() { .cookies(responseCookie("RespCookie", "RespCookieValue").build()) ))); - String requestPattern = "request=\\{method=GET, uri=/, origin=\"N/A\"}"; - String responsePattern = "response=\\{status=200 OK}"; + String requestPattern = "request=\\{version=HTTP/1.1, method=GET, uri=/, id=" + request.id() + "\\}"; + String responsePattern = "response=\\{version=HTTP/1.1, status=200 OK\\}"; assertThat(responseLogSupport.log(), contains( - loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + requestPattern), + loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, origin=null, " + requestPattern), loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + responsePattern))); } @@ -103,12 +122,9 @@ public void logsSecureRequests() { consume(interceptor.intercept(request, chain(response(OK)))); - String requestPattern = "request=\\{method=GET, uri=/, origin=\"N/A\", headers=\\[ReqHeader=ReqHeaderValue, Cookie=ReqCookie=ReqCookieValue\\]\\}"; - String responsePattern = "response=\\{status=200 OK, headers=\\[\\]\\}"; - assertThat(responseLogSupport.log(), contains( - loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + requestPattern), - loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, " + responsePattern))); + loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, origin=null, request=" + FORMATTED_REQUEST), + loggingEvent(INFO, "requestId=" + request.id() + ", secure=true, response=" + FORMATTED_RESPONSE))); } diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/ServerConfigSchemaTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/ServerConfigSchemaTest.kt index 99207d6060..77b3a1f28d 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/ServerConfigSchemaTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/ServerConfigSchemaTest.kt @@ -112,6 +112,12 @@ class ServerConfigSchemaTest : DescribeSpec({ outbound: enabled: true longFormat: false + hideHeaders: + - header1 + - header2 + hideCookies: + - cookie1 + - cookie2 """.trimIndent() )) shouldBe (Optional.empty()) } diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt index be2868c891..dc92ad44c1 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/routing/handlers/LoadBalancingGroupTest.kt @@ -16,6 +16,7 @@ package com.hotels.styx.routing.handlers import com.hotels.styx.api.HttpHandler +import com.hotels.styx.api.HttpHeaders import com.hotels.styx.api.HttpRequest import com.hotels.styx.api.HttpRequest.get import com.hotels.styx.api.configuration.ObjectStore @@ -50,13 +51,14 @@ class LoadBalancingGroupTest : FeatureSpec() { feature("Load Balancing") { val factory = LoadBalancingGroup.Factory() val routeDb = StyxObjectStore() + val headers = HttpHeaders.Builder().build(); - routeDb.insert("appx-01", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-01"))) - routeDb.insert("appx-02", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-02"))) - routeDb.insert("appx-03", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-03"))) + routeDb.insert("appx-01", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-01", headers))) + routeDb.insert("appx-02", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-02", headers))) + routeDb.insert("appx-03", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-03", headers))) - routeDb.insert("appy-01", RoutingObjectRecord.create("HostProxy", setOf("appY"), mockk(), StaticResponseHandler(200, "appy-01"))) - routeDb.insert("appy-02", RoutingObjectRecord.create("HostProxy", setOf("appY"), mockk(), StaticResponseHandler(200, "appy-02"))) + routeDb.insert("appy-01", RoutingObjectRecord.create("HostProxy", setOf("appY"), mockk(), StaticResponseHandler(200, "appy-01", headers))) + routeDb.insert("appy-02", RoutingObjectRecord.create("HostProxy", setOf("appY"), mockk(), StaticResponseHandler(200, "appy-02", headers))) routeDb.watch().waitUntil { it.entrySet().size == 5 } @@ -90,9 +92,9 @@ class LoadBalancingGroupTest : FeatureSpec() { scenario("... and detects new origins") { val frequencies = mutableMapOf() - routeDb.insert("appx-04", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-04"))) - routeDb.insert("appx-05", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-05"))) - routeDb.insert("appx-06", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-06"))) + routeDb.insert("appx-04", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-04", headers))) + routeDb.insert("appx-05", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-05", headers))) + routeDb.insert("appx-06", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-06", headers))) routeDb.watch().waitUntil { it.entrySet().size == 8 } @@ -119,9 +121,9 @@ class LoadBalancingGroupTest : FeatureSpec() { scenario("... and detects replaced origins") { val frequencies = mutableMapOf() - routeDb.insert("appx-04", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-04-a"))) - routeDb.insert("appx-05", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-05-b"))) - routeDb.insert("appx-06", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-06-c"))) + routeDb.insert("appx-04", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-04-a", headers))) + routeDb.insert("appx-05", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-05-b", headers))) + routeDb.insert("appx-06", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-06-c", headers))) routeDb.watch().waitUntil { it["appx-06"].isPresent } @@ -156,8 +158,8 @@ class LoadBalancingGroupTest : FeatureSpec() { } scenario("... and exposes load balancing metric") { - routeDb.insert("appx-A", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-A"))) - routeDb.insert("appx-B", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-B"))) + routeDb.insert("appx-A", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-A", headers))) + routeDb.insert("appx-B", RoutingObjectRecord.create("HostProxy", setOf("appX"), mockk(), StaticResponseHandler(200, "appx-B", headers))) routeDb.watch().waitUntil { it.entrySet().size == 4 } diff --git a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt index 8b5c3c891f..0a66d02c03 100644 --- a/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt +++ b/components/proxy/src/test/kotlin/com/hotels/styx/services/HealthChecksTest.kt @@ -17,6 +17,7 @@ package com.hotels.styx.services import com.fasterxml.jackson.databind.JsonNode import com.hotels.styx.api.Eventual +import com.hotels.styx.api.HttpHeaders import com.hotels.styx.api.HttpInterceptor import com.hotels.styx.api.HttpRequest.get import com.hotels.styx.api.LiveHttpRequest @@ -49,8 +50,10 @@ import kotlin.system.measureTimeMillis class HealthChecksTest : FeatureSpec({ feature("Probe function") { + val headers = HttpHeaders.Builder().build(); + scenario("Returns true when object is responsive") { - val staticResponse = StaticResponseHandler(200, "Hello") + val staticResponse = StaticResponseHandler(200, "Hello", headers) urlProbe(get("/healthcheck.txt").build(), 1.seconds) .invoke(staticResponse) @@ -72,7 +75,7 @@ class HealthChecksTest : FeatureSpec({ } scenario("Returns false when responds with 4xx error code") { - val errorHandler = StaticResponseHandler(400, "Hello") + val errorHandler = StaticResponseHandler(400, "Hello", headers) urlProbe(get("/healthcheck.txt").build(), 100.milliseconds) .invoke(errorHandler) @@ -81,7 +84,7 @@ class HealthChecksTest : FeatureSpec({ } scenario("Returns false when responds with 5xx error code") { - val errorHandler = StaticResponseHandler(500, "Hello") + val errorHandler = StaticResponseHandler(500, "Hello", headers) urlProbe(get("/healthcheck.txt").build(), 100.milliseconds) .invoke(errorHandler) diff --git a/components/server/src/main/java/com/hotels/styx/server/netty/connectors/HttpResponseWriter.java b/components/server/src/main/java/com/hotels/styx/server/netty/connectors/HttpResponseWriter.java index d5c592bf49..22a8f753fe 100644 --- a/components/server/src/main/java/com/hotels/styx/server/netty/connectors/HttpResponseWriter.java +++ b/components/server/src/main/java/com/hotels/styx/server/netty/connectors/HttpResponseWriter.java @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -74,7 +74,6 @@ public CompletableFuture write(LiveHttpResponse response) { contentBytesWritten.get(), writeOpsAcked.get(), writeOps.get(), - response, writeOp.cause()}); future.completeExceptionally(writeOp.cause()); } diff --git a/components/server/src/test/java/com/hotels/styx/server/netty/codec/NettyToStyxRequestDecoderTest.java b/components/server/src/test/java/com/hotels/styx/server/netty/codec/NettyToStyxRequestDecoderTest.java index edc06dbddb..9e6d8e3c2a 100644 --- a/components/server/src/test/java/com/hotels/styx/server/netty/codec/NettyToStyxRequestDecoderTest.java +++ b/components/server/src/test/java/com/hotels/styx/server/netty/codec/NettyToStyxRequestDecoderTest.java @@ -17,9 +17,9 @@ import com.google.common.base.Strings; import com.hotels.styx.api.Buffer; +import com.hotels.styx.api.ByteStream; import com.hotels.styx.api.HttpHeader; import com.hotels.styx.api.HttpMethod; -import com.hotels.styx.api.ByteStream; import com.hotels.styx.api.LiveHttpRequest; import com.hotels.styx.server.BadRequestException; import com.hotels.styx.server.UniqueIdSupplier; @@ -309,7 +309,6 @@ public void shouldReleaseAlreadyReadBufferInCaseOfError() throws Exception { channel.writeInbound(httpContentOne); channel.pipeline().fireExceptionCaught(new RuntimeException("Some Error")); - assertThat(httpContentOne.refCnt(), Matchers.is(0)); } @@ -332,7 +331,10 @@ private FullHttpResponse send(HttpRequest request) { } private void assertThatHttpHeadersAreSame(Iterable headers, HttpHeaders headers1) { - assertThat(newArrayList(headers).toString(), is(newArrayList(headers1).toString())); + assertThat(newArrayList(headers).size(), is(headers1.size())); + for (HttpHeader header : headers) { + assertThat(header.value(), is(headers1.get(header.name()))); + } } private static HttpRequest newPostRequest(String path) { diff --git a/docs/user-guide/configure-overview.md b/docs/user-guide/configure-overview.md index 21279270b4..74189e2e34 100644 --- a/docs/user-guide/configure-overview.md +++ b/docs/user-guide/configure-overview.md @@ -125,12 +125,24 @@ request-logging: # Logs are produced on server and origin side, so there is an information on # how the server-side (inbound) and origin-side (outbound) request/response look like. # In long format log entry contains additionally headers and cookies. + # The hideHeaders and hideCookies options take a list of header or cookie names. + # Any header or cookie in these lists will be obfuscated in the logged message output + # e.g. + + headers=[Content-Type=****, Cookie=sessionID=****;samlToken=****] + + # Config example inbound: enabled: ${REQUEST_LOGGING_INBOUND_ENABLED:false} longFormat: ${REQUEST_LOGGING_INBOUND_LONG_FORMAT:false} outbound: enabled: ${REQUEST_LOGGING_OUTBOUND_ENABLED:false} longFormat: ${REQUEST_LOGGING_OUTBOUND_LONG_FORMAT:false} + hideHeaders: + - Content-Type + hideCookies: + - sessionID + - samlToken # Configures the names of the headers that Styx adds to messages it proxies (see headers.md) # If not configured, defaults will be used. diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadRequestsSpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadRequestsSpec.scala index 082e9acddf..d7f8c5a3d7 100644 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadRequestsSpec.scala +++ b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadRequestsSpec.scala @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadResponseFromOriginSpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadResponseFromOriginSpec.scala index 332feccbad..be9a131346 100644 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadResponseFromOriginSpec.scala +++ b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadResponseFromOriginSpec.scala @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpMessageLoggingSpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpMessageLoggingSpec.scala deleted file mode 100644 index c451e37fa5..0000000000 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpMessageLoggingSpec.scala +++ /dev/null @@ -1,141 +0,0 @@ -/* - Copyright (C) 2013-2018 Expedia Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package com.hotels.styx.proxy - -import java.nio.charset.StandardCharsets.UTF_8 - -import ch.qos.logback.classic.Level._ -import com.github.tomakehurst.wiremock.client.WireMock.{get => _, _} -import com.hotels.styx.api.HttpRequest.get -import com.hotels.styx.client.StyxHeaderConfig.ORIGIN_ID_DEFAULT -import com.hotels.styx.support.ResourcePaths.fixturesHome -import com.hotels.styx.support.backends.FakeHttpServer -import com.hotels.styx.support.configuration._ -import com.hotels.styx.support.matchers.LoggingEventMatcher._ -import com.hotels.styx.support.matchers.LoggingTestSupport -import com.hotels.styx.support.server.UrlMatchingStrategies._ -import com.hotels.styx.{StyxClientSupplier, StyxProxySpec} -import io.netty.handler.codec.http.HttpHeaders.Names._ -import io.netty.handler.codec.http.HttpHeaders.Values._ -import com.hotels.styx.api.HttpResponseStatus.OK -import org.hamcrest.MatcherAssert._ -import org.hamcrest.Matchers._ -import org.scalatest.FunSpec -import org.scalatest.concurrent.Eventually - -import scala.concurrent.duration._ - -class HttpMessageLoggingSpec extends FunSpec - with StyxProxySpec - with StyxClientSupplier - with Eventually { - - val crtFile = fixturesHome(this.getClass, "/ssl/testCredentials.crt").toString - val keyFile = fixturesHome(this.getClass, "/ssl/testCredentials.key").toString - - override val styxConfig = StyxConfig( - ProxyConfig( - Connectors( - HttpConnectorConfig(), - HttpsConnectorConfig( - cipherSuites = Seq("TLS_RSA_WITH_AES_128_GCM_SHA256"), - certificateFile = crtFile, - certificateKeyFile = keyFile)) - ), - yamlText = "" + - "request-logging:\n" + - " inbound:\n" + - " enabled: true\n" + - " longFormat: true\n" - ) - - val mockServer = FakeHttpServer.HttpStartupConfig() - .start() - .stub(urlStartingWith("/foobar"), aResponse - .withStatus(OK.code()) - .withHeader(TRANSFER_ENCODING, CHUNKED) - .withBody("I should be here!") - ) - - var logger: LoggingTestSupport = _ - - override protected def beforeAll(): Unit = { - super.beforeAll() - - styxServer.setBackends( - "/foobar" -> HttpBackend("appOne", Origins(mockServer), responseTimeout = 5.seconds) - ) - - val request = get(s"http://localhost:${mockServer.port()}/foobar").build() - val resp = decodedRequest(request) - resp.status() should be (OK) - resp.bodyAs(UTF_8) should be ("I should be here!") - } - - override protected def afterAll(): Unit = { - mockServer.stop() - super.afterAll() - } - - override protected def beforeEach(): Unit = { - super.beforeEach() - logger = new LoggingTestSupport("com.hotels.styx.http-messages.inbound") - } - - override protected def afterEach(): Unit = { - logger.stop() - super.afterEach() - } - - describe("Styx request/response logging") { - it("Should log request and response") { - val request = get(styxServer.routerURL("/foobar")) - .build() - - val resp = decodedRequest(request) - - assertThat(resp.status(), is(OK)) - - eventually(timeout(3.seconds)) { - assertThat(logger.log.size(), is(2)) - - assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, secure=false, request=\\{method=GET, uri=http://localhost:[0-9]+/foobar, origin=\"N/A\", headers=\\[Host=localhost:[0-9]+\\]}"))) - - assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, secure=false, response=\\{status=200 OK, headers=\\[Server=Jetty\\(6.1.26\\), " + ORIGIN_ID_DEFAULT + "=generic-app-01, Via=1.1 styx\\]\\}"))) - } - } - - it("Should log HTTPS request") { - val request = get(styxServer.secureRouterURL("/foobar")).build() - - val resp = decodedRequest(request, secure = true) - - assertThat(resp.status(), is(OK)) - - eventually(timeout(3.seconds)) { - assertThat(logger.log.size(), is(2)) - - assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, secure=true, request=\\{method=GET, uri=https://localhost:[0-9]+/foobar, origin=\"N/A\", headers=.*}"))) - - assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, secure=true, response=\\{status=200 OK, headers=\\[Server=Jetty\\(6.1.26\\), " + ORIGIN_ID_DEFAULT + "=generic-app-01, Via=1.1 styx\\]\\}"))) - } - } - } -} diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpOutboundMessageLoggingSpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpOutboundMessageLoggingSpec.scala index 9db25e2282..4b5b8a221b 100644 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpOutboundMessageLoggingSpec.scala +++ b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/HttpOutboundMessageLoggingSpec.scala @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -100,7 +100,7 @@ class HttpOutboundMessageLoggingSpec extends FunSpec } describe("Styx outbound request/response logging") { - it("Should log request and response") { + it("Should log outbound request and response") { val request = get(styxServer.routerURL("/foobar")) .build() @@ -112,10 +112,10 @@ class HttpOutboundMessageLoggingSpec extends FunSpec assertThat(logger.log.size(), is(2)) assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, request=\\{method=GET, uri=http://localhost:[0-9]+/foobar, origin=\"localhost:[0-9]+\", headers=\\[.*\\]}"))) + "requestId=[-a-z0-9]+, origin=appOne:generic-app-01:localhost:[0-9]+, request=\\{version=HTTP/1.1, method=GET, uri=http://localhost:[0-9]+/foobar, headers=\\[.*\\], id=[-a-z0-9]+\\}"))) assertThat(logger.log(), hasItem(loggingEvent(INFO, - "requestId=[-a-z0-9]+, response=\\{status=200 OK, headers=\\[Transfer-Encoding=chunked, Server=Jetty\\(6.1.26\\)\\]\\}"))) + "requestId=[-a-z0-9]+, response=\\{version=HTTP/1.1, status=200 OK, headers=\\[Transfer-Encoding=chunked, Server=Jetty\\(6.1.26\\)\\]\\}"))) } } } diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/LoggingSpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/LoggingSpec.scala index 1ca031bf14..da2d8286e5 100644 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/LoggingSpec.scala +++ b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/LoggingSpec.scala @@ -103,7 +103,7 @@ class LoggingSpec extends FunSpec assertThat(logger.log(), hasItem( loggingEvent( ERROR, - """Failure status="500 Internal Server Error" during request=LiveHttpRequest.*""", + """Failure status="500 Internal Server Error" during request=.*""", classOf[PluginException], "bad-plugin: Throw exception at Request"))) } @@ -124,7 +124,7 @@ class LoggingSpec extends FunSpec assertThat(logger.log(), hasItem( loggingEvent( ERROR, - """Failure status="500 Internal Server Error" during request=LiveHttpRequest.*""", + """Failure status="500 Internal Server Error" during request=.*""", classOf[PluginException], "bad-plugin: Throw exception at Response"))) } diff --git a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/resiliency/ProxyResiliencySpec.scala b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/resiliency/ProxyResiliencySpec.scala index aaefcf62d7..ef7a2d8ace 100644 --- a/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/resiliency/ProxyResiliencySpec.scala +++ b/system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/resiliency/ProxyResiliencySpec.scala @@ -1,5 +1,5 @@ /* - Copyright (C) 2013-2018 Expedia Inc. + Copyright (C) 2013-2019 Expedia Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/HttpMessageLoggingSpec.kt b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/HttpMessageLoggingSpec.kt new file mode 100644 index 0000000000..5c03c16669 --- /dev/null +++ b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/HttpMessageLoggingSpec.kt @@ -0,0 +1,117 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.logging + +import ch.qos.logback.classic.Level.INFO +import com.hotels.styx.StyxConfig +import com.hotels.styx.StyxServer +import com.hotels.styx.api.HttpHeaderNames.HOST +import com.hotels.styx.api.HttpRequest +import com.hotels.styx.client.StyxHttpClient +import com.hotels.styx.startup.StyxServerComponents +import com.hotels.styx.support.matchers.LoggingTestSupport +import com.hotels.styx.support.proxyHttpHostHeader +import com.hotels.styx.support.wait +import io.kotlintest.Spec +import io.kotlintest.specs.FeatureSpec + +class HttpMessageLoggingSpec : FeatureSpec() { + + init { + feature("Styx request/response logging") { + + scenario("Logger should hide cookies and headers") { + + client.send(HttpRequest.get("/a/path") + .header(HOST, styxServer.proxyHttpHostHeader()) + .header("header1", "h1") + .header("header2", "h2") + .header("cookie", "cookie1=c1;cookie2=c2") + .build()) + .wait() + + val expectedRequest = Regex("requestId=[-a-z0-9]+, secure=false, origin=null, " + + "request=\\{version=HTTP/1.1, method=GET, uri=/a/path, headers=\\[Host=localhost:[0-9]+, header1=\\*\\*\\*\\*, header2=h2, cookie=cookie1=\\*\\*\\*\\*;cookie2=c2\\], id=[-a-z0-9]+\\}") + + val expectedResponse = Regex("requestId=[-a-z0-9]+, secure=false, " + + "response=\\{version=HTTP/1.1, status=200 OK, headers=\\[header1=\\*\\*\\*\\*, header2=h2, cookie=cookie1=\\*\\*\\*\\*;cookie2=c2, Via=1.1 styx\\]\\}") + + logger.log().shouldContain(INFO, expectedRequest) + logger.log().shouldContain(INFO, expectedResponse) + } + } + } + + val logger = LoggingTestSupport("com.hotels.styx.http-messages.inbound") + + val client: StyxHttpClient = StyxHttpClient.Builder().build() + + val yamlText = """ + proxy: + connectors: + http: + port: 0 + + https: + port: 0 + sslProvider: JDK + sessionTimeoutMillis: 300000 + sessionCacheSize: 20000 + + request-logging: + inbound: + enabled: true + longFormat: true + hideCookies: + - cookie1 + hideHeaders: + - header1 + + admin: + connectors: + http: + port: 0 + + routingObjects: + root: + type: StaticResponseHandler + config: + status: 200 + content: "" + headers: + - name: "header1" + value: "h1" + - name: "header2" + value: "h2" + - name: "cookie" + value: "cookie1=c1;cookie2=c2" + + httpPipeline: root + """.trimIndent() + + val styxServer = StyxServer(StyxServerComponents.Builder() + .styxConfig(StyxConfig.fromYaml(yamlText)) + .build()) + + override fun beforeSpec(spec: Spec) { + styxServer.startAsync().awaitRunning() + } + + override fun afterSpec(spec: Spec) { + styxServer.stopAsync().awaitTerminated() + } + +} \ No newline at end of file diff --git a/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/LoggingAssertion.kt b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/LoggingAssertion.kt new file mode 100644 index 0000000000..80777f81db --- /dev/null +++ b/system-tests/ft-suite/src/test/kotlin/com/hotels/styx/logging/LoggingAssertion.kt @@ -0,0 +1,37 @@ +/* + Copyright (C) 2013-2019 Expedia Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +package com.hotels.styx.logging + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.spi.ILoggingEvent + +fun List.shouldContain(level: Level, message: Regex) { + for (event in this) { + if (loggingEventMatches(event, level, message)) return + } + throw AssertionError("\nExpected log to contain event matching:\n" + + "[" + level + "] " + message + "\n" + + "But actual log:" + + formatLogList(this)); +} + +private fun loggingEventMatches(event: ILoggingEvent, level: Level, message: Regex) : Boolean { + return event.level == level && event.formattedMessage.matches(message); +} + +private fun formatLogList(logList: List): String { + return logList.fold("") {a,b -> a + "\n" + b} +}