add additional places to catch hanging filters

Netflix · Oct 21, 2022 · 8a8868c · 8a8868c
1 parent 3218584
commit 8a8868c
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 2 deletions.
diff --git a/zuul-core/src/main/java/com/netflix/zuul/netty/filter/BaseZuulFilterRunner.java b/zuul-core/src/main/java/com/netflix/zuul/netty/filter/BaseZuulFilterRunner.java
@@ -34,6 +34,7 @@
 import com.netflix.zuul.message.http.HttpRequestInfo;
 import com.netflix.zuul.message.http.HttpRequestMessage;
 import com.netflix.zuul.message.http.HttpResponseMessage;
+import com.netflix.zuul.netty.SpectatorUtils;
 import com.netflix.zuul.netty.server.MethodBinding;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.http.HttpContent;
@@ -140,7 +141,15 @@ protected final void invokeNextStage(final O zuulMesg, final HttpContent chunk)
             try (TaskCloseable ignored =
                     traceTask(this, s -> s.getClass().getSimpleName() + ".fireChannelReadChunk")) {
                 addPerfMarkTags(zuulMesg);
-                getChannelHandlerContext(zuulMesg).fireChannelRead(chunk);
+                ChannelHandlerContext channelHandlerContext = getChannelHandlerContext(zuulMesg);
+                if (!channelHandlerContext.channel().isActive()) {
+                    zuulMesg.getContext().cancel();
+                    zuulMesg.disposeBufferedBody();
+                    SpectatorUtils.newCounter("zuul.filterChain.chunk.hanging",
+                            zuulMesg.getClass().getSimpleName()).increment();
+                } else {
+                    channelHandlerContext.fireChannelRead(chunk);
+                }
             }
         }
     }
@@ -157,7 +166,16 @@ protected final void invokeNextStage(final O zuulMesg) {
             try (TaskCloseable ignored =
                     traceTask(this, s -> s.getClass().getSimpleName() + ".fireChannelRead")) {
                 addPerfMarkTags(zuulMesg);
-                getChannelHandlerContext(zuulMesg).fireChannelRead(zuulMesg);
+                ChannelHandlerContext channelHandlerContext = getChannelHandlerContext(zuulMesg);
+                if (!channelHandlerContext.channel().isActive()) {
+                    zuulMesg.getContext().cancel();
+                    zuulMesg.disposeBufferedBody();
+                    SpectatorUtils.newCounter("zuul.filterChain.message.hanging",
+                            zuulMesg.getClass().getSimpleName()).increment();
+                }
+                else {
+                    channelHandlerContext.fireChannelRead(zuulMesg);
+                }
             }
         }
     }

diff --git a/zuul-core/src/main/java/com/netflix/zuul/netty/filter/ZuulFilterChainHandler.java b/zuul-core/src/main/java/com/netflix/zuul/netty/filter/ZuulFilterChainHandler.java
@@ -34,6 +34,7 @@
 import com.netflix.zuul.message.http.HttpResponseMessage;
 import com.netflix.zuul.message.http.HttpResponseMessageImpl;
 import com.netflix.zuul.netty.RequestCancelledEvent;
+import com.netflix.zuul.netty.SpectatorUtils;
 import com.netflix.zuul.stats.status.StatusCategory;
 import com.netflix.zuul.stats.status.StatusCategoryUtils;
 import io.netty.channel.ChannelHandlerContext;
@@ -149,6 +150,8 @@ private void finishResponseFilters(ChannelHandlerContext ctx) {
             if (zuulResponse != null) {
                 // fire a last content into the filter chain to unblock any filters awaiting a buffered body
                 responseFilterChain.filter(zuulResponse, new DefaultLastHttpContent());
+                SpectatorUtils.newCounter("zuul.filterChain.bodyBuffer.hanging",
+                                zuulRequest.getContext().getRouteVIP()).increment();
             }
         }
     }