diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
index 85bf68fadf..5954c6bcfb 100644
--- a/lib/hls/hls_parser.js
+++ b/lib/hls/hls_parser.js
@@ -2706,8 +2706,17 @@ shaka.hls.HlsParser = class {
     if (this.lowLatencyMode_ && this.partialTargetDuration_) {
       // For low latency streaming, use the partial segment target duration.
       if (this.lowLatencyByterangeOptimization_) {
-        this.lastTargetDuration_ = Math.min(
-            lastTargetDuration, this.lastTargetDuration_);
+        // We always have at least 1 partial segment part, and most servers
+        // allow you to make a request with _HLS_msn=X&_HLS_part=0 with a
+        // distance of 4 partial segments.  With this we ensure that we
+        // obtain the minimum latency in this type of case.
+        if (this.partialTargetDuration_ * 5 <= lastTargetDuration) {
+          this.lastTargetDuration_ = Math.min(
+              this.partialTargetDuration_, this.lastTargetDuration_);
+        } else {
+          this.lastTargetDuration_ = Math.min(
+              lastTargetDuration, this.lastTargetDuration_);
+        }
       } else {
         this.lastTargetDuration_ = Math.min(
             this.partialTargetDuration_, this.lastTargetDuration_);