#766 fix proxy stream (#787)

* Fix buffer option related condition which prevents the proxy block to stream a backend response #766

* Fix malformed mime header in test case (byte code)

* Fix buffer condition and related body parse in combination with syncedVariables provided by each backend instance

* fix test expectation url

* Fix random order related to the default block while preparing roundtrips

* fix test expectation

* Refactor buffer option pkg; include buffer option to json sync calls

* increase timeout

* increase produce delay; add hint for local var in test

* inc time before calling metrics endpoint in metrics test

* add changelog #766

* fix metrics test; just one call to metrics ep

* (ci) skip metrics test

* Fix wrong seq reference; TODO: validation check!

CHANGELOG.md

@@ -6,6 +6,11 @@ Unreleased changes are available as `avenga/couper:edge` container.
 
 ---
 
+## [1.12.2](https://github.com/coupergateway/couper/releases/tag/v1.12.2)
+
+* **Fixed**
+  * Reading the origin response-body even if there is no variables reference; piping the response-body again to the client ([#766](https://github.com/coupergateway/couper/issues/766))
+
 ## [1.12.1](https://github.com/avenga/couper/releases/tag/v1.12.1)
 
 * **Fixed**

config/configload/endpoint_sequence.go

@@ -8,8 +8,9 @@ import (
 
 	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/config/body"
+	srvpkg "github.com/avenga/couper/config/runtime/server"
 	"github.com/avenga/couper/config/sequence"
-	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/eval/variables"
 )
 
 // buildSequences collects possible dependencies from 'backend_responses' variable.
@@ -56,7 +57,9 @@ func buildSequences(names map[string]*hclsyntax.Body, endpoint *config.Endpoint)
 		}
 	}
 
-	for _, s := range sequences {
+	sortedSequences := srvpkg.SortDefault(sequences)
+	for _, name := range sortedSequences {
+		s := sequences[name]
 		if !s.HasParent() {
 			endpoint.Sequences = append(endpoint.Sequences, s)
 		}
@@ -71,7 +74,7 @@ func responseReferences(b *hclsyntax.Body) []string {
 
 	for _, expr := range body.CollectExpressions(b) {
 		for _, traversal := range expr.Variables() {
-			if traversal.RootName() != eval.BackendResponses || len(traversal) < 2 {
+			if traversal.RootName() != variables.BackendResponses || len(traversal) < 2 {
 				continue
 			}
 

config/runtime/endpoint.go

@@ -12,7 +12,7 @@ import (
 	"github.com/avenga/couper/config/runtime/server"
 	"github.com/avenga/couper/config/sequence"
 	"github.com/avenga/couper/errors"
-	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/eval/buffer"
 	"github.com/avenga/couper/handler"
 	"github.com/avenga/couper/handler/producer"
 )
@@ -157,7 +157,7 @@ func NewEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 		blockBodies = append(blockBodies, requestConf.Backend, requestConf.HCLBody())
 	}
 
-	markDepencencies(allProducers, endpointConf.Sequences)
+	markDependencies(allProducers, endpointConf.Sequences)
 	addIndependentProducers(allProducers, endpointConf)
 
 	// TODO: redirect
@@ -181,7 +181,7 @@ func NewEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 		}}
 	}
 
-	bufferOpts := eval.MustBuffer(append(blockBodies, endpointConf.Remain)...)
+	bufferOpts := buffer.Must(append(blockBodies, endpointConf.Remain)...)
 
 	apiName := ""
 	if apiConf != nil {
@@ -203,33 +203,35 @@ func NewEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 	}, nil
 }
 
-func markDepencencies(allProducers map[string]producer.Roundtrip, items sequence.List) {
+func markDependencies(allProducers map[string]producer.Roundtrip, items sequence.List) {
 	for _, item := range items {
 		pr := allProducers[item.Name]
 		var prevs []string
 		deps := item.Deps()
+		if deps == nil {
+			continue
+		}
 		for _, dep := range deps {
 			prevs = append(prevs, dep.Name)
 		}
 		pr.SetDependsOn(strings.Join(prevs, ","))
-		markDepencencies(allProducers, deps)
+		markDependencies(allProducers, deps)
 	}
 }
 
 func addIndependentProducers(allProducers map[string]producer.Roundtrip, endpointConf *config.Endpoint) {
 	// TODO simplify
 	allDeps := sequence.Dependencies(endpointConf.Sequences)
-	for name := range allProducers {
-		inSeq := false
+	sortedProducers := server.SortDefault(allProducers)
+outer:
+	for _, name := range sortedProducers {
 		for _, deps := range allDeps {
 			for _, dep := range deps {
 				if name == dep {
-					inSeq = true
+					continue outer // in sequence
 				}
 			}
 		}
-		if !inSeq {
-			endpointConf.Sequences = append(endpointConf.Sequences, &sequence.Item{Name: name})
-		}
+		endpointConf.Sequences = append(endpointConf.Sequences, &sequence.Item{Name: name})
 	}
 }

config/runtime/error_handler.go

@@ -11,13 +11,14 @@ import (
 	"github.com/avenga/couper/config"
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/eval/buffer"
 	"github.com/avenga/couper/handler"
 )
 
 func newErrorHandler(ctx *hcl.EvalContext, conf *config.Couper, opts *protectedOptions, log *logrus.Entry,
-	defs ACDefinitions, references ...string) (http.Handler, eval.BufferOption, error) {
+	defs ACDefinitions, references ...string) (http.Handler, buffer.Option, error) {
 	kindsHandler := map[string]http.Handler{}
-	var ehBufferOption eval.BufferOption
+	var ehBufferOption buffer.Option
 	for _, ref := range references {
 		definition, ok := defs[ref]
 		if !ok {
@@ -66,7 +67,7 @@ func newErrorHandler(ctx *hcl.EvalContext, conf *config.Couper, opts *protectedO
 
 			epOpts, err := NewEndpointOptions(ctx, epConf, nil, opts.srvOpts, log, conf, opts.memStore)
 			if err != nil {
-				return nil, eval.BufferNone, err
+				return nil, buffer.None, err
 			}
 			if epOpts.ErrorTemplate == nil || h.ErrorFile == "" {
 				epOpts.ErrorTemplate = opts.epOpts.ErrorTemplate

config/runtime/server.go

@@ -27,6 +27,7 @@ import (
 	"github.com/avenga/couper/definitions"
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/eval/buffer"
 	"github.com/avenga/couper/handler"
 	"github.com/avenga/couper/handler/middleware"
 	"github.com/avenga/couper/oauth2"
@@ -315,7 +316,7 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 				newAC(srvConf, parentAPI).
 					Merge(config.
 						NewAccessControl(endpointConf.AccessControl, endpointConf.DisableAccessControl)).List(), nil)
-			epOpts.BufferOpts |= eval.MustBuffer(acBodies...)
+			epOpts.BufferOpts |= buffer.Must(acBodies...)
 
 			errorHandlerDefinitions := ACDefinitions{ // misuse of definitions obj for now
 				"endpoint": &AccessControl{ErrorHandler: endpointConf.ErrorHandler},

config/runtime/server/default.go

@@ -0,0 +1,28 @@
+package server
+
+import "github.com/avenga/couper/config"
+
+// SortDefault ensures that default roundtrips are the last item to be able to pipe the origin response body.
+func SortDefault[V any](m map[string]V) []string {
+	idx := -1
+
+	var trips []string
+	for k := range m {
+		trips = append(trips, k)
+	}
+
+	for i, t := range trips {
+		if t == config.DefaultNameLabel {
+			idx = i
+			break
+		}
+	}
+	edx := len(trips) - 1
+	if idx < 0 || edx == 0 || idx == edx {
+		return trips
+	}
+
+	trips[idx], trips[edx] = trips[edx], trips[idx] // swap
+
+	return trips
+}

definitions/job.go

@@ -12,6 +12,7 @@ import (
 	"github.com/avenga/couper/config/request"
 	"github.com/avenga/couper/errors"
 	"github.com/avenga/couper/eval"
+	"github.com/avenga/couper/eval/variables"
 	"github.com/avenga/couper/handler/middleware"
 	"github.com/avenga/couper/logging"
 	"github.com/avenga/couper/server/writer"
@@ -75,7 +76,7 @@ func (j *Job) Run(ctx context.Context, logEntry *logrus.Entry) {
 			outReq := req.Clone(context.WithValue(ctx, request.UID, uid))
 
 			evalCtx := eval.ContextFromRequest(outReq).WithClientRequest(outReq) // setup syncMap, upstream custom logs
-			delete(evalCtx.HCLContext().Variables, eval.ClientRequest)           // this is the noop req from above, not helpful
+			delete(evalCtx.HCLContext().Variables, variables.ClientRequest)      // this is the noop req from above, not helpful
 
 			outCtx := context.WithValue(evalCtx, request.LogEntry, logEntry)
 			outCtx = context.WithValue(outCtx, request.LogCustomAccess, []hcl.Body{j.conf.Remain}) // local custom logs

eval/attributes/attributes.go

@@ -0,0 +1,20 @@
+package attributes
+
+// common "inline" meta-attributes
+const (
+	SetReqHeaders  = "set_request_headers"
+	AddReqHeaders  = "add_request_headers"
+	DelReqHeaders  = "remove_request_headers"
+	AddQueryParams = "add_query_params"
+	DelQueryParams = "remove_query_params"
+	SetQueryParams = "set_query_params"
+	AddFormParams  = "add_form_params"
+	DelFormParams  = "remove_form_params"
+	SetFormParams  = "set_form_params"
+
+	SetResHeaders = "set_response_headers"
+	AddResHeaders = "add_response_headers"
+	DelResHeaders = "remove_response_headers"
+
+	CustomLogFields = "custom_log_fields"
+)