render Propfind responses as xml stream

changelog/unreleased/stream-propfind-responses.md

@@ -0,0 +1,5 @@
+Enhancement: Stream PROPFIND responses
+
+We switched to streamed encoding of propfind responses to decrease time to first byte.
+
+https://github.com/cs3org/reva/pull/3024

internal/http/services/owncloud/ocdav/propfind/propfind.go

@@ -401,25 +401,8 @@ func (p *Handler) propfindResponse(ctx context.Context, w http.ResponseWriter, r
 		}
 	}
 
-	propRes, err := MultistatusResponse(ctx, &pf, resourceInfos, p.PublicURL, namespace, linkshares)
-	if err != nil {
-		log.Error().Err(err).Msg("error formatting propfind")
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
-	w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol")
-	w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8")
-	if sendTusHeaders {
-		w.Header().Add(net.HeaderAccessControlExposeHeaders, strings.Join([]string{net.HeaderTusResumable, net.HeaderTusVersion, net.HeaderTusExtension}, ", "))
-		w.Header().Set(net.HeaderTusResumable, "1.0.0")
-		w.Header().Set(net.HeaderTusVersion, "1.0.0")
-		w.Header().Set(net.HeaderTusExtension, "creation,creation-with-upload,checksum,expiration")
-	}
+	RenderMultistatusResponse(ctx, w, &pf, resourceInfos, p.PublicURL, namespace, linkshares, sendTusHeaders)
 
-	w.WriteHeader(http.StatusMultiStatus)
-	if _, err := w.Write(propRes); err != nil {
-		log.Err(err).Msg("error writing response")
-	}
 }
 
 // TODO this is just a stat -> rename
@@ -865,24 +848,49 @@ func ReadPropfind(r io.Reader) (pf XML, status int, err error) {
 	return pf, 0, nil
 }
 
-// MultistatusResponse converts a list of resource infos into a multistatus response string
-func MultistatusResponse(ctx context.Context, pf *XML, mds []*provider.ResourceInfo, publicURL, ns string, linkshares map[string]struct{}) ([]byte, error) {
-	responses := make([]*ResponseXML, 0, len(mds))
+func renderMultistatusHeader(_ context.Context, w http.ResponseWriter, sendTusHeaders bool) error {
+	// TODO transfer encoding chunked?
+	w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol")
+	w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8")
+	if sendTusHeaders {
+		w.Header().Add(net.HeaderAccessControlExposeHeaders, strings.Join([]string{net.HeaderTusResumable, net.HeaderTusVersion, net.HeaderTusExtension}, ", "))
+		w.Header().Set(net.HeaderTusResumable, "1.0.0")
+		w.Header().Set(net.HeaderTusVersion, "1.0.0")
+		w.Header().Set(net.HeaderTusExtension, "creation,creation-with-upload,checksum,expiration")
+	}
+
+	w.WriteHeader(http.StatusMultiStatus)
+	_, err := w.Write([]byte(`<d:multistatus xmlns:s="http://sabredav.org/ns" xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">`))
+	return err
+}
+func renderMultistatusFooter(_ context.Context, w http.ResponseWriter, sendTusHeaders bool) error {
+	_, err := w.Write([]byte(`</d:multistatus>`))
+	return err
+}
+
+// RenderMultistatusResponse converts a list of resource infos into a multistatus response string
+func RenderMultistatusResponse(ctx context.Context, w http.ResponseWriter, pf *XML, mds []*provider.ResourceInfo, publicURL, ns string, linkshares map[string]struct{}, sendTusHeaders bool) {
+	log := appctx.GetLogger(ctx)
+	if err := renderMultistatusHeader(ctx, w, sendTusHeaders); err != nil {
+		log.Err(err).Msg("error writing xml header")
+	}
+	enc := xml.NewEncoder(w)
 	for i := range mds {
 		res, err := mdToPropResponse(ctx, pf, mds[i], publicURL, ns, linkshares)
 		if err != nil {
-			return nil, err
+			log.Error().Err(err).Msg("error formatting resource")
+			return
+		}
+		err = enc.Encode(res)
+		if err != nil {
+			log.Error().Err(err).Msg("error writing xml")
+			return
 		}
-		responses = append(responses, res)
 	}
 
-	msr := NewMultiStatusResponseXML()
-	msr.Responses = responses
-	msg, err := xml.Marshal(msr)
-	if err != nil {
-		return nil, err
+	if err := renderMultistatusFooter(ctx, w, sendTusHeaders); err != nil {
+		log.Err(err).Msg("error writing xml footer")
 	}
-	return msg, nil
 }
 
 // mdToPropResponse converts the CS3 metadata into a webdav PropResponse

internal/http/services/owncloud/ocdav/publicfile.go

@@ -111,19 +111,8 @@ func (s *svc) handlePropfindOnToken(w http.ResponseWriter, r *http.Request, ns s
 
 	infos := s.getPublicFileInfos(onContainer, depth == net.DepthZero, tokenStatInfo)
 
-	propRes, err := propfind.MultistatusResponse(ctx, &pf, infos, s.c.PublicURL, ns, nil)
-	if err != nil {
-		sublog.Error().Err(err).Msg("error formatting propfind")
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
+	propfind.RenderMultistatusResponse(ctx, w, &pf, infos, s.c.PublicURL, ns, nil, false) // TODO why not announce TUS by sending the headers?
 
-	w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol")
-	w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusMultiStatus)
-	if _, err := w.Write(propRes); err != nil {
-		sublog.Err(err).Msg("error writing response")
-	}
 }
 
 // there are only two possible entries

internal/http/services/owncloud/ocdav/report.go

@@ -26,7 +26,6 @@ import (
 	rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
-	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/net"
 	"github.com/cs3org/reva/v2/internal/http/services/owncloud/ocdav/propfind"
 	"github.com/cs3org/reva/v2/pkg/appctx"
 	ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
@@ -110,18 +109,8 @@ func (s *svc) doFilterFiles(w http.ResponseWriter, r *http.Request, ff *reportFi
 			infos = append(infos, statRes.Info)
 		}
 
-		responsesXML, err := propfind.MultistatusResponse(ctx, &propfind.XML{Prop: ff.Prop}, infos, s.c.PublicURL, namespace, nil)
-		if err != nil {
-			log.Error().Err(err).Msg("error formatting propfind")
-			w.WriteHeader(http.StatusInternalServerError)
-			return
-		}
-		w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol")
-		w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8")
-		w.WriteHeader(http.StatusMultiStatus)
-		if _, err := w.Write(responsesXML); err != nil {
-			log.Err(err).Msg("error writing response")
-		}
+		propfind.RenderMultistatusResponse(ctx, w, &propfind.XML{Prop: ff.Prop}, infos, s.c.PublicURL, namespace, nil, false) // TODO why not announce TUS by sending headers?
+
 	}
 }
 

internal/http/services/owncloud/ocdav/versions.go

@@ -189,20 +189,7 @@ func (h *VersionsHandler) doListVersions(w http.ResponseWriter, r *http.Request,
 		infos = append(infos, vi)
 	}
 
-	propRes, err := propfind.MultistatusResponse(ctx, &pf, infos, s.c.PublicURL, "", nil)
-	if err != nil {
-		sublog.Error().Err(err).Msg("error formatting propfind")
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
-	w.Header().Set(net.HeaderDav, "1, 3, extended-mkcol")
-	w.Header().Set(net.HeaderContentType, "application/xml; charset=utf-8")
-	w.WriteHeader(http.StatusMultiStatus)
-	_, err = w.Write(propRes)
-	if err != nil {
-		sublog.Error().Err(err).Msg("error writing body")
-		return
-	}
+	propfind.RenderMultistatusResponse(ctx, w, &pf, infos, s.c.PublicURL, "", nil, false)
 
 }