feat: Make max request payload size configurable in all git/webhook r…

…elated eventsources (#2093) * make max request payload size configurable in all git/webhook related eventsources Signed-off-by: Daniel Soifer <[email protected]>
argoproj · Sep 10, 2022 · 8c71512 · 8c71512
1 parent 156de21
commit 8c71512
Show file tree

Hide file tree

Showing 18 changed files with 520 additions and 418 deletions.
diff --git a/api/event-source.html b/api/event-source.html
diff --git a/api/event-source.md b/api/event-source.md
diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
@@ -2756,6 +2756,11 @@
           "description": "REST API endpoint",
           "type": "string"
         },
+        "maxPayloadSize": {
+          "description": "MaxPayloadSize is the maximum webhook payload size that the server will accept. Requests exceeding that limit will be rejected with \"request too large\" response. Default value: 1048576 (1MB).",
+          "format": "int64",
+          "type": "integer"
+        },
         "metadata": {
           "additionalProperties": {
             "type": "string"

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
diff --git a/eventsources/sources/awssns/start.go b/eventsources/sources/awssns/start.go
@@ -116,7 +116,7 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 		route.Metrics.EventProcessingDuration(route.EventSourceName, route.EventName, float64(time.Since(start)/time.Millisecond))
 	}(time.Now())
 
-	request.Body = http.MaxBytesReader(writer, request.Body, 256*1024) // SNS message size limit is 256KB
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	body, err := io.ReadAll(request.Body)
 	if err != nil {
 		logger.Errorw("failed to parse the request body", zap.Error(err))

diff --git a/eventsources/sources/bitbucket/start.go b/eventsources/sources/bitbucket/start.go
@@ -77,7 +77,7 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 		return
 	}
 
-	request.Body = http.MaxBytesReader(writer, request.Body, 256*1024)
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	body, err := io.ReadAll(request.Body)
 	if err != nil {
 		logger.Desugar().Error("failed to parse request body", zap.Error(err))

diff --git a/eventsources/sources/bitbucketserver/start.go b/eventsources/sources/bitbucketserver/start.go
@@ -84,7 +84,8 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 		route.Metrics.EventProcessingDuration(route.EventSourceName, route.EventName, float64(time.Since(start)/time.Millisecond))
 	}(time.Now())
 
-	body, err := router.parseAndValidateBitbucketServerRequest(writer, request)
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
+	body, err := router.parseAndValidateBitbucketServerRequest(request)
 	if err != nil {
 		logger.Errorw("failed to parse/validate request", zap.Error(err))
 		common.SendErrorResponse(writer, err.Error())
@@ -396,8 +397,7 @@ func (router *Router) createRequestBodyFromWebhook(hook bitbucketv1.Webhook) ([]
 	return requestBody, nil
 }
 
-func (router *Router) parseAndValidateBitbucketServerRequest(writer http.ResponseWriter, request *http.Request) ([]byte, error) {
-	request.Body = http.MaxBytesReader(writer, request.Body, 256*1024)
+func (router *Router) parseAndValidateBitbucketServerRequest(request *http.Request) ([]byte, error) {
 	body, err := io.ReadAll(request.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse request body")

diff --git a/eventsources/sources/github/start.go b/eventsources/sources/github/start.go
@@ -140,6 +140,7 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 		route.Metrics.EventProcessingDuration(route.EventSourceName, route.EventName, float64(time.Since(start)/time.Millisecond))
 	}(time.Now())
 
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	body, err := parseValidateRequest(request, []byte(router.hookSecret))
 	if err != nil {
 		logger.Errorw("request is not valid event notification, discarding it", zap.Error(err))

diff --git a/eventsources/sources/gitlab/start.go b/eventsources/sources/gitlab/start.go
@@ -86,7 +86,7 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 			return
 		}
 	}
-	request.Body = http.MaxBytesReader(writer, request.Body, 256*1024)
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	body, err := io.ReadAll(request.Body)
 	if err != nil {
 		logger.Errorw("failed to parse request body", zap.Error(err))

diff --git a/eventsources/sources/slack/start.go b/eventsources/sources/slack/start.go
@@ -210,7 +210,7 @@ func (rc *Router) handleEvent(request *http.Request) ([]byte, []byte, error) {
 	var err error
 	var response []byte
 	var data []byte
-	body, err := getRequestBody(request)
+	body, err := rc.getRequestBody(request)
 	if err != nil {
 		return data, response, errors.Wrap(err, "failed to fetch request body")
 	}
@@ -269,9 +269,9 @@ func (rc *Router) handleSlashCommand(request *http.Request) ([]byte, error) {
 	return data, nil
 }
 
-func getRequestBody(request *http.Request) ([]byte, error) {
+func (rc *Router) getRequestBody(request *http.Request) ([]byte, error) {
 	// Read request payload
-	body, err := io.ReadAll(io.LimitReader(request.Body, 65536))
+	body, err := io.ReadAll(io.LimitReader(request.Body, rc.route.Context.GetMaxPayloadSize()))
 	// Reset request.Body ReadCloser to prevent side-effect if re-read
 	request.Body = io.NopCloser(bytes.NewBuffer(body))
 	if err != nil {
@@ -293,7 +293,7 @@ func (rc *Router) verifyRequest(request *http.Request) error {
 		}
 
 		// Read the request body
-		body, err := getRequestBody(request)
+		body, err := rc.getRequestBody(request)
 		if err != nil {
 			return err
 		}

diff --git a/eventsources/sources/storagegrid/start.go b/eventsources/sources/storagegrid/start.go
@@ -141,7 +141,7 @@ func (router *Router) HandleRoute(writer http.ResponseWriter, request *http.Requ
 	}
 
 	logger.Info("parsing the request body...")
-	request.Body = http.MaxBytesReader(writer, request.Body, 1024*1024)
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	body, err := io.ReadAll(request.Body)
 	if err != nil {
 		logger.Errorw("failed to parse request body", zap.Error(err))

diff --git a/eventsources/sources/stripe/start.go b/eventsources/sources/stripe/start.go
@@ -95,8 +95,7 @@ func (rc *Router) HandleRoute(writer http.ResponseWriter, request *http.Request)
 		route.Metrics.EventProcessingDuration(route.EventSourceName, route.EventName, float64(time.Since(start)/time.Millisecond))
 	}(time.Now())
 
-	const MaxBodyBytes = int64(1024 * 1024)
-	request.Body = http.MaxBytesReader(writer, request.Body, MaxBodyBytes)
+	request.Body = http.MaxBytesReader(writer, request.Body, route.Context.GetMaxPayloadSize())
 	payload, err := io.ReadAll(request.Body)
 	if err != nil {
 		logger.Errorw("error reading request body", zap.Error(err))

diff --git a/eventsources/sources/webhook/start.go b/eventsources/sources/webhook/start.go
@@ -188,7 +188,7 @@ func GetBody(writer *http.ResponseWriter, request *http.Request, route *webhook.
 			return &ret, nil
 		// default including "application/json" is parsing body as JSON
 		default:
-			request.Body = http.MaxBytesReader(*writer, request.Body, 1024*1024)
+			request.Body = http.MaxBytesReader(*writer, request.Body, route.Context.GetMaxPayloadSize())
 			body, err := getRequestBody(request)
 			if err != nil {
 				logger.Errorw("failed to read request body", zap.Error(err))