From 9324af559a549896c59b825b7ce8355349babe84 Mon Sep 17 00:00:00 2001 From: George Dobrovolsky Date: Fri, 20 Aug 2021 13:44:44 +0300 Subject: [PATCH] nginx-ingress: add rate limits options (#137) --- docs/nginx-ingress.md | 43 +++++------ generators/nginx_ingress/annotations.go | 19 +++++ generators/nginx_ingress/nginx_ingress.go | 38 ++++++++-- .../nginx_ingress/nginx_ingress_test.go | 72 +++++++++++++++++++ options/options.go | 10 ++- options/ratelimit.go | 10 +++ options/service.go | 11 --- options/timeout.go | 12 ++++ 8 files changed, 174 insertions(+), 41 deletions(-) create mode 100644 options/ratelimit.go create mode 100644 options/timeout.go diff --git a/docs/nginx-ingress.md b/docs/nginx-ingress.md index f37088a..cc9a018 100644 --- a/docs/nginx-ingress.md +++ b/docs/nginx-ingress.md @@ -29,27 +29,28 @@ CLI flags apply only at the global level i.e. applies to all paths and methods. To override settings on the path or HTTP method level, you are required to use the x-kusk extension at that path in your API specification.ß ## Full Options Reference -| Name | CLI Option | OpenAPI Spec x-kusk label | Descriptions | Overwritable at path / method | -|:----------------------------:|:------------------------------:|:----------------------------:|:------------------------------------------------------------------------------------------------------------------:|:-----------------------------:| -| OpenAPI or Swagger File | --in | N/A | Location of the OpenAPI or Swagger specification | ❌ | -| Namespace | --namespace | namespace | the namespace in which to create the generated resources (Required) | ❌ | -| Service Name | --service.name | service.name | the name of the service running in Kubernetes (Required) | ❌ | -| Service Namespace | --service.namespace | service.namespace | The namespace where the service named above resides (default value: default) | ❌ | -| Service Port | --service.port | service.port | Port the service is listening on (default value: 80) | ❌ | -| Path Base | --path.base | path.base | Prefix for your resource routes | ❌ | -| Path Trim Prefix | --path.trim_prefix | path.trim_prefix | Trim the specified prefix from URl before passing request onto service | ❌ | -| Path split | --path.split | path.split | Boolean; whether or not to force generator to generate a mapping for each path | ❌ | -| Ingress Host | --host | host | The value to set the host field to in the Ingress resource | ❌ | -| Nginx Ingress Rewrite Target | --nginx_ingress.rewrite_target | nginx_ingress.rewrite_target | Manually set the rewrite target for where traffic must be redirected | ❌ | -| Request Timeout | --timeouts.request_timeout | timeouts.request_timeout | Total request timeout (seconds) | ✅ | -| Idle Timeout | --timeouts.idle_timeout | timeouts.idle_timeout | Idle connection timeout (seconds) | ✅ | -| CORS Origins | N/A | cors.origins | Array of origins | ✅ | -| CORS Methods | N/A | cors.methods | Array of methods | ✅ | -| CORS Headers | N/A | cors.headers | Array of headers | ✅ | -| CORS ExposeHeaders | N/A | cors.expose_headers | Array of headers to expose | ✅ | -| CORS Credentials | N/A | cors.credentials | Boolean: enable credentials (default value: false) | ✅ | -| CORS Max Age | N/A | cors.max_age | Integer:how long the response to the preflight request can be cached for without sending another preflight request | ✅ | - +| Name | CLI Option | OpenAPI Spec x-kusk label | Descriptions | Overwritable at path / method | +|------------------------------|--------------------------------|------------------------------|--------------------------------------------------------------------------------------------------------------------|-------------------------------| +| OpenAPI or Swagger File | --in | N/A | Location of the OpenAPI or Swagger specification | ❌ | +| Namespace | --namespace | namespace | the namespace in which to create the generated resources (Required) | ❌ | +| Service Name | --service.name | service.name | the name of the service running in Kubernetes (Required) | ❌ | +| Service Namespace | --service.namespace | service.namespace | The namespace where the service named above resides (default value: default) | ❌ | +| Service Port | --service.port | service.port | Port the service is listening on (default value: 80) | ❌ | +| Path Base | --path.base | path.base | Prefix for your resource routes | ❌ | +| Path Trim Prefix | --path.trim_prefix | path.trim_prefix | Trim the specified prefix from URl before passing request onto service | ❌ | +| Path split | --path.split | path.split | Boolean; whether or not to force generator to generate a mapping for each path | ❌ | +| Ingress Host | --host | host | The value to set the host field to in the Ingress resource | ❌ | +| Nginx Ingress Rewrite Target | --nginx_ingress.rewrite_target | nginx_ingress.rewrite_target | Manually set the rewrite target for where traffic must be redirected | ❌ | +| Rate limit (RPS) | --rate_limits.rps | rate_limits.rps | Request per second rate limit | ✅ | +| Rate limit (burst) | --rate_limits.burst | rate_limits.burst | Rate limit burst | ✅ | +| Request Timeout | --timeouts.request_timeout | timeouts.request_timeout | Total request timeout (seconds) | ✅ | +| Idle Timeout | --timeouts.idle_timeout | timeouts.idle_timeout | Idle connection timeout (seconds) | ✅ | +| CORS Origins | N/A | cors.origins | Array of origins | ✅ | +| CORS Methods | N/A | cors.methods | Array of methods | ✅ | +| CORS Headers | N/A | cors.headers | Array of headers | ✅ | +| CORS ExposeHeaders | N/A | cors.expose_headers | Array of headers to expose | ✅ | +| CORS Credentials | N/A | cors.credentials | Boolean: enable credentials (default value: false) | ✅ | +| CORS Max Age | N/A | cors.max_age | Integer:how long the response to the preflight request can be cached for without sending another preflight request | ✅ | ## Basic Usage ### CLI Flags ```shell diff --git a/generators/nginx_ingress/annotations.go b/generators/nginx_ingress/annotations.go index e6470d9..1a452dd 100644 --- a/generators/nginx_ingress/annotations.go +++ b/generators/nginx_ingress/annotations.go @@ -23,6 +23,7 @@ func (g *Generator) generateAnnotations( path *options.PathOptions, nginx *options.NGINXIngressOptions, cors *options.CORSOptions, + rateLimits *options.RateLimitOptions, timeoutOpts *options.TimeoutOptions, ) map[string]string { annotations := map[string]string{} @@ -71,6 +72,24 @@ func (g *Generator) generateAnnotations( } // End CORS + // Rate limits + if rps := rateLimits.RPS; rps != 0 { + annotations["nginx.ingress.kubernetes.io/limit-rps"] = fmt.Sprint(rps) + + if burst := rateLimits.Burst; burst != 0 { + // https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#rate-limiting + // nginx-ingress uses a burst multiplier to configure burst for a rate limited path, + // i.e. burst = rps * burstMultiplier + var burstMultiplier = burst / rps + if burstMultiplier < 1 { + burstMultiplier = 1 + } + + annotations["nginx.ingress.kubernetes.io/limit-burst-multiplier"] = fmt.Sprint(burstMultiplier) + } + } + // End rate limits + // Timeouts if requestTimeout := timeoutOpts.RequestTimeout; requestTimeout > 0 { strTimeout := strconv.Itoa(int(requestTimeout) / 2) diff --git a/generators/nginx_ingress/nginx_ingress.go b/generators/nginx_ingress/nginx_ingress.go index c5ea616..b6a3077 100644 --- a/generators/nginx_ingress/nginx_ingress.go +++ b/generators/nginx_ingress/nginx_ingress.go @@ -69,6 +69,18 @@ func (g *Generator) Flags() *pflag.FlagSet { "an Ingress Host to listen on", ) + fs.Uint32( + "rate_limits.rps", + 0, + "request per second rate limit", + ) + + fs.Uint32( + "rate_limits.burst", + 0, + "request per second burst", + ) + fs.Uint32( "timeouts.request_timeout", 0, @@ -109,10 +121,8 @@ func (g *Generator) Generate(opts *options.Options, spec *openapi3.T) (string, e name := fmt.Sprintf("%s-%s", opts.Service.Name, ingressResourceNameFromPath(path)) - var corsOpts options.CORSOptions - // take global CORS options - corsOpts = opts.CORS + corsOpts := opts.CORS // if path-level CORS options are different, override with them if pathSubOpts, ok := opts.PathSubOptions[path]; ok { @@ -122,10 +132,19 @@ func (g *Generator) Generate(opts *options.Options, spec *openapi3.T) (string, e } } - var timeoutOpts options.TimeoutOptions + // take global rate limit options + rateLimitOpts := opts.RateLimits + + // if path-level rate limit options are different, override with them + if pathSubOpts, ok := opts.PathSubOptions[path]; ok { + if !reflect.DeepEqual(options.RateLimitOptions{}, pathSubOpts.RateLimits) && + !reflect.DeepEqual(rateLimitOpts, pathSubOpts.RateLimits) { + rateLimitOpts = pathSubOpts.RateLimits + } + } // take global Timeout options - timeoutOpts = opts.Timeouts + timeoutOpts := opts.Timeouts // if path-level Timeout options are different, override with them if pathSubOpts, ok := opts.PathSubOptions[path]; ok { @@ -141,6 +160,7 @@ func (g *Generator) Generate(opts *options.Options, spec *openapi3.T) (string, e &opts.Path, &opts.NGINXIngress, &corsOpts, + &rateLimitOpts, &timeoutOpts, ) @@ -184,7 +204,7 @@ func (g *Generator) Generate(opts *options.Options, spec *openapi3.T) (string, e opts.Namespace, g.generatePath(&opts.Path, &opts.NGINXIngress), pathTypePrefix, - g.generateAnnotations(&opts.Path, &opts.NGINXIngress, &opts.CORS, &opts.Timeouts), + g.generateAnnotations(&opts.Path, &opts.NGINXIngress, &opts.CORS, &opts.RateLimits, &opts.Timeouts), &opts.Service, opts.Host, ) @@ -306,6 +326,12 @@ func (g *Generator) shouldSplit(opts *options.Options, spec *openapi3.T) bool { return true } + // a path has non-zero, different from global scope rate limits options + if !reflect.DeepEqual(options.RateLimitOptions{}, pathSubOptions.RateLimits) && + !reflect.DeepEqual(opts.RateLimits, pathSubOptions.RateLimits) { + return true + } + // a path has non-zero, different from global scope timeouts options if !reflect.DeepEqual(options.TimeoutOptions{}, pathSubOptions.Timeouts) && !reflect.DeepEqual(opts.Timeouts, pathSubOptions.Timeouts) { diff --git a/generators/nginx_ingress/nginx_ingress_test.go b/generators/nginx_ingress/nginx_ingress_test.go index 85d5f89..ba3c17e 100644 --- a/generators/nginx_ingress/nginx_ingress_test.go +++ b/generators/nginx_ingress/nginx_ingress_test.go @@ -441,6 +441,78 @@ spec: pathType: Prefix status: loadBalancer: {} +`, + }, + { + name: "rate limit options", + options: options.Options{ + Namespace: "booksapp", + Service: options.ServiceOptions{ + Namespace: "booksapp", + Name: "webapp", + Port: 7000, + }, + Path: options.PathOptions{ + Base: "/bookstore", + TrimPrefix: "/bookstore", + }, + RateLimits: options.RateLimitOptions{ + RPS: 100, + Burst: 400, + }, + }, + spec: `openapi: 3.0.1 +x-kusk: + namespace: booksapp + rate_limits: + rps: 100 + burst: 400 + path: + base: /bookstore + trim_prefix: /bookstore + service: + name: webapp + namespace: booksapp + port: 7000 +paths: + /: + get: {} + + /books/{id}: + get: + parameters: + - name: id + in: path + required: true + schema: + type: integer + format: int64 +`, + res: `--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/limit-burst-multiplier: "4" + nginx.ingress.kubernetes.io/limit-rps: "100" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + creationTimestamp: null + name: webapp-ingress + namespace: booksapp +spec: + ingressClassName: nginx + rules: + - http: + paths: + - backend: + service: + name: webapp + port: + number: 7000 + path: /bookstore(/|$)(.*) + pathType: Prefix +status: + loadBalancer: {} `, }, } diff --git a/options/options.go b/options/options.go index b45dfb1..51dc7e2 100644 --- a/options/options.go +++ b/options/options.go @@ -9,9 +9,10 @@ import ( type SubOptions struct { Disabled bool `yaml:"disabled,omitempty" json:"disabled,omitempty"` - Host string `yaml:"host,omitempty" json:"host,omitempty"` - CORS CORSOptions `yaml:"cors,omitempty" json:"cors,omitempty"` - Timeouts TimeoutOptions `yaml:"timeouts,omitempty" json:"timeouts,omitempty"` + Host string `yaml:"host,omitempty" json:"host,omitempty"` + CORS CORSOptions `yaml:"cors,omitempty" json:"cors,omitempty"` + RateLimits RateLimitOptions `yaml:"rate_limits,omitempty" json:"rate_limits,omitempty"` + Timeouts TimeoutOptions `yaml:"timeouts,omitempty" json:"timeouts,omitempty"` } type Options struct { @@ -44,6 +45,8 @@ type Options struct { // They are filled during extension parsing, the map key is method+path. OperationSubOptions map[string]SubOptions `yaml:"-" json:"-"` + RateLimits RateLimitOptions `yaml:"rate_limits,omitempty" json:"rate_limits,omitempty"` + Timeouts TimeoutOptions `yaml:"timeouts,omitempty" json:"timeouts,omitempty"` } @@ -87,6 +90,7 @@ func (o *Options) FillDefaultsAndValidate() error { &o.Cluster, &o.CORS, &o.NGINXIngress, + &o.RateLimits, &o.Timeouts, }) diff --git a/options/ratelimit.go b/options/ratelimit.go new file mode 100644 index 0000000..86c3a5d --- /dev/null +++ b/options/ratelimit.go @@ -0,0 +1,10 @@ +package options + +type RateLimitOptions struct { + RPS uint32 `json:"rps,omitempty" yaml:"rps,omitempty"` + Burst uint32 `json:"burst,omitempty" yaml:"burst,omitempty"` +} + +func (o *RateLimitOptions) Validate() error { + return nil +} diff --git a/options/service.go b/options/service.go index cfe222b..8834fc0 100644 --- a/options/service.go +++ b/options/service.go @@ -15,17 +15,6 @@ type ServiceOptions struct { Port int32 `yaml:"port,omitempty" json:"port,omitempty"` } -type TimeoutOptions struct { - // RequestTimeout is total request timeout - RequestTimeout uint32 `yaml:"request_timeout,omitempty" json:"request_timeout,omitempty"` - // IdleTimeout is timeout for idle connection - IdleTimeout uint32 `yaml:"idle_timeout,omitempty" json:"idle_timeout,omitempty"` -} - -func (o *TimeoutOptions) Validate() error { - return nil -} - func (o *ServiceOptions) Validate() error { return v.ValidateStruct(o, v.Field(&o.Namespace, v.Required.Error("service.namespace is required")), diff --git a/options/timeout.go b/options/timeout.go new file mode 100644 index 0000000..ac57da9 --- /dev/null +++ b/options/timeout.go @@ -0,0 +1,12 @@ +package options + +type TimeoutOptions struct { + // RequestTimeout is total request timeout + RequestTimeout uint32 `yaml:"request_timeout,omitempty" json:"request_timeout,omitempty"` + // IdleTimeout is timeout for idle connection + IdleTimeout uint32 `yaml:"idle_timeout,omitempty" json:"idle_timeout,omitempty"` +} + +func (o *TimeoutOptions) Validate() error { + return nil +}