Skip to content

Commit

Permalink
Fixes #5310
Browse files Browse the repository at this point in the history
Signed-off-by: aarnautu <[email protected]>
  • Loading branch information
aarnautu committed Mar 6, 2023
1 parent a7c1f9c commit 2bf6b5d
Show file tree
Hide file tree
Showing 12 changed files with 1,248 additions and 19 deletions.
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type Config struct {
NDBuiltinCache bool `json:"nd_builtin_cache,omitempty"`
PersistenceDirectory *string `json:"persistence_directory,omitempty"`
DistributedTracing json.RawMessage `json:"distributed_tracing,omitempty"`
Server json.RawMessage `json:"server,omitempty"`
Storage *struct {
Disk json.RawMessage `json:"disk,omitempty"`
} `json:"storage,omitempty"`
Expand Down
8 changes: 8 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,10 @@ func TestActiveConfig(t *testing.T) {
"plugins": {
"some-plugin": {}
},
"server": {
"gzip_min_length": 1024,
"gzip_compression_level": 1
},
"discovery": {"name": "config"}`

serviceObj := `"services": {
Expand Down Expand Up @@ -249,6 +253,10 @@ func TestActiveConfig(t *testing.T) {
"plugins": {
"some-plugin": {}
},
"server": {
"gzip_min_length": 1024,
"gzip_compression_level": 1
},
"default_authorization_decision": "/system/authz/allow",
"default_decision": "/system/main",
"discovery": {"name": "config"}`, version.Version)
Expand Down
14 changes: 14 additions & 0 deletions docs/content/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,10 @@ distributed_tracing:
service_name: opa
sample_percentage: 50
encryption: "off"

server:
gzip_min_length: 1400,
gzip_compression_level: 9
```
#### Environment Variable Substitution
Expand Down Expand Up @@ -925,3 +929,13 @@ with data put into the configured `directory`.
| `storage.disk.badger` | `string` | No (default: empty) | "Superflags" passed to Badger allowing to modify advanced options. |

See [the docs on disk storage](../misc-disk/) for details about the settings.

### Server

The `server` configuration sets the gzip compression settings for `/v0/data`, `/v1/data` and `/v1/compile` HTTP `POST` endpoints
The gzip compression settings are used when the client sends `Accept-Encoding: gzip`

| Field | Type | Required | Description |
|---------------------------------|-------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `server.gzip_min_length` | `int` | No, (default: 1024) | Specifies the minimum length of the response to compress |
| `server.gzip_compression_level` | `int` | No, (default: 9) | Specifies the compression level. Accepted values: a value of either 0 (no compression), 1 (best speed, lowest compression) or 9 (slowest, best compression). See https://pkg.go.dev/compress/flate#pkg-constants |
13 changes: 13 additions & 0 deletions docs/content/rest-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -732,6 +732,10 @@ The path separator is used to access values inside object and array documents. I
- **instrument** - Instrument query evaluation and return a superset of performance metrics in addition to result. See [Performance Metrics](#performance-metrics) for more detail.
- **strict-builtin-errors** - Treat built-in function call errors as fatal and return an error immediately.

#### Request Headers

- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.gzip_min_length` value. See the configuration section

#### Status Codes

- **200** - no error
Expand Down Expand Up @@ -820,6 +824,8 @@ The request body contains an object that specifies a value for [The input Docume
#### Request Headers

- **Content-Type: application/x-yaml**: Indicates the request body is a YAML encoded object.
- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.gzip_min_length` value. See the configuration section

#### Query Parameters

Expand Down Expand Up @@ -939,6 +945,8 @@ array documents.
#### Request Headers

- **Content-Type: application/x-yaml**: Indicates the request body is a YAML encoded object.
- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.gzip_min_length` value. See the configuration section

#### Query Parameters

Expand Down Expand Up @@ -1290,6 +1298,11 @@ Compile API requests contain the following fields:
| `options` | `object[string, any]` | No | Additional options to use during partial evaluation. Only `disableInlining` option is supported. (default: undefined). |
| `unknowns` | `array[string]` | No | The terms to treat as unknown during partial evaluation (default: `["input"]`]). |

### Request Headers

- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.gzip_min_length` value

#### Query Parameters

- **pretty** - If parameter is `true`, response will formatted for humans.
Expand Down
78 changes: 78 additions & 0 deletions plugins/server/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package server

import (
"compress/gzip"
"fmt"

"github.com/open-policy-agent/opa/util"
)

var defaultGzipMinLength = 1024
var defaultGzipCompressionLevel = gzip.BestCompression

// Config represents the configuration for the Server settings
type Config struct {
GzipMinLength *int `json:"gzip_min_length,omitempty"` // the minimum length of a response that will be gzipped
GzipCompressionLevel *int `json:"gzip_compression_level,omitempty"` // the compression level for gzip
}

// ConfigBuilder assists in the construction of the plugin configuration.
type ConfigBuilder struct {
raw []byte
}

// NewConfigBuilder returns a new ConfigBuilder to build and parse the server config
func NewConfigBuilder() *ConfigBuilder {
return &ConfigBuilder{}
}

// WithBytes sets the raw server config
func (b *ConfigBuilder) WithBytes(config []byte) *ConfigBuilder {
b.raw = config
return b
}

// Parse returns a valid Config object with defaults injected.
func (b *ConfigBuilder) Parse() (*Config, error) {
if b.raw == nil {
defaultConfig := &Config{
GzipMinLength: &defaultGzipMinLength,
GzipCompressionLevel: &defaultGzipCompressionLevel,
}
return defaultConfig, nil
}

var result Config

if err := util.Unmarshal(b.raw, &result); err != nil {
return nil, err
}

return &result, result.validateAndInjectDefaults()
}

func (c *Config) validateAndInjectDefaults() error {
if c.GzipMinLength == nil {
c.GzipMinLength = &defaultGzipMinLength
}

if c.GzipCompressionLevel == nil {
c.GzipCompressionLevel = &defaultGzipCompressionLevel
}

if *c.GzipMinLength <= 0 {
return fmt.Errorf("invalid value for server.gzip_min_length field, should be a positive number")
}

acceptedCompressionLevels := map[int]bool{
gzip.NoCompression: true,
gzip.BestSpeed: true,
gzip.BestCompression: true,
}
_, compressionLevelAccepted := acceptedCompressionLevels[*c.GzipCompressionLevel]
if !compressionLevelAccepted {
return fmt.Errorf("invalid value for server.gzip_compression_level field, should match gzip's library compression levels")
}

return nil
}
105 changes: 105 additions & 0 deletions plugins/server/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package server

import (
"fmt"
"testing"
)

func TestConfigValidation(t *testing.T) {
tests := []struct {
input string
wantErr bool
}{
{
input: `{}`,
wantErr: false,
},
{
input: `{"gzip_min_length": "not-a-number"}`,
wantErr: true,
},
{
input: `{"gzip_min_length": 42}`,
wantErr: false,
},
{
input: `{"gzip_min_length": "42"}`,
wantErr: true,
},
{
input: `{"gzip_min_length": 0}`,
wantErr: true,
},
{
input: `{"gzip_min_length": -10}`,
wantErr: true,
},
{
input: `{"random_key": 0}`,
wantErr: false,
},
{
input: `{"gzip_min_length": -10, "gzip_compression_level": 13}`,
wantErr: true,
},
{
input: `{"gzip_compression_level": "not-an-number"}`,
wantErr: true,
},
{
input: `{"gzip_compression_level": 1}`,
wantErr: false,
},
{
input: `{"gzip_compression_level": 13}`,
wantErr: true,
},
{
input: `{"gzip_min_length": 42, "gzip_compression_level": 9}`,
wantErr: false,
},
}

for i, test := range tests {
t.Run(fmt.Sprintf("TestConfigValidation_case_%d", i), func(t *testing.T) {
_, err := NewConfigBuilder().WithBytes([]byte(test.input)).Parse()
if err != nil && !test.wantErr {
t.Fail()
}
if err == nil && test.wantErr {
t.Fail()
}
})
}
}

func TestConfigValue(t *testing.T) {
tests := []struct {
input string
minLengthExpectedValue int
compressionLevelExpectedValue int
}{
{
input: `{}`,
minLengthExpectedValue: 1024,
compressionLevelExpectedValue: 9,
},
{
input: `{"gzip_min_length": 42, "gzip_compression_level": 1}`,
minLengthExpectedValue: 42,
compressionLevelExpectedValue: 1,
},
}

for i, test := range tests {
t.Run(fmt.Sprintf("TestConfigValue_case_%d", i), func(t *testing.T) {
config, err := NewConfigBuilder().WithBytes([]byte(test.input)).Parse()
if err != nil {
t.Fail()
}
if *config.GzipMinLength != test.minLengthExpectedValue || *config.GzipCompressionLevel != test.compressionLevelExpectedValue {
t.Fail()
}
})
}
}
50 changes: 49 additions & 1 deletion runtime/logging.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package runtime

import (
"bytes"
"compress/gzip"
"io"
"net/http"
"strings"
Expand Down Expand Up @@ -73,7 +74,22 @@ func (h *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
bs, r.Body, err = readBody(r.Body)
}
if err == nil {
fields["req_body"] = string(bs)
if gzipReceived(r.Header) {
// the request is compressed
reader := bytes.NewReader(bs)
gzReader, err := gzip.NewReader(reader)
if err != nil {
h.logger.Error("Failed to read the compressed payload: %v", err.Error())
}
plainOutput, err := io.ReadAll(gzReader)
if err != nil {
h.logger.Error("Failed to decompressed the payload: %v", err.Error())
}
defer gzReader.Close()
fields["req_body"] = string(plainOutput)
} else {
fields["req_body"] = string(bs)
}
} else {
fields["err"] = err
}
Expand Down Expand Up @@ -127,6 +143,18 @@ func (h *LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// metrics endpoint does so when the client accepts it (e.g. prometheus)
fields["resp_body"] = "[compressed payload]"

case gzipAccepted(r.Header) && gzipReceived(w.Header()) && (isDataEndpoint(r) || isCompileEndpoint(r)):
// data and compile endpoints might compress the response
gzReader, err := gzip.NewReader(recorder.buf)
if err != nil {
h.logger.Error("Failed to read the compressed payload: %v", err.Error())
}
plainOutput, err := io.ReadAll(gzReader)
if err != nil {
h.logger.Error("Failed to decompressed the payload: %v", err.Error())
}
fields["resp_body"] = string(plainOutput)

default:
fields["resp_body"] = recorder.buf.String()
}
Expand All @@ -148,6 +176,18 @@ func gzipAccepted(header http.Header) bool {
return false
}

func gzipReceived(header http.Header) bool {
a := header.Get("Content-Encoding")
parts := strings.Split(a, ",")
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "gzip" || strings.HasPrefix(part, "gzip;") {
return true
}
}
return false
}

func isPprofEndpoint(req *http.Request) bool {
return strings.HasPrefix(req.URL.Path, "/debug/pprof/")
}
Expand All @@ -156,6 +196,14 @@ func isMetricsEndpoint(req *http.Request) bool {
return strings.HasPrefix(req.URL.Path, "/metrics")
}

func isDataEndpoint(req *http.Request) bool {
return strings.HasPrefix(req.URL.Path, "/v1/data") || strings.HasPrefix(req.URL.Path, "/v0/data")
}

func isCompileEndpoint(req *http.Request) bool {
return strings.HasPrefix(req.URL.Path, "/v1/compile")
}

type recorder struct {
logger logging.Logger
inner http.ResponseWriter
Expand Down
Loading

0 comments on commit 2bf6b5d

Please sign in to comment.