From e62278c1b5742d91d3204f2958646b30e34ddd91 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 19 Mar 2018 16:22:47 +0100 Subject: [PATCH 1/5] defaulting: add simpler unit test for debugging --- defaulter_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/defaulter_test.go b/defaulter_test.go index 59d0038..e8a7c6d 100644 --- a/defaulter_test.go +++ b/defaulter_test.go @@ -65,3 +65,37 @@ func TestDefaulter(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, x) } + +func TestDefaulterSimple(t *testing.T) { + schema := spec.Schema{ + SchemaProps: spec.SchemaProps{ + Properties: map[string]spec.Schema{ + "int": spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: float64(42), + }, + }, + "str": spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "Hello", + }, + }, + }, + }, + } + validator := NewSchemaValidator(&schema, nil, "", strfmt.Default) + x := map[string]interface{}{} + t.Logf("Before: %v", x) + r := validator.Validate(x) + assert.False(t, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) + + r.ApplyDefaults() + t.Logf("After: %v", x) + var expected interface{} + err := json.Unmarshal([]byte(`{ + "int": 42, + "str": "Hello" + }`), &expected) + assert.NoError(t, err) + assert.Equal(t, expected, x) +} From c7c5d5c3a1f597aa5f4d0297cfdbdfbd3e7a7b06 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 19 Mar 2018 16:21:04 +0100 Subject: [PATCH 2/5] defaulting: add benchmark --- defaulter_test.go | 56 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/defaulter_test.go b/defaulter_test.go index e8a7c6d..adbdd52 100644 --- a/defaulter_test.go +++ b/defaulter_test.go @@ -30,23 +30,13 @@ import ( var defaulterFixturesPath = filepath.Join("fixtures", "defaulting") func TestDefaulter(t *testing.T) { - fname := filepath.Join(defaulterFixturesPath, "schema.json") - b, err := ioutil.ReadFile(fname) + schema, err := defaulterFixture() assert.NoError(t, err) - var schema spec.Schema - assert.NoError(t, json.Unmarshal(b, &schema)) - - err = spec.ExpandSchema(&schema, nil, nil /*new(noopResCache)*/) - assert.NoError(t, err, fname+" should expand cleanly") - validator := NewSchemaValidator(&schema, nil, "", strfmt.Default) - x := map[string]interface{}{ - "nested": map[string]interface{}{}, - "all": map[string]interface{}{}, - "any": map[string]interface{}{}, - "one": map[string]interface{}{}, - } + validator := NewSchemaValidator(schema, nil, "", strfmt.Default) + x := defaulterFixtureInput() t.Logf("Before: %v", x) + r := validator.Validate(x) assert.False(t, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) @@ -99,3 +89,41 @@ func TestDefaulterSimple(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, x) } + +func BenchmarkDefaulting(b *testing.B) { + b.ReportAllocs() + + schema, err := defaulterFixture() + assert.NoError(b, err) + + for n := 0; n < b.N; n++ { + validator := NewSchemaValidator(schema, nil, "", strfmt.Default) + x := defaulterFixtureInput() + r := validator.Validate(x) + assert.False(b, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) + r.ApplyDefaults() + } +} + +func defaulterFixtureInput() map[string]interface{} { + return map[string]interface{}{ + "nested": map[string]interface{}{}, + "all": map[string]interface{}{}, + "any": map[string]interface{}{}, + "one": map[string]interface{}{}, + } +} + +func defaulterFixture() (*spec.Schema, error) { + fname := filepath.Join(defaulterFixturesPath, "schema.json") + b, err := ioutil.ReadFile(fname) + if err != nil { + return nil, err + } + var schema spec.Schema + if err := json.Unmarshal(b, &schema); err != nil { + return nil, err + } + + return &schema, spec.ExpandSchema(&schema, nil, nil /*new(noopResCache)*/) +} From 94e900b98db19679593dc9666f6315936272047f Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 13 Apr 2018 15:57:33 +0200 Subject: [PATCH 3/5] defaulting: implement orthogonally to validation --- helpers.go | 2 +- object_validator.go | 19 +- defaulter.go => post/defaulter.go | 30 +- defaulter_test.go => post/defaulter_test.go | 23 +- result.go | 290 +++++++++++++++++++- schema.go | 6 +- slice_validator.go | 19 +- 7 files changed, 331 insertions(+), 58 deletions(-) rename defaulter.go => post/defaulter.go (56%) rename defaulter_test.go => post/defaulter_test.go (87%) diff --git a/helpers.go b/helpers.go index 8d788e3..dfe78ba 100644 --- a/helpers.go +++ b/helpers.go @@ -159,7 +159,7 @@ func (h *paramHelper) safeExpandedParamsFor(path, method, operationID string, re resolvedParams = append(resolvedParams, *resolvedParam) } } - // remove params with invalid expansion from slice + // remove params with invalid expansion from Slice operation.Parameters = resolvedParams for _, ppr := range s.analyzer.SafeParamsFor(method, path, diff --git a/object_validator.go b/object_validator.go index 8c601fe..14e1982 100644 --- a/object_validator.go +++ b/object_validator.go @@ -102,6 +102,7 @@ func (o *objectValidator) Validate(data interface{}) *Result { o.precheck(res, val) + // check validity of field names if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows { // Case: additionalProperties: false for k := range val { @@ -175,7 +176,8 @@ func (o *objectValidator) Validate(data interface{}) *Result { // Cases: properties which are not regular properties and have not been matched by the PatternProperties validator if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil { // AdditionalProperties as Schema - res.Merge(NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value)) + r := NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value) + res.mergeForField(data.(map[string]interface{}), key, r) } else if regularProperty && !(matched || succeededOnce) { // TODO: this is dead code since regularProperty=false here res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key)) @@ -189,7 +191,8 @@ func (o *objectValidator) Validate(data interface{}) *Result { // Property types: // - regular Property - for pName, pSchema := range o.Properties { + for pName := range o.Properties { + pSchema := o.Properties[pName] // one instance per iteration rName := pName if o.Path != "" { rName = o.Path + "." + pName @@ -198,17 +201,12 @@ func (o *objectValidator) Validate(data interface{}) *Result { // Recursively validates each property against its schema if v, ok := val[pName]; ok { r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats).Validate(v) - res.Merge(r) + res.mergeForField(data.(map[string]interface{}), pName, r) } else if pSchema.Default != nil { // If a default value is defined, creates the property from defaults // NOTE: JSON schema does not enforce default values to be valid against schema. Swagger does. createdFromDefaults[pName] = true - pName := pName // shadow - // TODO: should validate the default first and ignore the value if invalid - def := pSchema.Default - res.Defaulters = append(res.Defaulters, DefaulterFunc(func() { - val[pName] = def - })) + res.addPropertySchemata(data.(map[string]interface{}), pName, &pSchema) } } @@ -230,7 +228,8 @@ func (o *objectValidator) Validate(data interface{}) *Result { if !regularProperty && (matched /*|| succeededOnce*/) { for _, pName := range patterns { if v, ok := o.PatternProperties[pName]; ok { - res.Merge(NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value)) + r := NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value) + res.mergeForField(data.(map[string]interface{}), key, r) } } } diff --git a/defaulter.go b/post/defaulter.go similarity index 56% rename from defaulter.go rename to post/defaulter.go index 0ff3adc..182edf4 100644 --- a/defaulter.go +++ b/post/defaulter.go @@ -1,4 +1,4 @@ -// Copyright 2015 go-swagger maintainers +// Copyright 2018 go-swagger maintainers // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package validate +package post -// Defaulter defines an interface to define values from default values -// provided in a schema. -type Defaulter interface { - Apply() -} - -// DefaulterFunc is a function to be called to apply default values to an object. -type DefaulterFunc func() +import ( + "github.com/go-openapi/validate" +) -// Apply runs the defaulter function, thus applying default values to an object. -func (f DefaulterFunc) Apply() { - f() +// ApplyDefaults applies defaults to data. +func ApplyDefaults(r *validate.Result) { + fieldSchemata := r.FieldSchemata() + for key, schemata := range fieldSchemata { + LookForDefaultingScheme: + for _, s := range schemata { + if s.Default != nil { + key.Object()[key.Field()] = s.Default + break LookForDefaultingScheme + } + } + } } diff --git a/defaulter_test.go b/post/defaulter_test.go similarity index 87% rename from defaulter_test.go rename to post/defaulter_test.go index adbdd52..bb7378a 100644 --- a/defaulter_test.go +++ b/post/defaulter_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package validate +package post import ( "encoding/json" @@ -25,22 +25,23 @@ import ( "github.com/go-openapi/spec" "github.com/go-openapi/strfmt" + "github.com/go-openapi/validate" ) -var defaulterFixturesPath = filepath.Join("fixtures", "defaulting") +var defaulterFixturesPath = filepath.Join("..", "fixtures", "defaulting") func TestDefaulter(t *testing.T) { schema, err := defaulterFixture() assert.NoError(t, err) - validator := NewSchemaValidator(schema, nil, "", strfmt.Default) + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) x := defaulterFixtureInput() t.Logf("Before: %v", x) r := validator.Validate(x) assert.False(t, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) - r.ApplyDefaults() + ApplyDefaults(r) t.Logf("After: %v", x) var expected interface{} err = json.Unmarshal([]byte(`{ @@ -60,12 +61,12 @@ func TestDefaulterSimple(t *testing.T) { schema := spec.Schema{ SchemaProps: spec.SchemaProps{ Properties: map[string]spec.Schema{ - "int": spec.Schema{ + "int": { SchemaProps: spec.SchemaProps{ Default: float64(42), }, }, - "str": spec.Schema{ + "str": { SchemaProps: spec.SchemaProps{ Default: "Hello", }, @@ -73,13 +74,13 @@ func TestDefaulterSimple(t *testing.T) { }, }, } - validator := NewSchemaValidator(&schema, nil, "", strfmt.Default) + validator := validate.NewSchemaValidator(&schema, nil, "", strfmt.Default) x := map[string]interface{}{} t.Logf("Before: %v", x) r := validator.Validate(x) assert.False(t, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) - r.ApplyDefaults() + ApplyDefaults(r) t.Logf("After: %v", x) var expected interface{} err := json.Unmarshal([]byte(`{ @@ -92,16 +93,16 @@ func TestDefaulterSimple(t *testing.T) { func BenchmarkDefaulting(b *testing.B) { b.ReportAllocs() - + schema, err := defaulterFixture() assert.NoError(b, err) for n := 0; n < b.N; n++ { - validator := NewSchemaValidator(schema, nil, "", strfmt.Default) + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) x := defaulterFixtureInput() r := validator.Validate(x) assert.False(b, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) - r.ApplyDefaults() + ApplyDefaults(r) } } diff --git a/result.go b/result.go index 052dfb3..9543f91 100644 --- a/result.go +++ b/result.go @@ -16,9 +16,11 @@ package validate import ( "fmt" + "reflect" "strings" "github.com/go-openapi/errors" + "github.com/go-openapi/spec" ) // Result represents a validation result set, composed of @@ -37,32 +39,245 @@ type Result struct { Errors []error Warnings []error MatchCount int - Defaulters []Defaulter + + // the object data + data interface{} + + // Schemata for the root object + rootObjectSchemata schemata + // Schemata for object fields + fieldSchemata []fieldSchemata + // Schemata for slice items + itemSchemata []itemSchemata + + cachedFieldSchemta map[FieldKey][]*spec.Schema + cachedItemSchemata map[ItemKey][]*spec.Schema +} + +// FieldKey is a pair of an object and a field, usable as a key for a map. +type FieldKey struct { + object reflect.Value // actually a map[string]interface{}, but the latter cannot be a key + field string +} + +// ItemKey is a pair of a slice and an index, usable as a key for a map. +type ItemKey struct { + slice reflect.Value // actually a []interface{}, but the latter cannot be a key + index int +} + +// NewItemKey returns a pair of an object and field usable as a key of a map. +func NewFieldKey(obj map[string]interface{}, field string) FieldKey { + return FieldKey{object: reflect.ValueOf(obj), field: field} +} + +// Object returns the underlying object of this key. +func (fk *FieldKey) Object() map[string]interface{} { + return fk.object.Interface().(map[string]interface{}) +} + +// Object returns the underlying field of this key. +func (fk *FieldKey) Field() string { + return fk.field +} + +// NewItemKey returns a pair of a slice and index usable as a key of a map. +func NewItemKey(slice []interface{}, i int) ItemKey { + return ItemKey{slice: reflect.ValueOf(slice), index: i} +} + +// Slice returns the underlying slice of this key. +func (ik *ItemKey) Slice() []interface{} { + return ik.slice.Interface().([]interface{}) +} + +// Index returns the underlying index of this key. +func (ik *ItemKey) Index() int { + return ik.index +} + +type fieldSchemata struct { + obj map[string]interface{} + field string + schemata schemata +} + +type itemSchemata struct { + slice []interface{} + index int + schemata schemata } // Merge merges this result with the other one(s), preserving match counts etc. func (r *Result) Merge(others ...*Result) *Result { for _, other := range others { - if other != nil { - r.AddErrors(other.Errors...) - r.AddWarnings(other.Warnings...) - r.MatchCount += other.MatchCount - r.Defaulters = append(r.Defaulters, other.Defaulters...) + if other == nil { + continue } + r.mergeWithoutRootSchemata(other) + r.rootObjectSchemata.Append(other.rootObjectSchemata) } return r } +// Data returns the original data object used for validation. Mutating this renders +// the result invalid. +func (r *Result) Data() interface{} { + return r.data +} + +// RootObjectSchemata returns the schemata which apply to the root object. +func (r *Result) RootObjectSchemata() []*spec.Schema { + return r.rootObjectSchemata.Slice() +} + +// FieldSchemata returns the schemata which apply to fields in objects. +func (r *Result) FieldSchemata() map[FieldKey][]*spec.Schema { + if r.cachedFieldSchemta != nil { + return r.cachedFieldSchemta + } + + ret := make(map[FieldKey][]*spec.Schema, len(r.fieldSchemata)) + for _, fs := range r.fieldSchemata { + key := NewFieldKey(fs.obj, fs.field) + if fs.schemata.one != nil { + ret[key] = append(ret[key], fs.schemata.one) + } else if len(fs.schemata.multiple) > 0 { + ret[key] = append(ret[key], fs.schemata.multiple...) + } + } + r.cachedFieldSchemta = ret + return ret +} + +// ItemSchemata returns the schemata which apply to items in slices. +func (r *Result) ItemSchemata() map[ItemKey][]*spec.Schema { + if r.cachedItemSchemata != nil { + return r.cachedItemSchemata + } + + ret := make(map[ItemKey][]*spec.Schema, len(r.itemSchemata)) + for _, ss := range r.itemSchemata { + key := NewItemKey(ss.slice, ss.index) + if ss.schemata.one != nil { + ret[key] = append(ret[key], ss.schemata.one) + } else if len(ss.schemata.multiple) > 0 { + ret[key] = append(ret[key], ss.schemata.multiple...) + } + } + r.cachedItemSchemata = ret + return ret +} + +func (r *Result) resetCaches() { + r.cachedFieldSchemta = nil + r.cachedItemSchemata = nil +} + +// mergeForField merges other into r, assigning other's root schemata to the given Object and field name. +func (r *Result) mergeForField(obj map[string]interface{}, field string, other *Result) *Result { + if other == nil { + return r + } + r.mergeWithoutRootSchemata(other) + + if other.rootObjectSchemata.Len() > 0 { + if r.fieldSchemata == nil { + r.fieldSchemata = make([]fieldSchemata, len(obj)) + } + r.fieldSchemata = append(r.fieldSchemata, fieldSchemata{ + obj: obj, + field: field, + schemata: other.rootObjectSchemata, + }) + } + + return r +} + +// mergeForSlice merges other into r, assigning other's root schemata to the given slice and index. +func (r *Result) mergeForSlice(slice []interface{}, i int, other *Result) *Result { + if other == nil { + return r + } + r.mergeWithoutRootSchemata(other) + + if other.rootObjectSchemata.Len() > 0 { + if r.itemSchemata == nil { + r.itemSchemata = make([]itemSchemata, len(slice)) + } + r.itemSchemata = append(r.itemSchemata, itemSchemata{ + slice: slice, + index: i, + schemata: other.rootObjectSchemata, + }) + } + + return r +} + +// addRootObjectSchemata adds the given schemata for the root object of the result. +// The slice schemata might be reused. I.e. do not modify it after being added to a result. +func (r *Result) addRootObjectSchemata(s *spec.Schema) { + r.rootObjectSchemata.Append(schemata{one: s}) +} + +// addPropertySchemata adds the given schemata for the object and field. +// The slice schemata might be reused. I.e. do not modify it after being added to a result. +func (r *Result) addPropertySchemata(obj map[string]interface{}, fld string, schema *spec.Schema) { + if r.fieldSchemata == nil { + r.fieldSchemata = make([]fieldSchemata, 0, len(obj)) + } + r.fieldSchemata = append(r.fieldSchemata, fieldSchemata{obj: obj, field: fld, schemata: schemata{one: schema}}) +} + +// addSliceSchemata adds the given schemata for the slice and index. +// The slice schemata might be reused. I.e. do not modify it after being added to a result. +func (r *Result) addSliceSchemata(slice []interface{}, i int, schema *spec.Schema) { + if r.itemSchemata == nil { + r.itemSchemata = make([]itemSchemata, 0, len(slice)) + } + r.itemSchemata = append(r.itemSchemata, itemSchemata{slice: slice, index: i, schemata: schemata{one: schema}}) +} + +// mergeWithoutRootSchemata merges other into r, ignoring the rootObject schemata. +func (r *Result) mergeWithoutRootSchemata(other *Result) { + r.resetCaches() + r.AddErrors(other.Errors...) + r.AddWarnings(other.Warnings...) + r.MatchCount += other.MatchCount + + if other.fieldSchemata != nil { + if r.fieldSchemata == nil { + r.fieldSchemata = other.fieldSchemata + } else { + for _, x := range other.fieldSchemata { + r.fieldSchemata = append(r.fieldSchemata, x) + } + } + } + + if other.itemSchemata != nil { + if r.itemSchemata == nil { + r.itemSchemata = other.itemSchemata + } else { + for _, x := range other.itemSchemata { + r.itemSchemata = append(r.itemSchemata, x) + } + } + } +} + // MergeAsErrors merges this result with the other one(s), preserving match counts etc. // // Warnings from input are merged as Errors in the returned merged Result. func (r *Result) MergeAsErrors(others ...*Result) *Result { for _, other := range others { if other != nil { + r.resetCaches() r.AddErrors(other.Errors...) r.AddErrors(other.Warnings...) r.MatchCount += other.MatchCount - r.Defaulters = append(r.Defaulters, other.Defaulters...) } } return r @@ -74,10 +289,10 @@ func (r *Result) MergeAsErrors(others ...*Result) *Result { func (r *Result) MergeAsWarnings(others ...*Result) *Result { for _, other := range others { if other != nil { + r.resetCaches() r.AddWarnings(other.Errors...) r.AddWarnings(other.Warnings...) r.MatchCount += other.MatchCount - r.Defaulters = append(r.Defaulters, other.Defaulters...) } } return r @@ -210,9 +425,60 @@ func (r *Result) AsError() error { return errors.CompositeValidationError(r.Errors...) } -// ApplyDefaults ... -func (r *Result) ApplyDefaults() { - for _, d := range r.Defaulters { - d.Apply() +// schemata is an arbitrary number of schemata. It does a distinction between zero, +// one and many schemata to avoid slice allocations. +type schemata struct { + // one is set if there is exactly one schema. In that case multiple must be nil. + one *spec.Schema + // multiple is an arbitrary number of schemas. If it is set, one must be nil. + multiple []*spec.Schema +} + +func (s *schemata) Len() int { + if s.one != nil { + return 1 + } + return len(s.multiple) +} + +func (s *schemata) Slice() []*spec.Schema { + if s == nil { + return nil + } + if s.one != nil { + return []*spec.Schema{s.one} + } + return s.multiple +} + +// appendSchemata appends the schemata in other to s. It mutated s in-place. +func (s *schemata) Append(other schemata) { + if other.one == nil && len(other.multiple) == 0 { + return + } + if s.one == nil && len(s.multiple) == 0 { + *s = other + return + } + + if s.one != nil { + if other.one != nil { + s.multiple = []*spec.Schema{s.one, other.one} + } else { + t := make([]*spec.Schema, 0, 1+len(other.multiple)) + s.multiple = append(append(t, s.one), other.multiple...) + } + s.one = nil + } else { + if other.one != nil { + s.multiple = append(s.multiple, other.one) + } else { + if cap(s.multiple) >= len(s.multiple)+len(other.multiple) { + s.multiple = append(s.multiple, other.multiple...) + } else { + t := make([]*spec.Schema, 0, len(s.multiple)+len(other.multiple)) + s.multiple = append(append(t, s.multiple...), other.multiple...) + } + } } } diff --git a/schema.go b/schema.go index c57b705..860b30e 100644 --- a/schema.go +++ b/schema.go @@ -98,10 +98,13 @@ func (s *SchemaValidator) Applies(source interface{}, kind reflect.Kind) bool { // Validate validates the data against the schema func (s *SchemaValidator) Validate(data interface{}) *Result { - result := new(Result) + result := &Result{data: data} if s == nil { return result } + if s.Schema != nil { + result.addRootObjectSchemata(s.Schema) + } if data == nil { result.Merge(s.validators[0].Validate(data)) // type validator @@ -162,6 +165,7 @@ func (s *SchemaValidator) Validate(data interface{}) *Result { result.Inc() } result.Inc() + return result } diff --git a/slice_validator.go b/slice_validator.go index 063829f..ff603c2 100644 --- a/slice_validator.go +++ b/slice_validator.go @@ -57,30 +57,29 @@ func (s *schemaSliceValidator) Validate(data interface{}) *Result { for i := 0; i < size; i++ { validator.SetPath(fmt.Sprintf("%s.%d", s.Path, i)) value := val.Index(i) - result.Merge(validator.Validate(value.Interface())) + result.mergeForSlice(data.([]interface{}), i, validator.Validate(value.Interface())) } } - itemsSize := int64(0) + itemsSize := 0 if s.Items != nil && len(s.Items.Schemas) > 0 { - itemsSize = int64(len(s.Items.Schemas)) - for i := int64(0); i < itemsSize; i++ { + itemsSize = len(s.Items.Schemas) + for i := 0; i < itemsSize; i++ { validator := NewSchemaValidator(&s.Items.Schemas[i], s.Root, fmt.Sprintf("%s.%d", s.Path, i), s.KnownFormats) - if val.Len() <= int(i) { + if val.Len() <= i { break } - result.Merge(validator.Validate(val.Index(int(i)).Interface())) + result.mergeForSlice(data.([]interface{}), int(i), validator.Validate(val.Index(i).Interface())) } - } - if s.AdditionalItems != nil && itemsSize < int64(size) { + if s.AdditionalItems != nil && itemsSize < size { if s.Items != nil && len(s.Items.Schemas) > 0 && !s.AdditionalItems.Allows { result.AddErrors(arrayDoesNotAllowAdditionalItemsMsg()) } if s.AdditionalItems.Schema != nil { - for i := itemsSize; i < (int64(size)-itemsSize)+1; i++ { + for i := itemsSize; i < size-itemsSize+1; i++ { validator := NewSchemaValidator(s.AdditionalItems.Schema, s.Root, fmt.Sprintf("%s.%d", s.Path, i), s.KnownFormats) - result.Merge(validator.Validate(val.Index(int(i)).Interface())) + result.mergeForSlice(data.([]interface{}), int(i), validator.Validate(val.Index(int(i)).Interface())) } } } From deac891b9edaa139dcf90a350978c4d6cf58b1bb Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 13 Apr 2018 15:57:57 +0200 Subject: [PATCH 4/5] Add pruning post-validate algorithm --- fixtures/pruning/schema.json | 105 +++++++++++++++++++++++++++++ post/prune.go | 47 +++++++++++++ post/prune_test.go | 124 +++++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 fixtures/pruning/schema.json create mode 100644 post/prune.go create mode 100644 post/prune_test.go diff --git a/fixtures/pruning/schema.json b/fixtures/pruning/schema.json new file mode 100644 index 0000000..7f05c52 --- /dev/null +++ b/fixtures/pruning/schema.json @@ -0,0 +1,105 @@ +{ + "properties": { + "foo": { + "type": "integer" + }, + "bar": { + "type": "integer" + }, + "nested": { + "type": "object", + "properties": { + "inner": { + "type": "object", + "properties": { + "foo": { + "type": "integer" + }, + "bar": { + "type": "integer" + } + } + } + } + }, + "all": { + "allOf": [ + { + "type": "object", + "properties": { + "foo": { + "type": "integer" + } + } + }, + { + "type": "object", + "properties": { + "bar": { + "type": "integer" + } + } + } + ] + }, + "any": { + "anyOf": [ + { + "type": "object", + "properties": { + "foo": { + "type": "integer" + } + } + }, + { + "type": "object", + "properties": { + "bar": { + "type": "integer" + } + } + } + ] + }, + "one": { + "oneOf": [ + { + "type": "object", + "properties": { + "foo": { + "type": "integer" + } + }, + "required": ["foo"] + }, + { + "type": "object", + "properties": { + "bar": { + "type": "integer" + } + } + } + ] + }, + "not": { + "not": { + "type": "object", + "properties": { + "foo": { + "type": "integer" + } + } + } + }, + "array": { + "items": { + "properties": { + "foo": {} + } + } + } + }, + "required": ["foo", "bar", "nested", "all", "any", "one"] +} diff --git a/post/prune.go b/post/prune.go new file mode 100644 index 0000000..b2d0318 --- /dev/null +++ b/post/prune.go @@ -0,0 +1,47 @@ +// Copyright 2018 go-swagger maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package post + +import ( + "github.com/go-openapi/validate" +) + +// Prune recursively removes all non-specified fields from the data. +func Prune(r *validate.Result) { + prune(r.Data(), r) +} + +func prune(data interface{}, result *validate.Result) { + switch obj := data.(type) { + case map[string]interface{}: + pruneObject(obj, result) + for _, val := range obj { + prune(val, result) + } + case []interface{}: + for _, item := range obj { + prune(item, result) + } + } +} + +func pruneObject(obj map[string]interface{}, result *validate.Result) { + fieldSchemata := result.FieldSchemata() + for field := range obj { + if schemata, ok := fieldSchemata[validate.NewFieldKey(obj, field)]; !ok || len(schemata) == 0 { + delete(obj, field) + } + } +} diff --git a/post/prune_test.go b/post/prune_test.go new file mode 100644 index 0000000..e95a407 --- /dev/null +++ b/post/prune_test.go @@ -0,0 +1,124 @@ +// Copyright 2015 go-swagger maintainers +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package post + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/go-openapi/spec" + "github.com/go-openapi/strfmt" + validate "github.com/go-openapi/validate" +) + +var pruneFixturesPath = filepath.Join("..", "fixtures", "pruning") + +func TestPrune(t *testing.T) { + schema, err := pruningFixture() + assert.NoError(t, err) + + x := map[string]interface{}{ + "foo": 42, + "bar": 42, + "x": 42, + "nested": map[string]interface{}{ + "x": 42, + "inner": map[string]interface{}{ + "foo": 42, + "bar": 42, + "x": 42, + }, + }, + "all": map[string]interface{}{ + "foo": 42, + "bar": 42, + "x": 42, + }, + "any": map[string]interface{}{ + "foo": 42, + "bar": 42, + "x": 42, + }, + "one": map[string]interface{}{ + "bar": 42, + "x": 42, + }, + "array": []interface{}{ + map[string]interface{}{ + "foo": 42, + "bar": 123, + }, + map[string]interface{}{ + "x": 42, + "y": 123, + }, + }, + } + t.Logf("Before: %v", x) + + validator := validate.NewSchemaValidator(schema, nil, "", strfmt.Default) + r := validator.Validate(x) + assert.False(t, r.HasErrors(), fmt.Sprintf("unexpected validation error: %v", r.AsError())) + + Prune(r) + t.Logf("After: %v", x) + expected := map[string]interface{}{ + "foo": 42, + "bar": 42, + "nested": map[string]interface{}{ + "inner": map[string]interface{}{ + "foo": 42, + "bar": 42, + }, + }, + "all": map[string]interface{}{ + "foo": 42, + "bar": 42, + }, + "any": map[string]interface{}{ + // intentionally only list one: the first matching + "foo": 42, + }, + "one": map[string]interface{}{ + "bar": 42, + }, + "array": []interface{}{ + map[string]interface{}{ + "foo": 42, + }, + map[string]interface{}{}, + }, + } + assert.Equal(t, expected, x) +} + +func pruningFixture() (*spec.Schema, error) { + fname := filepath.Join(pruneFixturesPath, "schema.json") + b, err := ioutil.ReadFile(fname) + if err != nil { + return nil, err + } + var schema spec.Schema + if err := json.Unmarshal(b, &schema); err != nil { + return nil, err + } + + return &schema, spec.ExpandSchema(&schema, nil, nil /*new(noopResCache)*/) +} From 563c6e54589ebde4fb8c9335f62193b78f4aca97 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 19 Mar 2018 16:54:09 +0100 Subject: [PATCH 5/5] optimize: reduce quantor allocations --- schema_props.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema_props.go b/schema_props.go index 02f0c9b..c3dfa79 100644 --- a/schema_props.go +++ b/schema_props.go @@ -43,15 +43,15 @@ func (s *schemaPropsValidator) SetPath(path string) { } func newSchemaPropsValidator(path string, in string, allOf, oneOf, anyOf []spec.Schema, not *spec.Schema, deps spec.Dependencies, root interface{}, formats strfmt.Registry) *schemaPropsValidator { - var anyValidators []SchemaValidator + anyValidators := make([]SchemaValidator, 0, len(anyOf)) for _, v := range anyOf { anyValidators = append(anyValidators, *NewSchemaValidator(&v, root, path, formats)) } - var allValidators []SchemaValidator + allValidators := make([]SchemaValidator, 0, len(allOf)) for _, v := range allOf { allValidators = append(allValidators, *NewSchemaValidator(&v, root, path, formats)) } - var oneValidators []SchemaValidator + oneValidators := make([]SchemaValidator, 0, len(oneOf)) for _, v := range oneOf { oneValidators = append(oneValidators, *NewSchemaValidator(&v, root, path, formats)) }