Skip to content

Commit

Permalink
Yaml (OpenSlides#849)
Browse files Browse the repository at this point in the history 
* Use new yaml lib for models.yml

* Update for permissions and test-data

* Fix tests
  • Loading branch information
@ostcar
ostcar authored Mar 2, 2024
1 parent 6723dd7 commit 68f78c4
Show file tree
Hide file tree
Showing 5 changed files with 62 additions and 37 deletions.
7 changes: 6 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.22

require (
github.com/alecthomas/kong v0.8.1
github.com/goccy/go-yaml v1.11.3
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/gomodule/redigo v1.9.2
github.com/jackc/pgx/v5 v5.5.3
Expand All @@ -14,7 +15,6 @@ require (
github.com/stretchr/testify v1.9.0
golang.org/x/sync v0.6.0
golang.org/x/sys v0.17.0
gopkg.in/yaml.v3 v3.0.1
)

require (
Expand All @@ -28,6 +28,7 @@ require (
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/imdario/mergo v0.3.15 // indirect
Expand All @@ -36,6 +37,8 @@ require (
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/kr/text v0.1.0 // indirect
github.com/lib/pq v1.10.8 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
Expand All @@ -52,5 +55,7 @@ require (
golang.org/x/mod v0.9.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
26 changes: 23 additions & 3 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,26 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
Expand All @@ -60,8 +70,15 @@ github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NB
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE=
github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
Expand Down Expand Up @@ -123,6 +140,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -138,8 +157,9 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
56 changes: 28 additions & 28 deletions internal/models/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"io"
"strings"

"gopkg.in/yaml.v3"
"github.com/goccy/go-yaml"
)

// Unmarshal parses the content of models.yml to a datastruct.q
Expand All @@ -24,8 +24,8 @@ type Model struct {
}

// UnmarshalYAML decodes a yaml model to models.Model.
func (m *Model) UnmarshalYAML(node *yaml.Node) error {
return node.Decode(&m.Fields)
func (m *Model) UnmarshalYAML(node []byte) error {
return yaml.Unmarshal(node, &m.Fields)
}

// Field of a model.
Expand All @@ -52,13 +52,13 @@ func (f *Field) RestrictionMode() string {
}

// UnmarshalYAML decodes a model attribute from yaml.
func (f *Field) UnmarshalYAML(value *yaml.Node) error {
func (f *Field) UnmarshalYAML(node []byte) error {
var typer struct {
Type string `yaml:"type"`
RestrictionMode string `yaml:"restriction_mode"`
Required bool `yaml:"required"`
}
if err := value.Decode(&typer); err != nil {
if err := yaml.Unmarshal(node, &typer); err != nil {
return fmt.Errorf("field object without type: %w", err)
}

Expand All @@ -74,8 +74,8 @@ func (f *Field) UnmarshalYAML(value *yaml.Node) error {

case "relation":
var relation AttributeRelation
if err := value.Decode(&relation); err != nil {
return fmt.Errorf("invalid object of type %s at line %d object: %w", typer.Type, value.Line, err)
if err := yaml.Unmarshal(node, &relation); err != nil {
return fmt.Errorf("invalid object of type %s: %w", typer.Type, err)
}
relation.list = list
f.relation = &relation
Expand All @@ -86,8 +86,8 @@ func (f *Field) UnmarshalYAML(value *yaml.Node) error {

case "generic-relation":
var relation AttributeGenericRelation
if err := value.Decode(&relation); err != nil {
return fmt.Errorf("invalid object of type %s at line %d object: %w", typer.Type, value.Line, err)
if err := yaml.Unmarshal(node, &relation); err != nil {
return fmt.Errorf("invalid object of type %s object: %w", typer.Type, err)
}
relation.list = list
f.relation = &relation
Expand All @@ -109,12 +109,12 @@ type ToCollectionField struct {
}

// UnmarshalYAML decodes the models.yml to a To object.
func (t *ToCollectionField) UnmarshalYAML(value *yaml.Node) error {
func (t *ToCollectionField) UnmarshalYAML(node []byte) error {
var s string
if err := value.Decode(&s); err == nil {
if err := yaml.Unmarshal(node, &s); err == nil {
cf := strings.Split(s, "/")
if len(cf) != 2 {
return fmt.Errorf("invalid value of `to` in line %d, expected one `/`: %s", value.Line, s)
return fmt.Errorf("invalid value of `to`, expected one `/`: %s", s)
}
t.Collection = cf[0]
t.ToField.Name = cf[1]
Expand All @@ -125,8 +125,8 @@ func (t *ToCollectionField) UnmarshalYAML(value *yaml.Node) error {
Collection string `yaml:"collection"`
Field ToField `yaml:"field"`
}
if err := value.Decode(&d); err != nil {
return fmt.Errorf("decoding to collection field at line %d: %w", value.Line, err)
if err := yaml.Unmarshal(node, &d); err != nil {
return fmt.Errorf("decoding to collection field: %w", err)
}
t.Collection = d.Collection
t.ToField = d.Field
Expand All @@ -140,9 +140,9 @@ type ToField struct {
}

// UnmarshalYAML decodes the models.yml to a ToField object.
func (t *ToField) UnmarshalYAML(value *yaml.Node) error {
func (t *ToField) UnmarshalYAML(node []byte) error {
var s string
if err := value.Decode(&s); err == nil {
if err := yaml.Unmarshal(node, &s); err == nil {
t.Name = s
t.Type = "normal"
return nil
Expand All @@ -152,8 +152,8 @@ func (t *ToField) UnmarshalYAML(value *yaml.Node) error {
Name string `yaml:"name"`
Type string `yaml:"type"`
}
if err := value.Decode(&d); err != nil {
return fmt.Errorf("decoding to field at line %d: %w", value.Line, err)
if err := yaml.Unmarshal(node, &d); err != nil {
return fmt.Errorf("decoding to field: %w", err)
}
t.Name = d.Name
t.Type = d.Type
Expand Down Expand Up @@ -183,20 +183,20 @@ type To struct {
}

// UnmarshalYAML decodes the models.yml to a To object.
func (t *To) UnmarshalYAML(value *yaml.Node) error {
func (t *To) UnmarshalYAML(node []byte) error {
var s string
if err := value.Decode(&s); err == nil {
if err := yaml.Unmarshal(node, &s); err == nil {
cf := strings.Split(s, "/")
if len(cf) != 2 {
return fmt.Errorf("invalid value of `to` in line %d, expected one `/`: %s", value.Line, s)
return fmt.Errorf("invalid value of `to`, expected one `/`: %s", s)
}
t.CollectionField.Collection = cf[0]
t.CollectionField.ToField.Name = cf[1]
return nil
}

if err := value.Decode(&(t.CollectionField)); err != nil {
return fmt.Errorf("decoding to field at line %d: %w", value.Line, err)
if err := yaml.Unmarshal(node, &(t.CollectionField)); err != nil {
return fmt.Errorf("decoding to field: %w", err)
}
return nil
}
Expand All @@ -223,12 +223,12 @@ type ToGeneric struct {
}

// UnmarshalYAML unmarshalls data to a ToGeneric object.
func (t *ToGeneric) UnmarshalYAML(value *yaml.Node) error {
func (t *ToGeneric) UnmarshalYAML(node []byte) error {
var d struct {
Collections []string `yaml:"collections"`
Field ToField `yaml:"field"`
}
if err := value.Decode(&d); err == nil {
if err := yaml.Unmarshal(node, &d); err == nil {
t.CollectionFields = make([]ToCollectionField, len(d.Collections))
for i, collection := range d.Collections {
t.CollectionFields[i].Collection = collection
Expand All @@ -238,14 +238,14 @@ func (t *ToGeneric) UnmarshalYAML(value *yaml.Node) error {
}

var e []string
if err := value.Decode(&e); err != nil {
return fmt.Errorf("decoding to generic field at line %d: %w", value.Line, err)
if err := yaml.Unmarshal(node, &e); err != nil {
return fmt.Errorf("decoding to generic field: %w", err)
}
t.CollectionFields = make([]ToCollectionField, len(e))
for i, collectionfield := range e {
cf := strings.Split(collectionfield, "/")
if len(cf) != 2 {
return fmt.Errorf("invalid value of `to` in line %d, expected one `/`: %s", value.Line, collectionfield)
return fmt.Errorf("invalid value of `to`, expected one `/`: %s", collectionfield)
}
t.CollectionFields[i].Collection = cf[0]
t.CollectionFields[i].ToField.Name = cf[1]
Expand Down
2 changes: 1 addition & 1 deletion internal/restrict/perm/generate/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"strings"
"text/template"

"gopkg.in/yaml.v3"
"github.com/goccy/go-yaml"
)

func main() {
Expand Down
8 changes: 4 additions & 4 deletions pkg/datastore/dsmock/yaml_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"strings"

"github.com/OpenSlides/openslides-autoupdate-service/pkg/datastore/dskey"
"gopkg.in/yaml.v3"
"github.com/goccy/go-yaml"
)

// YAMLData creates key values from a yaml object.
Expand All @@ -29,13 +29,13 @@ func YAMLData(input string) map[dskey.Key][]byte {
parts := strings.Split(dbKey, "/")
switch len(parts) {
case 1:
map1, ok := dbValue.(map[interface{}]interface{})
map1, ok := dbValue.(map[string]interface{})
if !ok {
panic(fmt.Errorf("invalid type in db key %s: %T", dbKey, dbValue))
}
for rawID, rawObject := range map1 {
id, ok := rawID.(int)
if !ok {
id, err := strconv.Atoi(rawID)
if err != nil {
panic(fmt.Errorf("invalid id type: got %T expected int", rawID))
}
field, ok := rawObject.(map[string]interface{})
Expand Down

0 comments on commit 68f78c4

Please sign in to comment.