diff --git a/cmd/report/query.go b/cmd/report/query.go index 255bb47..132f11f 100644 --- a/cmd/report/query.go +++ b/cmd/report/query.go @@ -59,7 +59,7 @@ func (o *queryCmdOptions) run() error { } request := reportV1.ExecuteDbQueryRequest{ - Query: q.query, + Query: q.Query, Limit: o.pageSize, } @@ -77,10 +77,10 @@ func (o *queryCmdOptions) run() error { } var formatter format.Formatter - if q.template == "" { + if q.Template == "" { formatter, err = format.NewFormatter("", w, o.format, o.goTemplate) } else { - formatter, err = format.NewFormatter("", w, "template", q.template) + formatter, err = format.NewFormatter("", w, "template", q.Template) } if err != nil { return fmt.Errorf("failed to create formatter: %w", err) @@ -131,8 +131,8 @@ func newQueryCmd() *cobra.Command { q := listStoredQueries(d) var names []string for _, s := range q { - if strings.HasPrefix(s.name, toComplete) { - names = append(names, s.name+"\t"+s.description) + if strings.HasPrefix(s.Name, toComplete) { + names = append(names, s.Name+"\t"+s.Description) } } return names, cobra.ShellCompDirectiveDefault @@ -157,10 +157,10 @@ func newQueryCmd() *cobra.Command { // query is a struct for holding query definitions type query struct { - name string - description string - query string - template string + Name string `json:"name"` + Description string `json:"description"` + Query string `json:"query"` + Template string `json:"template"` } func (o *queryCmdOptions) parseQuery() (*query, error) { @@ -168,7 +168,7 @@ func (o *queryCmdOptions) parseQuery() (*query, error) { if strings.HasPrefix(o.queryOrFile, "r.") { q = &query{ - query: o.queryOrFile, + Query: o.queryOrFile, } } else { filename, err := findQueryFile(o.queryOrFile) @@ -182,7 +182,7 @@ func (o *queryCmdOptions) parseQuery() (*query, error) { } } - q.query = fmt.Sprintf(q.query, o.queryArgs...) + q.Query = fmt.Sprintf(q.Query, o.queryArgs...) return q, nil } @@ -228,9 +228,9 @@ func readQuery(name string) (*query, error) { return nil, err } } else { - qd.query = string(data) + qd.Query = string(data) } - qd.name = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name)) + qd.Name = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name)) return qd, nil } diff --git a/cmd/report/query_test.go b/cmd/report/query_test.go new file mode 100644 index 0000000..88bbc7f --- /dev/null +++ b/cmd/report/query_test.go @@ -0,0 +1,92 @@ +package report + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_queryCmdOptions_parseQuery(t *testing.T) { + tests := []struct { + name string + fieldsFromFlags *queryCmdOptions + fieldsAfterParse *queryCmdOptions + want *query + wantErr assert.ErrorAssertionFunc + }{ + {"template file", + &queryCmdOptions{queryOrFile: "testdata/template1.yaml"}, + &queryCmdOptions{ + queryOrFile: "testdata/template1.yaml", + }, + &query{ + Name: "template1", + Description: "Example template\n", + Query: "r.db('veidemann').table('config_crawl_entities')\n", + Template: "{{.id}} {{.meta.name}}\n", + }, + assert.NoError}, + {"template file with template flag", + &queryCmdOptions{ + queryOrFile: "testdata/template1.yaml", + goTemplate: "{{.id}}", + }, + &queryCmdOptions{ + queryOrFile: "testdata/template1.yaml", + goTemplate: "{{.id}}", + }, + &query{ + Name: "template1", + Description: "Example template\n", + Query: "r.db('veidemann').table('config_crawl_entities')\n", + Template: "{{.id}} {{.meta.name}}\n", + }, + assert.NoError}, + {"template file with format flag", + &queryCmdOptions{ + queryOrFile: "testdata/template1.yaml", + format: "yaml", + }, + &queryCmdOptions{ + queryOrFile: "testdata/template1.yaml", + format: "yaml", + }, + &query{ + Name: "template1", + Description: "Example template\n", + Query: "r.db('veidemann').table('config_crawl_entities')\n", + Template: "{{.id}} {{.meta.name}}\n", + }, + assert.NoError}, + {"nonexisting template file", + &queryCmdOptions{ + queryOrFile: "missing.yaml", + }, + &queryCmdOptions{queryOrFile: "missing.yaml"}, + nil, + assert.Error}, + {"query", + &queryCmdOptions{ + queryOrFile: "r.db('veidemann').table('config_crawl_entities')", + }, + &queryCmdOptions{ + queryOrFile: "r.db('veidemann').table('config_crawl_entities')", + }, + &query{ + Name: "", + Description: "", + Query: "r.db('veidemann').table('config_crawl_entities')", + Template: "", + }, + assert.NoError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.fieldsFromFlags.parseQuery() + if !tt.wantErr(t, err, "parseQuery()") { + return + } + assert.Equalf(t, tt.fieldsAfterParse, tt.fieldsFromFlags, "Fields after parseQuery()") + assert.Equalf(t, tt.want, got, "parseQuery()") + }) + } +} diff --git a/cmd/report/testdata/template1.yaml b/cmd/report/testdata/template1.yaml new file mode 100644 index 0000000..c4299c1 --- /dev/null +++ b/cmd/report/testdata/template1.yaml @@ -0,0 +1,8 @@ +description: | + Example template + +query: | + r.db('veidemann').table('config_crawl_entities') + +template: | + {{.id}} {{.meta.name}} diff --git a/format/JsonFormatter.go b/format/JsonFormatter.go index 86434a3..6e9ec68 100644 --- a/format/JsonFormatter.go +++ b/format/JsonFormatter.go @@ -14,6 +14,7 @@ package format import ( + "encoding/json" "fmt" "io" "reflect" @@ -28,17 +29,16 @@ type jsonFormatter struct { // newJsonFormatter creates a new json formatter func newJsonFormatter(s *MarshalSpec) Formatter { - return &jsonFormatter{ - MarshalSpec: s, + return &preFormatter{ + &jsonFormatter{ + MarshalSpec: s, + }, } } // WriteRecord writes a record to the formatters writer func (jf *jsonFormatter) WriteRecord(record interface{}) error { switch v := record.(type) { - case string: - _, err := fmt.Fprint(jf.rWriter, v) - return err case proto.Message: var values reflect.Value values = reflect.ValueOf(v).Elem().FieldByName("Value") @@ -65,7 +65,12 @@ func (jf *jsonFormatter) WriteRecord(record interface{}) error { } } default: - return fmt.Errorf("illegal record type '%T'", record) + j, err := json.Marshal(v) + if err != nil { + return err + } + _, err = fmt.Fprint(jf.rWriter, string(j)) + return err } return nil } diff --git a/format/TemplateFormatter.go b/format/TemplateFormatter.go index d123078..b810bd7 100644 --- a/format/TemplateFormatter.go +++ b/format/TemplateFormatter.go @@ -46,7 +46,7 @@ func newTemplateFormatter(s *MarshalSpec) (Formatter, error) { return nil, err } t.parsedTemplate = pt - return t, nil + return &preFormatter{t}, nil } // WriteRecord writes a record to the formatters writer @@ -108,16 +108,6 @@ func parseTemplate(templateString string) (*template.Template, error) { return fmt.Sprintf("%-24.24s", ts.AsTime().Format(time.RFC3339)) } }, - "rethinktime": func(ts map[string]interface{}) string { - if ts == nil { - return " " - } else { - dateTime, _ := ts["dateTime"].(map[string]interface{}) - date, _ := dateTime["date"].(map[string]interface{}) - time, _ := dateTime["time"].(map[string]interface{}) - return fmt.Sprintf("%04.f-%02.f-%02.fT%02.f:%02.f:%02.f", date["year"], date["month"], date["day"], time["hour"], time["minute"], time["second"]) - } - }, "json": func(v interface{}) (string, error) { if v == nil { return "", nil diff --git a/format/YamlFormatter.go b/format/YamlFormatter.go index 1b3db15..332b105 100644 --- a/format/YamlFormatter.go +++ b/format/YamlFormatter.go @@ -29,29 +29,16 @@ type yamlFormatter struct { // newYamlFormatter creates a new yaml formatter func newYamlFormatter(s *MarshalSpec) Formatter { - return &yamlFormatter{ - MarshalSpec: s, + return &preFormatter{ + &yamlFormatter{ + MarshalSpec: s, + }, } } // WriteRecord writes a record to the formatters writer func (yf *yamlFormatter) WriteRecord(record interface{}) error { switch v := record.(type) { - case string: - final, err := yaml.JSONToYAML([]byte(v)) - if err != nil { - fmt.Printf("err: %v\n", err) - return err - } - - _, err = fmt.Fprint(yf.rWriter, string(final)) - if err != nil { - return err - } - _, err = fmt.Fprintln(yf.rWriter, "---") - if err != nil { - return err - } case proto.Message: var values reflect.Value values = reflect.ValueOf(v).Elem().FieldByName("Value") @@ -88,7 +75,20 @@ func (yf *yamlFormatter) WriteRecord(record interface{}) error { return err } default: - return fmt.Errorf("illegal record type '%T'", record) + final, err := yaml.Marshal(record) + if err != nil { + fmt.Printf("err: %v\n", err) + return err + } + + _, err = fmt.Fprint(yf.rWriter, string(final)) + if err != nil { + return err + } + _, err = fmt.Fprintln(yf.rWriter, "---") + if err != nil { + return err + } } return nil } diff --git a/format/formatter.go b/format/formatter.go index e87fcc7..14ae185 100644 --- a/format/formatter.go +++ b/format/formatter.go @@ -21,15 +21,16 @@ import ( "encoding/json" "errors" "fmt" - "io" - "os" - "strings" - "sync" - "github.com/invopop/yaml" "github.com/nlnwa/veidemann-api/go/config/v1" "github.com/rs/zerolog/log" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "io" + "os" + "strings" + "sync" + "time" ) var jsonMarshaler = &protojson.MarshalOptions{EmitUnpopulated: true} @@ -41,12 +42,158 @@ var res embed.FS // templateDir is the directory where the templates are located const templateDir = "res/" -// Formatter is the interface for formatters +// Formatter is the interface for formatters. +// Formatters should not be instantiated directly, but through the NewFormatter function. type Formatter interface { + // WriteRecord writes a record to the formatter. + // The input record is guaranteed to be a protobuf message + // or the result of parsing a json formatted string into an interface (see https://pkg.go.dev/encoding/json#Unmarshal). WriteRecord(interface{}) error Close() error } +// anyElement is a struct that can hold any type of element. +// It is used as input to the json unmarshaler. +type anyElement struct { + v interface{} +} + +// UnmarshalJSON implements the encoding/json.Unmarshaler interface. +// The function uses encoding.json.Unmarshal to unmarshal the input. +// If the input is a map[string]interface{}, the map is traversed to find any RethinkDb dates +// which are converted to RFC3339 formatted strings. +func (r *anyElement) UnmarshalJSON(b []byte) error { + var i interface{} + err := json.Unmarshal(b, &i) + if err != nil { + return err + } + + switch j := i.(type) { + case map[string]interface{}: + if d, ok := r.formatDate(j); ok { + r.v = d + return nil + } + + r.traverseMap(&j) + r.v = j + default: + r.v = i + } + + return err +} + +func (r *anyElement) traverseMap(i *map[string]interface{}) { + for k, v := range *i { + if m, ok := v.(map[string]interface{}); ok { + if d, ok := r.formatDate(m); ok { + (*i)[k] = d + } else { + r.traverseMap(&m) + } + } + } +} + +// getAsInt returns the value as an int if it is a float64 or int +func getAsInt(v interface{}) (int, bool) { + switch i := v.(type) { + case float64: + return int(i), true + case int: + return i, true + default: + return 0, false + } +} + +// formatDate first checks if i is a RethinkDb date. +// If so, date is the result of converting the date to a RFC3339 formatted string and isDate is true. +// Otherwise, date is the empty string and isDate is false. +func (r *anyElement) formatDate(element map[string]interface{}) (date string, isDate bool) { + var year, month, day, hour, minute, second, nano, offset int + + if dateTime, ok := element["dateTime"].(map[string]interface{}); !ok { + return "", false + } else { + if date, ok := dateTime["date"].(map[string]interface{}); !ok { + return "", false + } else { + if year, ok = getAsInt(date["year"]); !ok { + return "", false + } + if month, ok = getAsInt(date["month"]); !ok { + return "", false + } + if day, ok = getAsInt(date["day"]); !ok { + return "", false + } + } + if tm, ok := dateTime["time"].(map[string]interface{}); !ok { + return "", false + } else { + if hour, ok = getAsInt(tm["hour"]); !ok { + return "", false + } + if minute, ok = getAsInt(tm["minute"]); !ok { + return "", false + } + if second, ok = getAsInt(tm["second"]); !ok { + return "", false + } + if nano, ok = getAsInt(tm["nano"]); !ok { + return "", false + } + } + } + if of, ok := element["offset"].(map[string]interface{}); !ok { + return "", false + } else { + if offset, ok = getAsInt(of["totalSeconds"]); !ok { + return "", false + } + } + tz := time.UTC + if offset != 0 { + tz = time.FixedZone(fmt.Sprintf("OFF%.d", offset), offset) + } + d := time.Date(year, time.Month(month), day, hour, minute, second, nano, tz) + return d.Format(time.RFC3339Nano), true +} + +// preFormatter wraps a formatter and converts json strings to objects +type preFormatter struct { + formatter Formatter +} + +// WriteRecord implements the Formatter interface. +// If the input record is a string, it is parsed as json and the result is passed to the wrapped formatter. +// If the input record is a protobuf message, it is passed to the wrapped formatter. +// Otherwise, an error is returned. +func (p *preFormatter) WriteRecord(record interface{}) error { + switch v := record.(type) { + case string: + var j anyElement + err := json.Unmarshal([]byte(v), &j) + if err != nil { + return fmt.Errorf("failed to parse json: %w", err) + } + record = j.v + case proto.Message: + // Do nothing, just pass through + default: + return fmt.Errorf("unsupported type '%T'", v) + } + + return p.formatter.WriteRecord(record) +} + +func (p *preFormatter) Close() error { + return p.formatter.Close() +} + // MarshalSpec is the specification for a formatter type MarshalSpec struct { ObjectType string