diff --git a/flyteidl/clients/go/coreutils/extract_literal.go b/flyteidl/clients/go/coreutils/extract_literal.go new file mode 100644 index 0000000000..855e2694b1 --- /dev/null +++ b/flyteidl/clients/go/coreutils/extract_literal.go @@ -0,0 +1,81 @@ +// extract_literal.go +// Utility methods to extract a native golang value from a given Literal. +// Usage: +// 1] string literal extraction +// lit, _ := MakeLiteral("test_string") +// val, _ := ExtractFromLiteral(lit) +// 2] integer literal extraction. integer would be extracted in type int64. +// lit, _ := MakeLiteral([]interface{}{1, 2, 3}) +// val, _ := ExtractFromLiteral(lit) +// 3] float literal extraction. float would be extracted in type float64. +// lit, _ := MakeLiteral([]interface{}{1.0, 2.0, 3.0}) +// val, _ := ExtractFromLiteral(lit) +// 4] map of boolean literal extraction. +// mapInstance := map[string]interface{}{ +// "key1": []interface{}{1, 2, 3}, +// "key2": []interface{}{5}, +// } +// lit, _ := MakeLiteral(mapInstance) +// val, _ := ExtractFromLiteral(lit) +// For further examples check the test TestFetchLiteral in extract_literal_test.go + +package coreutils + +import ( + "fmt" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" +) + +func ExtractFromLiteral(literal *core.Literal) (interface{}, error) { + switch literalValue := literal.Value.(type) { + case *core.Literal_Scalar: + switch scalarValue := literalValue.Scalar.Value.(type) { + case *core.Scalar_Primitive: + switch scalarPrimitive := scalarValue.Primitive.Value.(type) { + case *core.Primitive_Integer: + scalarPrimitiveInt := scalarPrimitive.Integer + return scalarPrimitiveInt, nil + case *core.Primitive_FloatValue: + scalarPrimitiveFloat := scalarPrimitive.FloatValue + return scalarPrimitiveFloat, nil + case *core.Primitive_StringValue: + scalarPrimitiveString := scalarPrimitive.StringValue + return scalarPrimitiveString, nil + case *core.Primitive_Boolean: + scalarPrimitiveBoolean := scalarPrimitive.Boolean + return scalarPrimitiveBoolean, nil + case *core.Primitive_Datetime: + scalarPrimitiveDateTime := scalarPrimitive.Datetime + return scalarPrimitiveDateTime, nil + case *core.Primitive_Duration: + scalarPrimitiveDuration := scalarPrimitive.Duration + return scalarPrimitiveDuration, nil + } + return nil, fmt.Errorf("unsupported literal scalar type %T", scalarValue) + } + case *core.Literal_Collection: + collectionValue := literalValue.Collection.Literals + collection := make([]interface{}, len(collectionValue)) + for index, val := range collectionValue { + if collectionElem, err := ExtractFromLiteral(val); err == nil { + collection[index] = collectionElem + } else { + return nil, err + } + } + return collection, nil + case *core.Literal_Map: + mapLiteralValue := literalValue.Map.Literals + mapResult := make(map[string]interface{}, len(mapLiteralValue)) + for key, val := range mapLiteralValue { + if val, err := ExtractFromLiteral(val); err == nil { + mapResult[key] = val + } else { + return nil, err + } + } + return mapResult, nil + } + return nil, fmt.Errorf("unsupported literal type %T", literal) +} diff --git a/flyteidl/clients/go/coreutils/extract_literal_test.go b/flyteidl/clients/go/coreutils/extract_literal_test.go new file mode 100644 index 0000000000..ed1dccbbd6 --- /dev/null +++ b/flyteidl/clients/go/coreutils/extract_literal_test.go @@ -0,0 +1,104 @@ +// extract_literal_test.go +// Test class for the utility methods which extract a native golang value from a flyte Literal. + +package coreutils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFetchLiteral(t *testing.T) { + t.Run("Primitive", func(t *testing.T) { + lit, err := MakeLiteral("test_string") + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, "test_string", val) + }) + + t.Run("Array", func(t *testing.T) { + lit, err := MakeLiteral([]interface{}{1, 2, 3}) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + arr := []interface{}{int64(1), int64(2), int64(3)} + assert.Equal(t, arr, val) + }) + + t.Run("Map", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{1, 2, 3}, + "key2": []interface{}{5}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": []interface{}{int64(1), int64(2), int64(3)}, + "key2": []interface{}{int64(5)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("Map_Booleans", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{true, false, true}, + "key2": []interface{}{false}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + assert.Equal(t, mapInstance, val) + }) + + t.Run("Map_Floats", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": []interface{}{1.0, 2.0, 3.0}, + "key2": []interface{}{1.0}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": []interface{}{float64(1.0), float64(2.0), float64(3.0)}, + "key2": []interface{}{float64(1.0)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("NestedMap", func(t *testing.T) { + mapInstance := map[string]interface{}{ + "key1": map[string]interface{}{"key11": 1.0, "key12": 2.0, "key13": 3.0}, + "key2": map[string]interface{}{"key21": 1.0}, + } + lit, err := MakeLiteral(mapInstance) + assert.NoError(t, err) + val, err := ExtractFromLiteral(lit) + assert.NoError(t, err) + expectedMapInstance := map[string]interface{}{ + "key1": map[string]interface{}{"key11": float64(1.0), "key12": float64(2.0), "key13": float64(3.0)}, + "key2": map[string]interface{}{"key21": float64(1.0)}, + } + assert.Equal(t, expectedMapInstance, val) + }) + + t.Run("Binary", func(t *testing.T) { + s := MakeBinaryLiteral([]byte{'h'}) + assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) + _, err := ExtractFromLiteral(s) + assert.NotNil(t, err) + }) + + t.Run("NoneType", func(t *testing.T) { + p, err := MakeLiteral(nil) + assert.NoError(t, err) + assert.NotNil(t, p.GetScalar()) + _, err = ExtractFromLiteral(p) + assert.NotNil(t, err) + }) +} diff --git a/flyteidl/clients/go/coreutils/literals.go b/flyteidl/clients/go/coreutils/literals.go index 8644d09a75..4f5b855c9c 100644 --- a/flyteidl/clients/go/coreutils/literals.go +++ b/flyteidl/clients/go/coreutils/literals.go @@ -4,10 +4,17 @@ package coreutils import ( "fmt" "reflect" + "strconv" + "strings" "time" - "github.com/golang/protobuf/ptypes" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/storage" + + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/ptypes" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/pkg/errors" ) func MakePrimitive(v interface{}) (*core.Primitive, error) { @@ -142,6 +149,17 @@ func MakeBinaryLiteral(v []byte) *core.Literal { } } +func MakeGenericLiteral(v *structpb.Struct) *core.Literal { + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Generic{ + Generic: v, + }, + }, + }} +} + func MakeLiteral(v interface{}) (*core.Literal, error) { if v == nil { return &core.Literal{ @@ -163,6 +181,18 @@ func MakeLiteral(v interface{}) (*core.Literal, error) { return MakeLiteralForMap(o) case []byte: return MakeBinaryLiteral(v.([]byte)), nil + case *structpb.Struct: + return MakeGenericLiteral(v.(*structpb.Struct)), nil + case *core.Error: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Error{ + Error: v.(*core.Error), + }, + }, + }, + }, nil default: return MakePrimitiveLiteral(o) } @@ -196,11 +226,45 @@ func MakeDefaultLiteralForType(typ *core.LiteralType) (*core.Literal, error) { return MakeLiteral(time.Second) case core.SimpleType_BINARY: return MakeLiteral([]byte{}) - //case core.SimpleType_WAITABLE: - //case core.SimpleType_ERROR: + case core.SimpleType_ERROR: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Error{ + Error: &core.Error{ + Message: "Default Error message", + }, + }, + }, + }, + }, nil + case core.SimpleType_STRUCT: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Generic{ + Generic: &structpb.Struct{}, + }, + }, + }, + }, nil } - return nil, fmt.Errorf("not yet implemented. Default creation is not yet implemented. ") - + return nil, errors.Errorf("Not yet implemented. Default creation is not yet implemented for [%s] ", t.Simple.String()) + case *core.LiteralType_Blob: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Metadata: &core.BlobMetadata{ + Type: t.Blob, + }, + Uri: "/tmp/somepath", + }, + }, + }, + }, + }, nil case *core.LiteralType_CollectionType: single, err := MakeDefaultLiteralForType(t.CollectionType) if err != nil { @@ -235,6 +299,95 @@ func MakeDefaultLiteralForType(typ *core.LiteralType) (*core.Literal, error) { return nil, fmt.Errorf("failed to convert to a known Literal. Input Type [%v] not supported", typ.String()) } +func MakePrimitiveForType(t core.SimpleType, s string) (*core.Primitive, error) { + p := &core.Primitive{} + switch t { + case core.SimpleType_INTEGER: + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "failed to parse integer value") + } + p.Value = &core.Primitive_Integer{Integer: v} + case core.SimpleType_FLOAT: + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Float value") + } + p.Value = &core.Primitive_FloatValue{FloatValue: v} + case core.SimpleType_BOOLEAN: + v, err := strconv.ParseBool(s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Bool value") + } + p.Value = &core.Primitive_Boolean{Boolean: v} + case core.SimpleType_STRING: + p.Value = &core.Primitive_StringValue{StringValue: s} + case core.SimpleType_DURATION: + v, err := time.ParseDuration(s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Duration, valid formats: e.g. 300ms, -1.5h, 2h45m") + } + p.Value = &core.Primitive_Duration{Duration: ptypes.DurationProto(v)} + case core.SimpleType_DATETIME: + v, err := time.Parse(time.RFC3339, s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Datetime in RFC3339 format") + } + ts, err := ptypes.TimestampProto(v) + if err != nil { + return nil, errors.Wrap(err, "failed to convert datetime to proto") + } + p.Value = &core.Primitive_Datetime{Datetime: ts} + default: + return nil, fmt.Errorf("unsupported type %s", t.String()) + } + return p, nil +} + +func MakeLiteralForSimpleType(t core.SimpleType, s string) (*core.Literal, error) { + s = strings.Trim(s, " \n\t") + scalar := &core.Scalar{} + switch t { + case core.SimpleType_STRUCT: + st := &structpb.Struct{} + err := jsonpb.UnmarshalString(s, st) + if err != nil { + return nil, errors.Wrapf(err, "failed to load generic type as json.") + } + scalar.Value = &core.Scalar_Generic{ + Generic: st, + } + case core.SimpleType_BINARY: + scalar.Value = &core.Scalar_Binary{ + Binary: &core.Binary{ + Value: []byte(s), + // TODO Tag not supported at the moment + }, + } + case core.SimpleType_ERROR: + scalar.Value = &core.Scalar_Error{ + Error: &core.Error{ + Message: s, + }, + } + case core.SimpleType_NONE: + scalar.Value = &core.Scalar_NoneType{ + NoneType: &core.Void{}, + } + default: + p, err := MakePrimitiveForType(t, s) + if err != nil { + return nil, err + } + scalar.Value = &core.Scalar_Primitive{Primitive: p} + } + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: scalar, + }, + }, nil +} + func MustMakeLiteral(v interface{}) *core.Literal { p, err := MakeLiteral(v) if err != nil { @@ -260,3 +413,27 @@ func MakeLiteralMap(v map[string]interface{}) (*core.LiteralMap, error) { Literals: literals, }, nil } + +func MakeLiteralForBlob(path storage.DataReference, isDir bool, format string) *core.Literal { + dim := core.BlobType_SINGLE + if isDir { + dim = core.BlobType_MULTIPART + } + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Uri: path.String(), + Metadata: &core.BlobMetadata{ + Type: &core.BlobType{ + Dimensionality: dim, + Format: format, + }, + }, + }, + }, + }, + }, + } +} diff --git a/flyteidl/clients/go/coreutils/literals_test.go b/flyteidl/clients/go/coreutils/literals_test.go index 66da4865a4..e77686241c 100644 --- a/flyteidl/clients/go/coreutils/literals_test.go +++ b/flyteidl/clients/go/coreutils/literals_test.go @@ -1,3 +1,6 @@ +// extract_literal_test.go +// Test class for the utility methods which construct flyte literals. + package coreutils import ( @@ -5,8 +8,11 @@ import ( "testing" "time" - "github.com/golang/protobuf/ptypes" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/storage" + + "github.com/golang/protobuf/ptypes" + structpb "github.com/golang/protobuf/ptypes/struct" "github.com/stretchr/testify/assert" ) @@ -54,6 +60,8 @@ func TestMakePrimitive(t *testing.T) { j, err := ptypes.TimestampProto(v) assert.NoError(t, err) assert.Equal(t, j, p.GetDatetime()) + _, err = MakePrimitive(time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)) + assert.Error(t, err) } { v := time.Second * 10 @@ -170,25 +178,31 @@ func TestMakeBinaryLiteral(t *testing.T) { } func TestMakeDefaultLiteralForType(t *testing.T) { - - tests := [][]interface{}{ - {"Integer", core.SimpleType_INTEGER, "*core.Primitive_Integer"}, - {"Float", core.SimpleType_FLOAT, "*core.Primitive_FloatValue"}, - {"String", core.SimpleType_STRING, "*core.Primitive_StringValue"}, - {"Boolean", core.SimpleType_BOOLEAN, "*core.Primitive_Boolean"}, - {"Duration", core.SimpleType_DURATION, "*core.Primitive_Duration"}, - {"Datetime", core.SimpleType_DATETIME, "*core.Primitive_Datetime"}, + type args struct { + name string + ty core.SimpleType + tyName string + isPrimitive bool } - - for i := range tests { - name := tests[i][0].(string) - ty := tests[i][1].(core.SimpleType) - tyName := tests[i][2].(string) - - t.Run(name, func(t *testing.T) { - l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{Simple: ty}}) + tests := []args{ + {"None", core.SimpleType_NONE, "*core.Scalar_NoneType", false}, + {"Binary", core.SimpleType_BINARY, "*core.Scalar_Binary", false}, + {"Integer", core.SimpleType_INTEGER, "*core.Primitive_Integer", true}, + {"Float", core.SimpleType_FLOAT, "*core.Primitive_FloatValue", true}, + {"String", core.SimpleType_STRING, "*core.Primitive_StringValue", true}, + {"Boolean", core.SimpleType_BOOLEAN, "*core.Primitive_Boolean", true}, + {"Duration", core.SimpleType_DURATION, "*core.Primitive_Duration", true}, + {"Datetime", core.SimpleType_DATETIME, "*core.Primitive_Datetime", true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{Simple: test.ty}}) assert.NoError(t, err) - assert.Equal(t, tyName, reflect.TypeOf(l.GetScalar().GetPrimitive().Value).String()) + if test.isPrimitive { + assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().GetPrimitive().Value).String()) + } else { + assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().Value).String()) + } }) } @@ -197,4 +211,156 @@ func TestMakeDefaultLiteralForType(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) }) + + t.Run("Blob", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}}) + assert.NoError(t, err) + assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String()) + }) + + t.Run("Collection", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_CollectionType{CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}) + assert.NoError(t, err) + assert.Equal(t, "*core.LiteralCollection", reflect.TypeOf(l.GetCollection()).String()) + }) + + t.Run("Map", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_MapValueType{MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}) + assert.NoError(t, err) + assert.Equal(t, "*core.LiteralMap", reflect.TypeOf(l.GetMap()).String()) + }) + + t.Run("error", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_ERROR, + }}) + assert.NoError(t, err) + assert.NotNil(t, l.GetScalar().GetError()) + }) + + t.Run("struct", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }}) + assert.NoError(t, err) + assert.NotNil(t, l.GetScalar().GetGeneric()) + }) +} + +func TestMustMakeDefaultLiteralForType(t *testing.T) { + t.Run("error", func(t *testing.T) { + assert.Panics(t, func() { + MustMakeDefaultLiteralForType(nil) + }) + }) + + t.Run("Blob", func(t *testing.T) { + l := MustMakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}}) + assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String()) + }) +} + +func TestMakePrimitiveForType(t *testing.T) { + n := time.Now() + type args struct { + t core.SimpleType + s string + } + tests := []struct { + name string + args args + want *core.Primitive + wantErr bool + }{ + {"error-type", args{core.SimpleType_NONE, "x"}, nil, true}, + + {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true}, + {"int", args{core.SimpleType_INTEGER, "1"}, MustMakePrimitive(1), false}, + + {"error-bool", args{core.SimpleType_BOOLEAN, "x"}, nil, true}, + {"bool", args{core.SimpleType_BOOLEAN, "true"}, MustMakePrimitive(true), false}, + + {"error-float", args{core.SimpleType_FLOAT, "x"}, nil, true}, + {"float", args{core.SimpleType_FLOAT, "3.1416"}, MustMakePrimitive(3.1416), false}, + + {"string", args{core.SimpleType_STRING, "string"}, MustMakePrimitive("string"), false}, + + {"error-dt", args{core.SimpleType_DATETIME, "x"}, nil, true}, + {"dt", args{core.SimpleType_DATETIME, n.Format(time.RFC3339Nano)}, MustMakePrimitive(n), false}, + + {"error-dur", args{core.SimpleType_DURATION, "x"}, nil, true}, + {"dur", args{core.SimpleType_DURATION, time.Hour.String()}, MustMakePrimitive(time.Hour), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MakePrimitiveForType(tt.args.t, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("MakePrimitiveForType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("MakePrimitiveForType() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMakeLiteralForSimpleType(t *testing.T) { + type args struct { + t core.SimpleType + s string + } + tests := []struct { + name string + args args + want *core.Literal + wantErr bool + }{ + {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true}, + {"int", args{core.SimpleType_INTEGER, "1"}, MustMakeLiteral(1), false}, + + {"error-struct", args{core.SimpleType_STRUCT, "x"}, nil, true}, + {"struct", args{core.SimpleType_STRUCT, `{"x": 1}`}, MustMakeLiteral(&structpb.Struct{Fields: map[string]*structpb.Value{"x": {Kind: &structpb.Value_NumberValue{NumberValue: 1}}}}), false}, + + {"bin", args{core.SimpleType_BINARY, "x"}, MustMakeLiteral([]byte("x")), false}, + + {"error", args{core.SimpleType_ERROR, "err"}, MustMakeLiteral(&core.Error{Message: "err"}), false}, + + {"none", args{core.SimpleType_NONE, "null"}, MustMakeLiteral(nil), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MakeLiteralForSimpleType(tt.args.t, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("MakeLiteralForSimpleType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("MakeLiteralForSimpleType() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMakeLiteralForBlob(t *testing.T) { + type args struct { + path storage.DataReference + isDir bool + format string + } + tests := []struct { + name string + args args + want *core.Blob + }{ + {"simple-key", args{path: "/key", isDir: false, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_SINGLE}}}}, + {"simple-dir", args{path: "/key", isDir: true, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_MULTIPART}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := MakeLiteralForBlob(tt.args.path, tt.args.isDir, tt.args.format); !reflect.DeepEqual(got.GetScalar().GetBlob(), tt.want) { + t.Errorf("MakeLiteralForBlob() = %v, want %v", got, tt.want) + } + }) + } } diff --git a/flyteidl/go.mod b/flyteidl/go.mod index 44ad59b9d7..e3d7bc926e 100644 --- a/flyteidl/go.mod +++ b/flyteidl/go.mod @@ -15,6 +15,7 @@ require ( github.com/influxdata/influxdb v1.7.9 github.com/lyft/flytestdlib v0.3.0 github.com/mitchellh/mapstructure v1.1.2 + github.com/pkg/errors v0.9.1 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/satori/uuid v1.2.0 // indirect github.com/spf13/pflag v1.0.5 diff --git a/flyteidl/go.sum b/flyteidl/go.sum index c7770e6e2a..a646140336 100644 --- a/flyteidl/go.sum +++ b/flyteidl/go.sum @@ -6,6 +6,7 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU= +cloud.google.com/go v0.52.0 h1:GGslhk/BU052LPlnI1vpp3fcbUs+hQ3E+Doti/3/vF8= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -14,20 +15,26 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v10.2.1-beta+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v38.2.0+incompatible h1:ZeCdp1E/V5lI8oLR/BjWQh0OW9aFBYlgXGKRVIWNPXY= github.com/Azure/azure-sdk-for-go v38.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.2/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.4 h1:1cM+NmKw91+8h5vfjgzK4ZGLuN72k87XVZBWyGwNjUM= github.com/Azure/go-autorest/autorest v0.9.4/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -45,6 +52,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.16/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1/go.mod h1:jvdWlw8vowVGnZqSDC7yhPd7AifQeQbRDkZcQXV2nRg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -61,6 +69,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -75,6 +84,7 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -120,6 +130,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -149,12 +160,14 @@ github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graymeta/stow v0.0.0-20190522170649-903027f87de7/go.mod h1:B24dekNjtWVeREK+dyMHtI22d85VzCT+sX5bVWDtjoA= +github.com/graymeta/stow v0.2.4 h1:qDGstknYXqcnmBQ5TRJtxD9Qv1MuRbYRhLoSMeUDs7U= github.com/graymeta/stow v0.2.4/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrAeV2Kw= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -181,6 +194,7 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.7.9 h1:uSeBTNO4rBkbp1Be5FKRsAmglM9nlx25TzVQRQt1An4= github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -238,6 +252,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.49-0.20190728102658-a24ef33bc9b7/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -296,6 +311,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/satori/uuid v1.2.0/go.mod h1:B8HLsPLik/YNn6KKWVMDJ8nzCL8RP5WyfsnmvnAEwIU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -339,6 +355,7 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -490,6 +507,7 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.11.1-0.20191020000718-bf72a15fd9e9/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=