From a818d2bd7e3ca2889b75304155260eae71d51aa0 Mon Sep 17 00:00:00 2001 From: Scott Ganyo Date: Tue, 6 Jun 2023 16:11:47 -0700 Subject: [PATCH] support both serialized protos and YAML --- .../cmd/check/rules/rule1000/rule1000.go | 4 +- .../cmd/check/rules/rule1001/rule1001.go | 4 +- .../cmd/check/rules/rule1002/rule1002.go | 4 +- .../cmd/check/rules/rule1003/rule1003.go | 4 +- .../cmd/compute/complexity/complexity_test.go | 10 +- .../cmd/compute/conformance/conformance.go | 4 +- .../compute/conformance/conformance_test.go | 4 +- cmd/registry/cmd/compute/lint/lint_test.go | 8 +- .../cmd/compute/lintstats/lintstats.go | 7 +- cmd/registry/cmd/compute/score/score.go | 4 +- .../cmd/compute/scorecard/scorecard.go | 4 +- .../cmd/compute/vocabulary/vocabulary_test.go | 8 +- cmd/registry/cmd/resolve/resolve.go | 4 +- cmd/registry/patch/artifact.go | 20 +++ cmd/registry/patch/patch_test.go | 131 +++++++++++------- cmd/registry/scoring/expression.go | 21 +-- cmd/registry/scoring/score.go | 5 +- cmd/registry/scoring/score_test.go | 4 +- cmd/registry/scoring/scorecard.go | 7 +- cmd/registry/scoring/scorecard_test.go | 4 +- .../scoring/scoring-timestamp_test.go | 4 +- pkg/mime/types.go | 4 + 22 files changed, 162 insertions(+), 107 deletions(-) diff --git a/cmd/registry/cmd/check/rules/rule1000/rule1000.go b/cmd/registry/cmd/check/rules/rule1000/rule1000.go index 5a536e9b5..754cf8b4f 100644 --- a/cmd/registry/cmd/check/rules/rule1000/rule1000.go +++ b/cmd/registry/cmd/check/rules/rule1000/rule1000.go @@ -20,13 +20,13 @@ import ( "strings" "github.com/apigee/registry/cmd/registry/cmd/check/lint" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/apihub" "github.com/apigee/registry/pkg/application/check" "github.com/apigee/registry/pkg/mime" "github.com/apigee/registry/pkg/names" "github.com/apigee/registry/pkg/visitor" "github.com/apigee/registry/rpc" - "google.golang.org/protobuf/proto" ) var ruleNum = 1000 @@ -92,7 +92,7 @@ var requiredArtifacts = &lint.ProjectRule{ func checkTaxonomies(a *rpc.Artifact) []*check.Problem { message, err := mime.MessageForMimeType(a.GetMimeType()) if err == nil { - err = proto.Unmarshal(a.GetContents(), message) + err = patch.UnmarshalContents(a.GetContents(), a.GetMimeType(), message) } if err != nil { return []*check.Problem{{ diff --git a/cmd/registry/cmd/check/rules/rule1001/rule1001.go b/cmd/registry/cmd/check/rules/rule1001/rule1001.go index 0633c832b..8a83afd8e 100644 --- a/cmd/registry/cmd/check/rules/rule1001/rule1001.go +++ b/cmd/registry/cmd/check/rules/rule1001/rule1001.go @@ -22,11 +22,11 @@ import ( "strings" "github.com/apigee/registry/cmd/registry/cmd/check/lint" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/apihub" "github.com/apigee/registry/pkg/application/check" "github.com/apigee/registry/pkg/names" "github.com/apigee/registry/rpc" - "google.golang.org/protobuf/proto" ) var ruleNum = 1001 @@ -84,7 +84,7 @@ func taxonomies(ctx context.Context, project names.Project) *TaxList { }) if err == nil { tl := new(apihub.TaxonomyList) - if err = proto.Unmarshal(ac.Data, tl); err == nil { + if err = patch.UnmarshalContents(ac.Data, ac.GetContentType(), tl); err == nil { return &TaxList{tl} } } diff --git a/cmd/registry/cmd/check/rules/rule1002/rule1002.go b/cmd/registry/cmd/check/rules/rule1002/rule1002.go index bc10054b2..837950a8f 100644 --- a/cmd/registry/cmd/check/rules/rule1002/rule1002.go +++ b/cmd/registry/cmd/check/rules/rule1002/rule1002.go @@ -20,10 +20,10 @@ import ( "strings" "github.com/apigee/registry/cmd/registry/cmd/check/lint" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/check" "github.com/apigee/registry/pkg/mime" "github.com/apigee/registry/rpc" - "google.golang.org/protobuf/proto" ) var ruleNum = 1002 @@ -72,7 +72,7 @@ var internalMimeTypeContents = &lint.FieldRule{ } // does not validate contents, just proves type compatibility - err = proto.Unmarshal(contents, message) + err = patch.UnmarshalContents(contents, declared, message) if err != nil { return []*check.Problem{{ Severity: check.Problem_ERROR, diff --git a/cmd/registry/cmd/check/rules/rule1003/rule1003.go b/cmd/registry/cmd/check/rules/rule1003/rule1003.go index 619b79164..2a6fbc440 100644 --- a/cmd/registry/cmd/check/rules/rule1003/rule1003.go +++ b/cmd/registry/cmd/check/rules/rule1003/rule1003.go @@ -21,11 +21,11 @@ import ( "strings" "github.com/apigee/registry/cmd/registry/cmd/check/lint" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/apihub" "github.com/apigee/registry/pkg/application/check" "github.com/apigee/registry/pkg/names" "github.com/apigee/registry/rpc" - "google.golang.org/protobuf/proto" ) var ruleNum = 1003 @@ -62,7 +62,7 @@ var stateIsValidLifecycleStage = &lint.ApiVersionRule{ }) if err == nil { lc = new(apihub.Lifecycle) - err = proto.Unmarshal(ac.Data, lc) + err = patch.UnmarshalContents(ac.Data, ac.ContentType, lc) } if err != nil { return []*check.Problem{{ diff --git a/cmd/registry/cmd/compute/complexity/complexity_test.go b/cmd/registry/cmd/compute/complexity/complexity_test.go index d76c2d848..981ea4496 100644 --- a/cmd/registry/cmd/compute/complexity/complexity_test.go +++ b/cmd/registry/cmd/compute/complexity/complexity_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/apigee/registry/cmd/registry/cmd/apply" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/connection" "github.com/apigee/registry/pkg/connection/grpctest" "github.com/apigee/registry/pkg/names" @@ -33,7 +34,6 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" ) @@ -81,7 +81,7 @@ func TestComputeComplexity(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("protos").Artifact("complexity") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { complexity := &metrics.Complexity{} - err = proto.Unmarshal(message.Contents, complexity) + err = patch.UnmarshalContents(message.Contents, message.MimeType, complexity) if err != nil { return err } @@ -106,7 +106,7 @@ func TestComputeComplexity(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("openapi").Artifact("complexity") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { complexity := &metrics.Complexity{} - err = proto.Unmarshal(message.Contents, complexity) + err = patch.UnmarshalContents(message.Contents, message.MimeType, complexity) if err != nil { return err } @@ -131,7 +131,7 @@ func TestComputeComplexity(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("discovery").Artifact("complexity") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { complexity := &metrics.Complexity{} - err = proto.Unmarshal(message.Contents, complexity) + err = patch.UnmarshalContents(message.Contents, message.MimeType, complexity) if err != nil { return err } @@ -251,7 +251,7 @@ func TestComputeComplexityValues(t *testing.T) { t.Fatalf("Failed getting artifact contents %s: %s", test.getPattern, err) } gotProto := &metrics.Complexity{} - if err := proto.Unmarshal(contents.GetData(), gotProto); err != nil { + if err := patch.UnmarshalContents(contents.GetData(), contents.GetContentType(), gotProto); err != nil { t.Fatalf("Failed to unmarshal artifact: %s", err) } opts := cmp.Options{protocmp.Transform()} diff --git a/cmd/registry/cmd/compute/conformance/conformance.go b/cmd/registry/cmd/compute/conformance/conformance.go index 486fe96f4..d227132d2 100644 --- a/cmd/registry/cmd/compute/conformance/conformance.go +++ b/cmd/registry/cmd/compute/conformance/conformance.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/apigee/registry/cmd/registry/conformance" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/tasks" "github.com/apigee/registry/pkg/application/style" "github.com/apigee/registry/pkg/connection" @@ -28,7 +29,6 @@ import ( "github.com/apigee/registry/pkg/visitor" "github.com/apigee/registry/rpc" "github.com/spf13/cobra" - "google.golang.org/protobuf/proto" ) var styleguideFilter = fmt.Sprintf("mime_type.contains('%s')", mime.MimeTypeForKind("StyleGuide")) @@ -78,7 +78,7 @@ func Command() *cobra.Command { guides := make([]*style.StyleGuide, 0) if err := visitor.ListArtifacts(ctx, client, name.Project().Artifact("-"), styleguideFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error { guide := new(style.StyleGuide) - if err := proto.Unmarshal(artifact.GetContents(), guide); err != nil { + if err := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), guide); err != nil { log.FromContext(ctx).WithError(err).Debugf("Unmarshal() to StyleGuide failed on artifact: %s", artifact.GetName()) return nil } diff --git a/cmd/registry/cmd/compute/conformance/conformance_test.go b/cmd/registry/cmd/compute/conformance/conformance_test.go index 941dcd200..296d46531 100644 --- a/cmd/registry/cmd/compute/conformance/conformance_test.go +++ b/cmd/registry/cmd/compute/conformance/conformance_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/apigee/registry/cmd/registry/cmd/apply" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/style" "github.com/apigee/registry/pkg/connection/grpctest" "github.com/apigee/registry/rpc" @@ -32,7 +33,6 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" ) @@ -477,7 +477,7 @@ func TestConformance(t *testing.T) { } gotProto := &style.ConformanceReport{} - if err := proto.Unmarshal(contents.GetData(), gotProto); err != nil { + if err := patch.UnmarshalContents(contents.GetData(), contents.GetContentType(), gotProto); err != nil { t.Fatalf("Failed to unmarshal artifact: %s", err) } diff --git a/cmd/registry/cmd/compute/lint/lint_test.go b/cmd/registry/cmd/compute/lint/lint_test.go index f7dd44b29..2d90bf56f 100644 --- a/cmd/registry/cmd/compute/lint/lint_test.go +++ b/cmd/registry/cmd/compute/lint/lint_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/apigee/registry/cmd/registry/cmd/apply" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/application/style" "github.com/apigee/registry/pkg/connection" "github.com/apigee/registry/pkg/connection/grpctest" @@ -26,7 +27,6 @@ import ( "github.com/apigee/registry/pkg/visitor" "github.com/apigee/registry/rpc" "github.com/apigee/registry/server/registry" - "google.golang.org/protobuf/proto" ) // TestMain will set up a local RegistryServer and grpc.Server for all @@ -104,7 +104,7 @@ func TestComputeLint(t *testing.T) { artifactName := specName.Artifact("lint-test") if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { var lint style.Lint - if err = proto.Unmarshal(message.Contents, &lint); err != nil { + if err = patch.UnmarshalContents(message.Contents, message.MimeType, &lint); err != nil { return err } return nil @@ -123,7 +123,7 @@ func TestComputeLint(t *testing.T) { artifactName := specName.Artifact("lint-test") if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { var lint style.Lint - if err = proto.Unmarshal(message.Contents, &lint); err != nil { + if err = patch.UnmarshalContents(message.Contents, message.MimeType, &lint); err != nil { return err } return nil @@ -142,7 +142,7 @@ func TestComputeLint(t *testing.T) { artifactName := specName.Artifact("lint-test") if err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { var lint style.Lint - if err = proto.Unmarshal(message.Contents, &lint); err != nil { + if err = patch.UnmarshalContents(message.Contents, message.MimeType, &lint); err != nil { return err } return nil diff --git a/cmd/registry/cmd/compute/lintstats/lintstats.go b/cmd/registry/cmd/compute/lintstats/lintstats.go index a2ef3560f..5e5d0b1b6 100644 --- a/cmd/registry/cmd/compute/lintstats/lintstats.go +++ b/cmd/registry/cmd/compute/lintstats/lintstats.go @@ -19,6 +19,7 @@ import ( "fmt" "sort" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/gapic" "github.com/apigee/registry/pkg/application/style" "github.com/apigee/registry/pkg/connection" @@ -135,7 +136,7 @@ func computeLintStatsSpecs(ctx context.Context, } lint := &style.Lint{} - err = proto.Unmarshal(contents.GetData(), lint) + err = patch.UnmarshalContents(contents.GetData(), contents.GetContentType(), lint) if err != nil { return nil } @@ -153,7 +154,7 @@ func computeLintStatsSpecs(ctx context.Context, return nil // ignore missing results } complexity := &metrics.Complexity{} - err := proto.Unmarshal(contents.GetData(), complexity) + err := patch.UnmarshalContents(contents.GetData(), contents.GetContentType(), complexity) if err != nil { return nil } @@ -283,7 +284,7 @@ func aggregateLintStats(ctx context.Context, return // ignore missing results } stats := &style.LintStats{} - err := proto.Unmarshal(contents.GetData(), stats) + err := patch.UnmarshalContents(contents.GetData(), contents.GetContentType(), stats) if err != nil { return } diff --git a/cmd/registry/cmd/compute/score/score.go b/cmd/registry/cmd/compute/score/score.go index b9e349f67..31a6f40af 100644 --- a/cmd/registry/cmd/compute/score/score.go +++ b/cmd/registry/cmd/compute/score/score.go @@ -17,6 +17,7 @@ package score import ( "context" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/cmd/registry/scoring" "github.com/apigee/registry/cmd/registry/tasks" @@ -24,7 +25,6 @@ import ( "github.com/apigee/registry/pkg/connection" "github.com/apigee/registry/rpc" "github.com/spf13/cobra" - "google.golang.org/protobuf/proto" ) func Command() *cobra.Command { @@ -67,7 +67,7 @@ func Command() *cobra.Command { for _, d := range scoreDefinitions { // Extract definition definition := &scoring_message.ScoreDefinition{} - if err := proto.Unmarshal(d.GetContents(), definition); err != nil { + if err := patch.UnmarshalContents(d.GetContents(), d.GetMimeType(), definition); err != nil { return err } mergedPattern, mergedFilter, err := scoring.GenerateCombinedPattern(definition.GetTargetResource(), inputPattern, filter) diff --git a/cmd/registry/cmd/compute/scorecard/scorecard.go b/cmd/registry/cmd/compute/scorecard/scorecard.go index e5d963c19..ccd701d77 100644 --- a/cmd/registry/cmd/compute/scorecard/scorecard.go +++ b/cmd/registry/cmd/compute/scorecard/scorecard.go @@ -19,13 +19,13 @@ import ( scoring_message "github.com/apigee/registry/pkg/application/scoring" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/cmd/registry/scoring" "github.com/apigee/registry/cmd/registry/tasks" "github.com/apigee/registry/pkg/connection" "github.com/apigee/registry/rpc" "github.com/spf13/cobra" - "google.golang.org/protobuf/proto" ) func Command() *cobra.Command { @@ -68,7 +68,7 @@ func Command() *cobra.Command { for _, d := range scoreCardDefinitions { // Extract definition definition := &scoring_message.ScoreCardDefinition{} - if err := proto.Unmarshal(d.GetContents(), definition); err != nil { + if err := patch.UnmarshalContents(d.GetContents(), d.GetMimeType(), definition); err != nil { return err } mergedPattern, mergedFilter, err := scoring.GenerateCombinedPattern(definition.GetTargetResource(), inputPattern, filter) diff --git a/cmd/registry/cmd/compute/vocabulary/vocabulary_test.go b/cmd/registry/cmd/compute/vocabulary/vocabulary_test.go index 72a49932c..e02193d97 100644 --- a/cmd/registry/cmd/compute/vocabulary/vocabulary_test.go +++ b/cmd/registry/cmd/compute/vocabulary/vocabulary_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/apigee/registry/cmd/registry/cmd/apply" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/pkg/connection" "github.com/apigee/registry/pkg/connection/grpctest" "github.com/apigee/registry/pkg/names" @@ -26,7 +27,6 @@ import ( "github.com/apigee/registry/rpc" "github.com/apigee/registry/server/registry" metrics "github.com/google/gnostic/metrics" - "google.golang.org/protobuf/proto" ) // TestMain will set up a local RegistryServer and grpc.Server for all @@ -73,7 +73,7 @@ func TestComputeVocabulary(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("protos").Artifact("vocabulary") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { vocabulary := &metrics.Vocabulary{} - err = proto.Unmarshal(message.Contents, vocabulary) + err = patch.UnmarshalContents(message.Contents, message.MimeType, vocabulary) if err != nil { return err } @@ -98,7 +98,7 @@ func TestComputeVocabulary(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("openapi").Artifact("vocabulary") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { vocabulary := &metrics.Vocabulary{} - err = proto.Unmarshal(message.Contents, vocabulary) + err = patch.UnmarshalContents(message.Contents, message.MimeType, vocabulary) if err != nil { return err } @@ -123,7 +123,7 @@ func TestComputeVocabulary(t *testing.T) { artifactName := project.Api("apigeeregistry").Version("v1").Spec("discovery").Artifact("vocabulary") err = visitor.GetArtifact(ctx, registryClient, artifactName, true, func(ctx context.Context, message *rpc.Artifact) error { vocabulary := &metrics.Vocabulary{} - err = proto.Unmarshal(message.Contents, vocabulary) + err = patch.UnmarshalContents(message.Contents, message.MimeType, vocabulary) if err != nil { return err } diff --git a/cmd/registry/cmd/resolve/resolve.go b/cmd/registry/cmd/resolve/resolve.go index 11c9e10f6..7d64cbfc1 100644 --- a/cmd/registry/cmd/resolve/resolve.go +++ b/cmd/registry/cmd/resolve/resolve.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/apigee/registry/cmd/registry/controller" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/tasks" controller_message "github.com/apigee/registry/pkg/application/controller" "github.com/apigee/registry/pkg/connection" @@ -27,7 +28,6 @@ import ( "github.com/apigee/registry/rpc" "github.com/google/uuid" "github.com/spf13/cobra" - "google.golang.org/protobuf/proto" ) func fetchManifest( @@ -45,7 +45,7 @@ func fetchManifest( } contents := body.GetData() - err = proto.Unmarshal(contents, manifest) + err = patch.UnmarshalContents(contents, body.GetContentType(), manifest) if err != nil { return nil, err } diff --git a/cmd/registry/patch/artifact.go b/cmd/registry/patch/artifact.go index dad1ed4fa..6d2edf05d 100644 --- a/cmd/registry/patch/artifact.go +++ b/cmd/registry/patch/artifact.go @@ -216,3 +216,23 @@ func populateIdAndKind(bytes []byte, kind, id string) ([]byte, error) { return rBytes, nil } + +func UnmarshalContents(contents []byte, mimeType string, message proto.Message) error { + if !mime.IsYamlKind(mimeType) { + return proto.Unmarshal(contents, message) + } + var node yaml.Node + if err := yaml.Unmarshal(contents, &node); err != nil { + return err + } + encoding.StyleForJSON(&node) + bytes, err := yaml.Marshal(&node) + if err != nil { + return err + } + if err := protojson.Unmarshal(bytes, message); err != nil { + return err + } + + return err +} diff --git a/cmd/registry/patch/patch_test.go b/cmd/registry/patch/patch_test.go index 5aa903f9f..de228c7e3 100644 --- a/cmd/registry/patch/patch_test.go +++ b/cmd/registry/patch/patch_test.go @@ -17,6 +17,7 @@ package patch import ( "context" "os" + "strings" "testing" "github.com/apigee/registry/pkg/application/apihub" @@ -36,6 +37,7 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/api/iterator" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/testing/protocmp" ) @@ -1009,64 +1011,82 @@ func TestMessageArtifactPatches(t *testing.T) { }, }, } - ctx := context.Background() - registryClient, _ := grpctest.SetupRegistry(ctx, t, "patch-message-artifact-test", []seeder.RegistryResource{ - &rpc.ApiSpec{ - Name: root + "/apis/a/versions/v/specs/s", - }, - }) - for _, test := range tests { - t.Run(test.artifactID, func(t *testing.T) { - b, err := os.ReadFile(test.yamlFile) - if err != nil { - t.Fatalf("%s", err) - } - err = applyArtifactPatchBytes(ctx, registryClient, b, root, "patch.yaml") - if err != nil { - t.Fatalf("%s", err) - } - var collection string - if test.parent != "" { - collection = root + "/" + test.parent + "/artifacts/" - } else { - collection = root + "/artifacts/" - } - artifactName, err := names.ParseArtifact(collection + test.artifactID) - if err != nil { - t.Fatalf("%s", err) - } - err = visitor.GetArtifact(ctx, registryClient, artifactName, true, - func(ctx context.Context, artifact *rpc.Artifact) error { - contents, err := getArtifactMessageContents(artifact) + storageTypes := []struct { + mimeBase string + ctx context.Context + }{ + {"application/octet-stream", context.Background()}, + {"application/yaml", SetStoreArchivesAsYaml(context.Background())}, + } + + for _, storage := range storageTypes { + t.Run(storage.mimeBase, func(t *testing.T) { + ctx := storage.ctx + + registryClient, _ := grpctest.SetupRegistry(ctx, t, "patch-message-artifact-test", []seeder.RegistryResource{ + &rpc.ApiSpec{ + Name: root + "/apis/a/versions/v/specs/s", + }, + }) + + for _, test := range tests { + t.Run(test.artifactID, func(t *testing.T) { + b, err := os.ReadFile(test.yamlFile) if err != nil { t.Fatalf("%s", err) } - opts := cmp.Options{protocmp.Transform()} - if !cmp.Equal(test.message, contents, opts) { - t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(test.message, contents, opts)) - } - model, err := NewArtifact(ctx, registryClient, artifact) + err = applyArtifactPatchBytes(ctx, registryClient, b, root, "patch.yaml") if err != nil { t.Fatalf("%s", err) } - if model.Header.Metadata.Parent != test.parent { - t.Errorf("Incorrect export parent. Wanted %s, got %s", test.parent, model.Header.Metadata.Parent) - } - if model.Header.Metadata.Name != test.artifactID { - t.Errorf("Incorrect export name. Wanted %s, got %s", test.artifactID, model.Header.Metadata.Name) + var collection string + if test.parent != "" { + collection = root + "/" + test.parent + "/artifacts/" + } else { + collection = root + "/artifacts/" } - out, err := encoding.EncodeYAML(model) + artifactName, err := names.ParseArtifact(collection + test.artifactID) if err != nil { - t.Errorf("encoding.EncodeYAML(%+v) returned an error: %s", model, err) + t.Fatalf("%s", err) } - if !cmp.Equal(b, out, opts) { - t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(b, out, opts)) + err = visitor.GetArtifact(ctx, registryClient, artifactName, true, + func(ctx context.Context, artifact *rpc.Artifact) error { + // sanity check that we're really testing the right mime types + if !strings.HasPrefix(artifact.GetMimeType(), storage.mimeBase) { + t.Fatalf("unexpected storage type, want: %s, got: %s", storage.mimeBase, artifact.GetMimeType()) + } + contents, err := getArtifactMessageContents(artifact) + if err != nil { + t.Fatalf("%s", err) + } + opts := cmp.Options{protocmp.Transform()} + if !cmp.Equal(test.message, contents, opts) { + t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(test.message, contents, opts)) + } + model, err := NewArtifact(ctx, registryClient, artifact) + if err != nil { + t.Fatalf("%s", err) + } + if model.Header.Metadata.Parent != test.parent { + t.Errorf("Incorrect export parent. Wanted %s, got %s", test.parent, model.Header.Metadata.Parent) + } + if model.Header.Metadata.Name != test.artifactID { + t.Errorf("Incorrect export name. Wanted %s, got %s", test.artifactID, model.Header.Metadata.Name) + } + out, err := encoding.EncodeYAML(model) + if err != nil { + t.Errorf("encoding.EncodeYAML(%+v) returned an error: %s", model, err) + } + if !cmp.Equal(b, out, opts) { + t.Errorf("GetDiff returned unexpected diff (-want +got):\n%s", cmp.Diff(b, out, opts)) + } + return nil + }) + if err != nil { + t.Fatalf("%s", err) } - return nil }) - if err != nil { - t.Fatalf("%s", err) } }) } @@ -1199,14 +1219,21 @@ func getArtifactMessageContents(artifact *rpc.Artifact) (proto.Message, error) { if err != nil { return nil, err } - return unmarshal(artifact.GetContents(), message) -} + err = UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), message) -func unmarshal(value []byte, message proto.Message) (proto.Message, error) { - if err := proto.Unmarshal(value, message); err != nil { - return nil, err + // restore id and kind + s := strings.Split(artifact.GetName(), "/") + id := s[len(s)-1] + kind := mime.KindForMimeType(artifact.GetMimeType()) + fields := message.ProtoReflect().Descriptor().Fields() + if fd := fields.ByTextName("id"); fd != nil { + message.ProtoReflect().Set(fd, protoreflect.ValueOfString(id)) + } + if fd := fields.ByTextName("kind"); fd != nil { + message.ProtoReflect().Set(fd, protoreflect.ValueOfString(kind)) } - return message, nil + + return message, err } func TestEmptyArtifactPatches(t *testing.T) { diff --git a/cmd/registry/scoring/expression.go b/cmd/registry/scoring/expression.go index 93d048386..fb110f77c 100644 --- a/cmd/registry/scoring/expression.go +++ b/cmd/registry/scoring/expression.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/scoring/extensions" "github.com/apigee/registry/pkg/application/apihub" "github.com/apigee/registry/pkg/application/controller" @@ -67,30 +68,30 @@ func getMap(contents []byte, mimeType string) (map[string]interface{}, error) { switch messageType { case "gnostic.metrics.Complexity": - return unmarshalAndMap(contents, &metrics.Complexity{}) + return unmarshalAndMap(contents, mimeType, &metrics.Complexity{}) case "gnostic.metrics.Vocabulary": - return unmarshalAndMap(contents, &metrics.Vocabulary{}) + return unmarshalAndMap(contents, mimeType, &metrics.Vocabulary{}) case "google.cloud.apigeeregistry.v1.style.ConformanceReport": - return unmarshalAndMap(contents, &style.ConformanceReport{}) + return unmarshalAndMap(contents, mimeType, &style.ConformanceReport{}) case "google.cloud.apigeeregistry.v1.style.Lint": - return unmarshalAndMap(contents, &style.Lint{}) + return unmarshalAndMap(contents, mimeType, &style.Lint{}) case "google.cloud.apigeeregistry.v1.apihub.ReferenceList": - return unmarshalAndMap(contents, &apihub.ReferenceList{}) + return unmarshalAndMap(contents, mimeType, &apihub.ReferenceList{}) case "google.cloud.apigeeregistry.v1.controller.Receipt": - return unmarshalAndMap(contents, &controller.Receipt{}) + return unmarshalAndMap(contents, mimeType, &controller.Receipt{}) case "google.cloud.apigeeregistry.v1.scoring.Score": - return unmarshalAndMap(contents, &scoring.Score{}) + return unmarshalAndMap(contents, mimeType, &scoring.Score{}) case "google.cloud.apigeeregistry.v1.scoring.ScoreCard": - return unmarshalAndMap(contents, &scoring.ScoreCard{}) + return unmarshalAndMap(contents, mimeType, &scoring.ScoreCard{}) // TODO: Add support for JSON artifacts default: return nil, fmt.Errorf("unsupported artifact type: %s", messageType) } } -func unmarshalAndMap(contents []byte, message proto.Message) (map[string]interface{}, error) { +func unmarshalAndMap(contents []byte, mimeType string, message proto.Message) (map[string]interface{}, error) { // Convert to proto - err := proto.Unmarshal(contents, message) + err := patch.UnmarshalContents(contents, mimeType, message) if err != nil { return nil, fmt.Errorf("failed unmarshling: %s", err) } diff --git a/cmd/registry/scoring/score.go b/cmd/registry/scoring/score.go index 60728744a..d0d535cc8 100644 --- a/cmd/registry/scoring/score.go +++ b/cmd/registry/scoring/score.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/pkg/application/scoring" "github.com/apigee/registry/pkg/log" @@ -50,7 +51,7 @@ func FetchScoreDefinitions( err = client.ListArtifacts(ctx, artifact, listFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error { definition := &scoring.ScoreDefinition{} - if err1 := proto.Unmarshal(artifact.GetContents(), definition); err1 != nil { + if err1 := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), definition); err1 != nil { // don't return err, to proccess the rest of the artifacts from the list. log.Debugf(ctx, "Skipping definition %q: %s", artifact.GetName(), err1) return nil @@ -79,7 +80,7 @@ func CalculateScore( // Extract definition definition := &scoring.ScoreDefinition{} - if err := proto.Unmarshal(defArtifact.GetContents(), definition); err != nil { + if err := patch.UnmarshalContents(defArtifact.GetContents(), defArtifact.GetMimeType(), definition); err != nil { return err } diff --git a/cmd/registry/scoring/score_test.go b/cmd/registry/scoring/score_test.go index 79bcea566..7045715b4 100644 --- a/cmd/registry/scoring/score_test.go +++ b/cmd/registry/scoring/score_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/pkg/application/scoring" "github.com/apigee/registry/pkg/application/style" @@ -27,7 +28,6 @@ import ( "github.com/apigee/registry/server/registry/test/seeder" metrics "github.com/google/gnostic/metrics" "github.com/google/go-cmp/cmp" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" ) @@ -434,7 +434,7 @@ func TestCalculateScore(t *testing.T) { } gotScore := &scoring.Score{} - err = proto.Unmarshal(scoreArtifact.GetContents(), gotScore) + err = patch.UnmarshalContents(scoreArtifact.GetContents(), scoreArtifact.GetMimeType(), gotScore) if err != nil { t.Errorf("failed unmarshalling score artifact from registry: %s", err) } diff --git a/cmd/registry/scoring/scorecard.go b/cmd/registry/scoring/scorecard.go index 74a4e63a1..6b4ae65bb 100644 --- a/cmd/registry/scoring/scorecard.go +++ b/cmd/registry/scoring/scorecard.go @@ -18,6 +18,7 @@ import ( "context" "fmt" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/pkg/application/scoring" "github.com/apigee/registry/pkg/log" @@ -48,7 +49,7 @@ func FetchScoreCardDefinitions( err = client.ListArtifacts(ctx, artifact, listFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error { definition := &scoring.ScoreCardDefinition{} - if err1 := proto.Unmarshal(artifact.GetContents(), definition); err1 != nil { + if err1 := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), definition); err1 != nil { // don't return err, to proccess the rest of the artifacts from the list. log.Debugf(ctx, "Skipping definition %q: %s", artifact.GetName(), err1) return nil @@ -75,7 +76,7 @@ func CalculateScoreCard( // Extract definition definition := &scoring.ScoreCardDefinition{} - if err := proto.Unmarshal(defArtifact.GetContents(), definition); err != nil { + if err := patch.UnmarshalContents(defArtifact.GetContents(), defArtifact.GetMimeType(), definition); err != nil { return err } @@ -164,7 +165,7 @@ func processScorePatterns( needsUpdate = needsUpdate || takeAction || artifact.GetUpdateTime().AsTime().Add(patterns.ResourceUpdateThreshold).After(scoreCardArtifact.GetUpdateTime().AsTime()) // Extract Score from the fetched artifact score := &scoring.Score{} - if err := proto.Unmarshal(artifact.GetContents(), score); err != nil { + if err := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), score); err != nil { return scoreCardResult{ scoreCard: nil, needsUpdate: false, diff --git a/cmd/registry/scoring/scorecard_test.go b/cmd/registry/scoring/scorecard_test.go index 221cc232b..98d11a637 100644 --- a/cmd/registry/scoring/scorecard_test.go +++ b/cmd/registry/scoring/scorecard_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/pkg/application/scoring" "github.com/apigee/registry/pkg/connection/grpctest" @@ -25,7 +26,6 @@ import ( "github.com/apigee/registry/server/registry/test/seeder" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" ) @@ -522,7 +522,7 @@ func TestCalculateScoreCard(t *testing.T) { } gotScoreCard := &scoring.ScoreCard{} - err = proto.Unmarshal(scoreCardArtifact.GetContents(), gotScoreCard) + err = patch.UnmarshalContents(scoreCardArtifact.GetContents(), scoreCardArtifact.GetMimeType(), gotScoreCard) if err != nil { t.Errorf("failed unmarshalling ScoreCard artifact from registry: %s", err) } diff --git a/cmd/registry/scoring/scoring-timestamp_test.go b/cmd/registry/scoring/scoring-timestamp_test.go index c6eefff01..e6ed54580 100644 --- a/cmd/registry/scoring/scoring-timestamp_test.go +++ b/cmd/registry/scoring/scoring-timestamp_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/apigee/registry/cmd/registry/patch" "github.com/apigee/registry/cmd/registry/patterns" "github.com/apigee/registry/pkg/application/scoring" "github.com/apigee/registry/pkg/application/style" @@ -17,7 +18,6 @@ import ( metrics "github.com/google/gnostic/metrics" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -307,7 +307,7 @@ func TestTimestampCalculateScore(t *testing.T) { } gotScore := &scoring.Score{} - err = proto.Unmarshal(scoreArtifact.GetContents(), gotScore) + err = patch.UnmarshalContents(scoreArtifact.GetContents(), scoreArtifact.GetMimeType(), gotScore) if err != nil { t.Errorf("failed unmarshalling score artifact from registry: %s", err) } diff --git a/pkg/mime/types.go b/pkg/mime/types.go index b8156c655..8a3e34881 100644 --- a/pkg/mime/types.go +++ b/pkg/mime/types.go @@ -86,6 +86,10 @@ func IsPrintableType(mimeType string) bool { strings.HasPrefix(mimeType, "application/json") } +func IsYamlKind(mimeType string) bool { + return strings.HasPrefix(mimeType, "application/yaml;type=") && KindForMimeType(mimeType) != "" +} + // MimeTypeForMessageType returns a MIME type that represents a Protocol Buffer message type. func MimeTypeForMessageType(protoType string) string { return fmt.Sprintf("application/octet-stream;type=%s", protoType)