diff --git a/go.mod b/go.mod index 3e0affd9..385baa2e 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/agext/levenshtein v1.2.2 // indirect github.com/apparentlymart/go-textseg v1.0.0 github.com/creachadair/jrpc2 v0.41.0 - github.com/fsnotify/fsnotify v1.5.4 github.com/google/go-cmp v0.5.8 github.com/google/uuid v1.2.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-memdb v1.3.3 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-uuid v1.0.3 @@ -19,13 +19,13 @@ require ( github.com/hashicorp/terraform-exec v0.16.1 github.com/hashicorp/terraform-json v0.14.0 github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473 - github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531 + github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb github.com/kylelemons/godebug v1.1.0 // indirect github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5 github.com/mitchellh/cli v1.1.4 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/otiai10/copy v1.7.0 // indirect + github.com/otiai10/copy v1.7.0 github.com/pmezard/go-difflib v1.0.0 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 diff --git a/go.sum b/go.sum index 47aedea1..6d1447bd 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473 h1:Vp3YMcnM+TvVMV5TplAhGeuzz3A0562AywL32y71y3Y= github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473/go.mod h1:bdLC+qQlJIBHKbCMA6GipcuaKjmjcvZlnVdpU583z3Y= -github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531 h1:CVBByNVwgdRBKz6hdrL547Rw6RU4QF7sDnxvISdoBxM= -github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531/go.mod h1:rLQP6aOmOcA+C68h3Ea7utboW/UWwgn5m8i/pE5rm28= +github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb h1:ZVpBMZlvIwHzFfSfNCspuH5WG8p5N8fn1Rjqayoxbmc= +github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb/go.mod h1:zphXkRtXhZ1vC60ytqhUBSnKDyYHBaV321zNgYuLKxs= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -473,8 +473,10 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= diff --git a/internal/decoder/path_reader.go b/internal/decoder/path_reader.go index c55e7e77..049d21c8 100644 --- a/internal/decoder/path_reader.go +++ b/internal/decoder/path_reader.go @@ -4,18 +4,22 @@ import ( "context" "fmt" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/decoder" "github.com/hashicorp/hcl-lang/lang" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" "github.com/hashicorp/terraform-ls/internal/state" + tfaddr "github.com/hashicorp/terraform-registry-address" tfmod "github.com/hashicorp/terraform-schema/module" + "github.com/hashicorp/terraform-schema/registry" ) type ModuleReader interface { ModuleByPath(modPath string) (*state.Module, error) List() ([]*state.Module, error) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) - ModuleMeta(modPath string) (*tfmod.Meta, error) + LocalModuleMeta(modPath string) (*tfmod.Meta, error) + RegistryModuleMeta(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*registry.ModuleData, error) } type PathReader struct { diff --git a/internal/langserver/handlers/did_open.go b/internal/langserver/handlers/did_open.go index 0d41f52c..c53ad0c4 100644 --- a/internal/langserver/handlers/did_open.go +++ b/internal/langserver/handlers/did_open.go @@ -159,6 +159,19 @@ func (svc *service) decodeModule(ctx context.Context, modHandle document.DirHand } ids = append(ids, id) + id, err = svc.stateStore.JobStore.EnqueueJob(job.Job{ + Dir: modHandle, + Func: func(ctx context.Context) error { + return module.GetModuleDataFromRegistry(svc.srvCtx, svc.registryClient, + svc.modStore, svc.stateStore.RegistryModules, modHandle.Path()) + }, + Type: op.OpTypeGetModuleDataFromRegistry.String(), + }) + if err != nil { + return + } + ids = append(ids, id) + return }, }) diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index 2c7fbfd6..89b1345b 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/langserver/session" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" lsp "github.com/hashicorp/terraform-ls/internal/protocol" + "github.com/hashicorp/terraform-ls/internal/registry" "github.com/hashicorp/terraform-ls/internal/scheduler" "github.com/hashicorp/terraform-ls/internal/schemas" "github.com/hashicorp/terraform-ls/internal/settings" @@ -46,19 +47,21 @@ type service struct { closedDirWalker *module.Walker openDirWalker *module.Walker - fs *filesystem.Filesystem - modStore *state.ModuleStore - schemaStore *state.ProviderSchemaStore - tfDiscoFunc discovery.DiscoveryFunc - tfExecFactory exec.ExecutorFactory - tfExecOpts *exec.ExecutorOpts - telemetry telemetry.Sender - decoder *decoder.Decoder - stateStore *state.StateStore - server session.Server - diagsNotifier *diagnostics.Notifier - notifier *notifier.Notifier - indexer *module.Indexer + fs *filesystem.Filesystem + modStore *state.ModuleStore + schemaStore *state.ProviderSchemaStore + regMetadataStore *state.RegistryModuleStore + tfDiscoFunc discovery.DiscoveryFunc + tfExecFactory exec.ExecutorFactory + tfExecOpts *exec.ExecutorOpts + telemetry telemetry.Sender + decoder *decoder.Decoder + stateStore *state.StateStore + server session.Server + diagsNotifier *diagnostics.Notifier + notifier *notifier.Notifier + indexer *module.Indexer + registryClient registry.Client walkerCollector *module.WalkerCollector additionalHandlers map[string]rpch.Func @@ -73,13 +76,14 @@ func NewSession(srvCtx context.Context) session.Session { sessCtx, stopSession := context.WithCancel(srvCtx) return &service{ - logger: discardLogs, - srvCtx: srvCtx, - sessCtx: sessCtx, - stopSession: stopSession, - tfDiscoFunc: d.LookPath, - tfExecFactory: exec.NewExecutor, - telemetry: &telemetry.NoopSender{}, + logger: discardLogs, + srvCtx: srvCtx, + sessCtx: sessCtx, + stopSession: stopSession, + tfDiscoFunc: d.LookPath, + tfExecFactory: exec.NewExecutor, + telemetry: &telemetry.NoopSender{}, + registryClient: registry.NewClient(), } } diff --git a/internal/langserver/handlers/session_mock_test.go b/internal/langserver/handlers/session_mock_test.go index 0a7dd2cd..36ab0695 100644 --- a/internal/langserver/handlers/session_mock_test.go +++ b/internal/langserver/handlers/session_mock_test.go @@ -4,12 +4,15 @@ import ( "context" "io/ioutil" "log" + "net/http" + "net/http/httptest" "os" "sync" "testing" "github.com/creachadair/jrpc2/handler" "github.com/hashicorp/terraform-ls/internal/langserver/session" + "github.com/hashicorp/terraform-ls/internal/registry" "github.com/hashicorp/terraform-ls/internal/state" "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" @@ -21,10 +24,12 @@ type MockSessionInput struct { AdditionalHandlers map[string]handler.Func StateStore *state.StateStore WalkerCollector *module.WalkerCollector + RegistryServer *httptest.Server } type mockSession struct { - mockInput *MockSessionInput + mockInput *MockSessionInput + registryServer *httptest.Server stopFunc func() stopCalled bool @@ -42,6 +47,7 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { stateStore = ms.mockInput.StateStore walkerCollector = ms.mockInput.WalkerCollector handlers = ms.mockInput.AdditionalHandlers + ms.registryServer = ms.mockInput.RegistryServer } var tfCalls *exec.TerraformMockCalls @@ -53,6 +59,14 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { Path: "tf-mock", } + regClient := registry.NewClient() + if ms.registryServer == nil { + ms.registryServer = defaultRegistryServer() + } + ms.registryServer.Start() + + regClient.BaseURL = ms.registryServer.URL + svc := &service{ logger: testLogger(), srvCtx: srvCtx, @@ -63,11 +77,18 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session { additionalHandlers: handlers, stateStore: stateStore, walkerCollector: walkerCollector, + registryClient: regClient, } return svc } +func defaultRegistryServer() *httptest.Server { + return httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unexpected Registry API request", 500) + })) +} + func testLogger() *log.Logger { if testing.Verbose() { return log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile) @@ -77,6 +98,8 @@ func testLogger() *log.Logger { } func (ms *mockSession) stop() { + ms.registryServer.Close() + ms.stopCalledMu.Lock() defer ms.stopCalledMu.Unlock() diff --git a/internal/registry/registry.go b/internal/registry/registry.go new file mode 100644 index 00000000..d98e66fc --- /dev/null +++ b/internal/registry/registry.go @@ -0,0 +1,118 @@ +package registry + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "sort" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/hashicorp/go-version" + tfaddr "github.com/hashicorp/terraform-registry-address" +) + +const ( + defaultBaseURL = "https://registry.terraform.io" + defaultTimeout = 5 * time.Second +) + +type Client struct { + BaseURL string + Timeout time.Duration +} + +func NewClient() Client { + return Client{ + BaseURL: defaultBaseURL, + Timeout: defaultTimeout, + } +} + +func (c Client) GetModuleData(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*ModuleResponse, error) { + var response ModuleResponse + + v, err := c.GetMatchingModuleVersion(addr, cons) + if err != nil { + return nil, err + } + + client := cleanhttp.DefaultClient() + client.Timeout = defaultTimeout + + url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", c.BaseURL, + addr.PackageAddr.Namespace, + addr.PackageAddr.Name, + addr.PackageAddr.TargetSystem, + v.String()) + resp, err := client.Get(url) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return nil, fmt.Errorf("unexpected response %s: %s", resp.Status, string(bodyBytes)) + } + + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, err + } + + return &response, nil +} + +func (c Client) GetMatchingModuleVersion(addr tfaddr.ModuleSourceRegistry, con version.Constraints) (*version.Version, error) { + url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", c.BaseURL, + addr.PackageAddr.Namespace, + addr.PackageAddr.Name, + addr.PackageAddr.TargetSystem) + + client := cleanhttp.DefaultClient() + client.Timeout = defaultTimeout + + resp, err := client.Get(url) + if err != nil { + return nil, err + } + if resp.StatusCode != 200 { + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return nil, fmt.Errorf("unexpected response %s: %s", resp.Status, string(bodyBytes)) + } + + var response ModuleVersionsResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, err + } + + var foundVersions version.Collection + for _, module := range response.Modules { + for _, entry := range module.Versions { + ver, err := version.NewVersion(entry.Version) + if err == nil { + foundVersions = append(foundVersions, ver) + } + } + } + + sort.Sort(foundVersions) + + for _, fv := range foundVersions { + if con.Check(fv) { + return fv, nil + } + } + + return nil, fmt.Errorf("no suitable version found for %q %q", addr, con) +} diff --git a/internal/registry/registry_mock_responses_test.go b/internal/registry/registry_mock_responses_test.go new file mode 100644 index 00000000..c50376e3 --- /dev/null +++ b/internal/registry/registry_mock_responses_test.go @@ -0,0 +1,269 @@ +package registry + +// moduleVersionsMockResponse represents response from https://registry.terraform.io/v1/modules/puppetlabs/deployment/ec/versions +var moduleVersionsMockResponse = `{ + "modules": [ + { + "source": "puppetlabs/deployment/ec", + "versions": [ + { + "version": "0.0.5", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.6", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.8", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.2", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.1", + "root": { + "providers": [], + "dependencies": [] + }, + "submodules": [ + { + "path": "modules/ec-deployment", + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + } + ] + }, + { + "version": "0.0.4", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.3", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.7", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + } + ] + } + ] +}` + +// moduleDataMockResponse represents response from https://registry.terraform.io/v1/modules/puppetlabs/deployment/ec/0.0.8 +var moduleDataMockResponse = `{ + "id": "puppetlabs/deployment/ec/0.0.8", + "owner": "mattkirby", + "namespace": "puppetlabs", + "name": "deployment", + "version": "0.0.8", + "provider": "ec", + "provider_logo_url": "/images/providers/generic.svg?2", + "description": "", + "source": "https://github.com/puppetlabs/terraform-ec-deployment", + "tag": "v0.0.8", + "published_at": "2021-08-05T00:26:33.501756Z", + "downloads": 3059237, + "verified": false, + "root": { + "path": "", + "name": "deployment", + "readme": "# EC project Terraform module\n\nTerraform module which creates a Elastic Cloud project.\n\n## Usage\n\nDetails coming soon\n", + "empty": false, + "inputs": [ + { + "name": "autoscale", + "type": "string", + "description": "Enable autoscaling of elasticsearch", + "default": "\"true\"", + "required": false + }, + { + "name": "ec_stack_version", + "type": "string", + "description": "Version of Elastic Cloud stack to deploy", + "default": "\"\"", + "required": false + }, + { + "name": "name", + "type": "string", + "description": "Name of resources", + "default": "\"ecproject\"", + "required": false + }, + { + "name": "traffic_filter_sourceip", + "type": "string", + "description": "traffic filter source IP", + "default": "\"\"", + "required": false + }, + { + "name": "ec_region", + "type": "string", + "description": "cloud provider region", + "default": "\"gcp-us-west1\"", + "required": false + }, + { + "name": "deployment_templateid", + "type": "string", + "description": "ID of Elastic Cloud deployment type", + "default": "\"gcp-io-optimized\"", + "required": false + } + ], + "outputs": [ + { + "name": "elasticsearch_password", + "description": "elasticsearch password" + }, + { + "name": "deployment_id", + "description": "Elastic Cloud deployment ID" + }, + { + "name": "elasticsearch_version", + "description": "Stack version deployed" + }, + { + "name": "elasticsearch_cloud_id", + "description": "Elastic Cloud project deployment ID" + }, + { + "name": "elasticsearch_https_endpoint", + "description": "elasticsearch https endpoint" + }, + { + "name": "elasticsearch_username", + "description": "elasticsearch username" + } + ], + "dependencies": [], + "provider_dependencies": [ + { + "name": "ec", + "namespace": "elastic", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "resources": [ + { + "name": "ecproject", + "type": "ec_deployment" + }, + { + "name": "gcp_vpc_nat", + "type": "ec_deployment_traffic_filter" + }, + { + "name": "ec_tf_association", + "type": "ec_deployment_traffic_filter_association" + } + ] + }, + "submodules": [], + "examples": [], + "providers": [ + "ec" + ], + "versions": [ + "0.0.1", + "0.0.2", + "0.0.3", + "0.0.4", + "0.0.5", + "0.0.6", + "0.0.7", + "0.0.8" + ] +}` diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go new file mode 100644 index 00000000..ccd4a790 --- /dev/null +++ b/internal/registry/registry_test.go @@ -0,0 +1,149 @@ +package registry + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" + tfaddr "github.com/hashicorp/terraform-registry-address" +) + +func TestGetModuleData(t *testing.T) { + addr, err := tfaddr.ParseRawModuleSourceRegistry("puppetlabs/deployment/ec") + if err != nil { + t.Fatal(err) + } + + cons := version.MustConstraints(version.NewConstraint("0.0.8")) + + client := NewClient() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/versions" { + w.Write([]byte(moduleVersionsMockResponse)) + return + } + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/0.0.8" { + w.Write([]byte(moduleDataMockResponse)) + return + } + http.Error(w, fmt.Sprintf("unexpected request: %q", r.RequestURI), 400) + })) + client.BaseURL = srv.URL + t.Cleanup(srv.Close) + + data, err := client.GetModuleData(addr, cons) + if err != nil { + t.Fatal(err) + } + expectedData := &ModuleResponse{ + Version: "0.0.8", + Root: ModuleRoot{ + Inputs: []Input{ + { + Name: "autoscale", + Type: "string", + Description: "Enable autoscaling of elasticsearch", + Default: "\"true\"", + Required: false, + }, + { + Name: "ec_stack_version", + Type: "string", + Description: "Version of Elastic Cloud stack to deploy", + Default: "\"\"", + Required: false, + }, + { + Name: "name", + Type: "string", + Description: "Name of resources", + Default: "\"ecproject\"", + Required: false, + }, + { + Name: "traffic_filter_sourceip", + Type: "string", + Description: "traffic filter source IP", + Default: "\"\"", + Required: false, + }, + { + Name: "ec_region", + Type: "string", + Description: "cloud provider region", + Default: "\"gcp-us-west1\"", + Required: false, + }, + { + Name: "deployment_templateid", + Type: "string", + Description: "ID of Elastic Cloud deployment type", + Default: "\"gcp-io-optimized\"", + Required: false, + }, + }, + Outputs: []Output{ + { + Name: "elasticsearch_password", + Description: "elasticsearch password", + }, + { + Name: "deployment_id", + Description: "Elastic Cloud deployment ID", + }, + { + Name: "elasticsearch_version", + Description: "Stack version deployed", + }, + { + Name: "elasticsearch_cloud_id", + Description: "Elastic Cloud project deployment ID", + }, + { + Name: "elasticsearch_https_endpoint", + Description: "elasticsearch https endpoint", + }, + { + Name: "elasticsearch_username", + Description: "elasticsearch username", + }, + }, + }, + } + if diff := cmp.Diff(expectedData, data); diff != "" { + t.Fatalf("mismatched data: %s", diff) + } +} + +func TestGetMatchingModuleVersion(t *testing.T) { + addr, err := tfaddr.ParseRawModuleSourceRegistry("puppetlabs/deployment/ec") + if err != nil { + t.Fatal(err) + } + cons := version.MustConstraints(version.NewConstraint("0.0.8")) + client := NewClient() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/versions" { + w.Write([]byte(moduleVersionsMockResponse)) + return + } + http.Error(w, fmt.Sprintf("unexpected request: %q", r.RequestURI), 400) + })) + client.BaseURL = srv.URL + t.Cleanup(srv.Close) + + v, err := client.GetMatchingModuleVersion(addr, cons) + if err != nil { + t.Fatal(err) + } + + expectedVersion := version.Must(version.NewVersion("0.0.8")) + if !expectedVersion.Equal(v) { + t.Fatalf("expected version: %s, given: %s", expectedVersion, v) + } +} diff --git a/internal/registry/types.go b/internal/registry/types.go new file mode 100644 index 00000000..8d4a50fd --- /dev/null +++ b/internal/registry/types.go @@ -0,0 +1,36 @@ +package registry + +type ModuleResponse struct { + Version string `json:"version"` + Root ModuleRoot `json:"root"` +} + +type ModuleRoot struct { + Inputs []Input `json:"inputs"` + Outputs []Output `json:"outputs"` +} + +type Input struct { + Name string `json:"name"` + Type string `json:"type"` + Description string `json:"description"` + Default string `json:"default"` + Required bool `json:"required"` +} + +type Output struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type ModuleVersionsResponse struct { + Modules []ModuleVersionsEntry `json:"modules"` +} + +type ModuleVersionsEntry struct { + Versions []ModuleVersion `json:"versions"` +} + +type ModuleVersion struct { + Version string `json:"version"` +} diff --git a/internal/state/errors.go b/internal/state/errors.go index dc551d01..9b2b950b 100644 --- a/internal/state/errors.go +++ b/internal/state/errors.go @@ -25,13 +25,13 @@ func (e *NoSchemaError) Error() string { } type ModuleNotFoundError struct { - Path string + Source string } func (e *ModuleNotFoundError) Error() string { msg := "module not found" - if e.Path != "" { - return fmt.Sprintf("%s: %s", e.Path, msg) + if e.Source != "" { + return fmt.Sprintf("%s: %s", e.Source, msg) } return msg diff --git a/internal/state/module.go b/internal/state/module.go index 4a77861f..3a96254c 100644 --- a/internal/state/module.go +++ b/internal/state/module.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/hcl/v2" tfaddr "github.com/hashicorp/terraform-registry-address" tfmod "github.com/hashicorp/terraform-schema/module" + "github.com/hashicorp/terraform-schema/registry" "github.com/hashicorp/terraform-ls/internal/terraform/ast" "github.com/hashicorp/terraform-ls/internal/terraform/datadir" @@ -352,7 +353,7 @@ func (s *ModuleStore) ModuleCalls(modPath string) (tfmod.ModuleCalls, error) { return modCalls, err } -func (s *ModuleStore) ModuleMeta(modPath string) (*tfmod.Meta, error) { +func (s *ModuleStore) LocalModuleMeta(modPath string) (*tfmod.Meta, error) { mod, err := s.ModuleByPath(modPath) if err != nil { return nil, err @@ -369,6 +370,31 @@ func (s *ModuleStore) ModuleMeta(modPath string) (*tfmod.Meta, error) { }, nil } +func (s *ModuleStore) RegistryModuleMeta(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*registry.ModuleData, error) { + txn := s.db.Txn(false) + + it, err := txn.Get(registryModuleTableName, "source_addr", addr) + if err != nil { + return nil, err + } + + for item := it.Next(); item != nil; item = it.Next() { + mod := item.(*RegistryModuleData) + + if cons.Check(mod.Version) { + return ®istry.ModuleData{ + Version: mod.Version, + Inputs: mod.Inputs, + Outputs: mod.Outputs, + }, nil + } + } + + return nil, &ModuleNotFoundError{ + Source: addr.String(), + } +} + func moduleByPath(txn *memdb.Txn, path string) (*Module, error) { obj, err := txn.First(moduleTableName, "id", path) if err != nil { @@ -376,7 +402,7 @@ func moduleByPath(txn *memdb.Txn, path string) (*Module, error) { } if obj == nil { return nil, &ModuleNotFoundError{ - Path: path, + Source: path, } } return obj.(*Module), nil diff --git a/internal/state/module_test.go b/internal/state/module_test.go index 33ded925..45ec5072 100644 --- a/internal/state/module_test.go +++ b/internal/state/module_test.go @@ -20,6 +20,8 @@ import ( "github.com/zclconf/go-cty/cty" ) +var _ ModuleCallReader = &ModuleStore{} + func TestModuleStore_Add_duplicate(t *testing.T) { s, err := NewStateStore() if err != nil { diff --git a/internal/state/registry_modules.go b/internal/state/registry_modules.go new file mode 100644 index 00000000..10eda578 --- /dev/null +++ b/internal/state/registry_modules.go @@ -0,0 +1,66 @@ +package state + +import ( + "fmt" + + "github.com/hashicorp/go-version" + tfaddr "github.com/hashicorp/terraform-registry-address" + "github.com/hashicorp/terraform-schema/registry" +) + +type RegistryModuleData struct { + Source tfaddr.ModuleSourceRegistry + Version *version.Version + Inputs []registry.Input + Outputs []registry.Output +} + +func (s *RegistryModuleStore) Exists(sourceAddr tfaddr.ModuleSourceRegistry, constraint version.Constraints) (bool, error) { + txn := s.db.Txn(false) + + iter, err := txn.Get(s.tableName, "source_addr", sourceAddr) + if err != nil { + return false, err + } + + for obj := iter.Next(); obj != nil; obj = iter.Next() { + p := obj.(*RegistryModuleData) + if constraint.Check(p.Version) { + return true, nil + } + } + + return false, nil +} + +func (s *RegistryModuleStore) Cache(sourceAddr tfaddr.ModuleSourceRegistry, modVer *version.Version, + inputs []registry.Input, outputs []registry.Output) error { + + txn := s.db.Txn(true) + defer txn.Abort() + + obj, err := txn.First(s.tableName, "id", sourceAddr, modVer) + if err != nil { + return err + } + if obj != nil { + return &AlreadyExistsError{ + Idx: fmt.Sprintf("%s@%v", sourceAddr, modVer), + } + } + + modData := &RegistryModuleData{ + Source: sourceAddr, + Version: modVer, + Inputs: inputs, + Outputs: outputs, + } + + err = txn.Insert(s.tableName, modData) + if err != nil { + return err + } + + txn.Commit() + return nil +} diff --git a/internal/state/registry_modules_test.go b/internal/state/registry_modules_test.go new file mode 100644 index 00000000..52ae6a6e --- /dev/null +++ b/internal/state/registry_modules_test.go @@ -0,0 +1,117 @@ +package state + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl-lang/lang" + tfaddr "github.com/hashicorp/terraform-registry-address" + "github.com/hashicorp/terraform-schema/registry" + "github.com/zclconf/go-cty-debug/ctydebug" + "github.com/zclconf/go-cty/cty" +) + +func TestStateStore_cache_metadata(t *testing.T) { + s, err := NewStateStore() + if err != nil { + t.Fatal(err) + } + + source, err := tfaddr.ParseRawModuleSourceRegistry("terraform-aws-modules/eks/aws") + if err != nil { + t.Fatal(err) + } + + v := version.Must(version.NewVersion("3.10.0")) + c := version.MustConstraints(version.NewConstraint(">= 3.0")) + inputs := []registry.Input{ + { + Name: "foo", + Type: cty.String, + Description: lang.Markdown("baz"), + Default: cty.StringVal("woot"), + Required: false, + }, + } + outputs := []registry.Output{ + { + Name: "wakka", + Description: lang.Markdown("fozzy"), + }, + } + + // should be false + exists, err := s.RegistryModules.Exists(source, c) + if err != nil { + t.Fatal(err) + } + if exists == true { + t.Fatal("should not exist") + } + + // store a dummy data + err = s.RegistryModules.Cache(source, v, inputs, outputs) + if err != nil { + t.Fatal(err) + } + + // should be true + exists, err = s.RegistryModules.Exists(source, c) + if err != nil { + t.Fatal(err) + } + if exists != true { + t.Fatal("should exist") + } +} + +func TestModule_DeclaredModuleMeta(t *testing.T) { + ss, err := NewStateStore() + if err != nil { + t.Fatal(err) + } + + source, err := tfaddr.ParseRawModuleSourceRegistry("terraform-aws-modules/eks/aws") + if err != nil { + t.Fatal(err) + } + + v := version.Must(version.NewVersion("3.10.0")) + inputs := []registry.Input{ + { + Name: "foo", + Type: cty.String, + Description: lang.Markdown("baz"), + Default: cty.StringVal("woot"), + Required: false, + }, + } + outputs := []registry.Output{ + { + Name: "wakka", + Description: lang.Markdown("fozzy"), + }, + } + + // store some dummy data + err = ss.RegistryModules.Cache(source, v, inputs, outputs) + if err != nil { + t.Fatal(err) + } + + cons := version.MustConstraints(version.NewConstraint(">= 3.0")) + meta, err := ss.Modules.RegistryModuleMeta(source, cons) + if err != nil { + t.Fatal(err) + } + + expectedMeta := ®istry.ModuleData{ + Version: v, + Inputs: inputs, + Outputs: outputs, + } + if diff := cmp.Diff(expectedMeta, meta, ctydebug.CmpOptions); diff != "" { + t.Fatalf("mismatch chached metadata: %s", diff) + } +} diff --git a/internal/state/state.go b/internal/state/state.go index a3196022..0d69ce2f 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/go-version" tfaddr "github.com/hashicorp/terraform-registry-address" tfmod "github.com/hashicorp/terraform-schema/module" + "github.com/hashicorp/terraform-schema/registry" tfschema "github.com/hashicorp/terraform-schema/schema" ) @@ -22,6 +23,7 @@ const ( providerSchemaTableName = "provider_schema" providerIdsTableName = "provider_ids" walkerPathsTableName = "walker_paths" + registryModuleTableName = "registry_module" ) var dbSchema = &memdb.DBSchema{ @@ -127,6 +129,25 @@ var dbSchema = &memdb.DBSchema{ }, }, }, + registryModuleTableName: { + Name: registryModuleTableName, + Indexes: map[string]*memdb.IndexSchema{ + "id": { + Name: "id", + Unique: true, + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &StringerFieldIndexer{Field: "Source"}, + &VersionFieldIndexer{Field: "Version"}, + }, + }, + }, + "source_addr": { + Name: "source_addr", + Indexer: &StringerFieldIndexer{Field: "Source"}, + }, + }, + }, providerIdsTableName: { Name: providerIdsTableName, Indexes: map[string]*memdb.IndexSchema{ @@ -189,6 +210,7 @@ type StateStore struct { Modules *ModuleStore ProviderSchemas *ProviderSchemaStore WalkerPaths *WalkerPathStore + RegistryModules *RegistryModuleStore db *memdb.MemDB } @@ -215,7 +237,8 @@ type ModuleReader interface { type ModuleCallReader interface { ModuleCalls(modPath string) (tfmod.ModuleCalls, error) - ModuleMeta(modPath string) (*tfmod.Meta, error) + LocalModuleMeta(modPath string) (*tfmod.Meta, error) + RegistryModuleMeta(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*registry.ModuleData, error) } type ProviderSchemaStore struct { @@ -223,6 +246,11 @@ type ProviderSchemaStore struct { tableName string logger *log.Logger } +type RegistryModuleStore struct { + db *memdb.MemDB + tableName string + logger *log.Logger +} type SchemaReader interface { ProviderSchema(modPath string, addr tfaddr.Provider, vc version.Constraints) (*tfschema.ProviderSchema, error) @@ -260,6 +288,11 @@ func NewStateStore() (*StateStore, error) { tableName: providerSchemaTableName, logger: defaultLogger, }, + RegistryModules: &RegistryModuleStore{ + db: db, + tableName: registryModuleTableName, + logger: defaultLogger, + }, WalkerPaths: &WalkerPathStore{ db: db, tableName: walkerPathsTableName, @@ -276,6 +309,7 @@ func (s *StateStore) SetLogger(logger *log.Logger) { s.Modules.logger = logger s.ProviderSchemas.logger = logger s.WalkerPaths.logger = logger + s.RegistryModules.logger = logger } var defaultLogger = log.New(ioutil.Discard, "", 0) diff --git a/internal/terraform/module/module_ops.go b/internal/terraform/module/module_ops.go index 4f367adf..42754fd2 100644 --- a/internal/terraform/module/module_ops.go +++ b/internal/terraform/module/module_ops.go @@ -2,12 +2,15 @@ package module import ( "context" + "errors" "fmt" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl-lang/lang" "github.com/hashicorp/terraform-ls/internal/decoder" ilsp "github.com/hashicorp/terraform-ls/internal/lsp" + "github.com/hashicorp/terraform-ls/internal/registry" "github.com/hashicorp/terraform-ls/internal/state" "github.com/hashicorp/terraform-ls/internal/terraform/datadir" op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation" @@ -15,7 +18,10 @@ import ( tfaddr "github.com/hashicorp/terraform-registry-address" "github.com/hashicorp/terraform-schema/earlydecoder" "github.com/hashicorp/terraform-schema/module" + tfregistry "github.com/hashicorp/terraform-schema/registry" tfschema "github.com/hashicorp/terraform-schema/schema" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/json" ) type DeferFunc func(opError error) @@ -348,3 +354,101 @@ func DecodeVarsReferences(ctx context.Context, modStore *state.ModuleStore, sche return rErr } + +func GetModuleDataFromRegistry(ctx context.Context, regClient registry.Client, modStore *state.ModuleStore, modRegStore *state.RegistryModuleStore, modPath string) error { + // loop over module calls + calls, err := modStore.ModuleCalls(modPath) + if err != nil { + return err + } + + var errs *multierror.Error + + for _, declaredModule := range calls.Declared { + sourceAddr, ok := declaredModule.SourceAddr.(tfaddr.ModuleSourceRegistry) + if !ok { + // skip any modules which do not come from the Registry + continue + } + + // check if that address was already cached + // if there was an error finding in cache, so cache again + exists, err := modRegStore.Exists(sourceAddr, declaredModule.Version) + if err != nil { + errs = multierror.Append(errs, err) + continue + } + if exists { + // entry in cache, no need to look up + continue + } + + // get module data from Terraform Registry + metaData, err := regClient.GetModuleData(sourceAddr, declaredModule.Version) + if err != nil { + errs = multierror.Append(errs, err) + continue + } + + inputs := make([]tfregistry.Input, len(metaData.Root.Inputs)) + for i, input := range metaData.Root.Inputs { + inputs[i] = tfregistry.Input{ + Name: input.Name, + Description: lang.Markdown(input.Description), + Required: input.Required, + } + + inputType := cty.DynamicPseudoType + if input.Type != "" { + // Registry API unfortunately doesn't marshal types using + // cty marshalers, making it lossy, so we just try to decode + // on best-effort basis. + rawType := []byte(fmt.Sprintf("%q", input.Type)) + typ, err := json.UnmarshalType(rawType) + if err == nil { + inputType = typ + } + } + inputs[i].Type = inputType + + if input.Default != "" { + // Registry API unfortunately doesn't marshal values using + // cty marshalers, making it lossy, so we just try to decode + // on best-effort basis. + val, err := json.Unmarshal([]byte(input.Default), inputType) + if err == nil { + inputs[i].Default = val + } + } + } + outputs := make([]tfregistry.Output, len(metaData.Root.Outputs)) + for i, output := range metaData.Root.Outputs { + outputs[i] = tfregistry.Output{ + Name: output.Name, + Description: lang.Markdown(output.Description), + } + } + + modVersion, err := version.NewVersion(metaData.Version) + if err != nil { + errs = multierror.Append(errs, err) + continue + } + + // if not, cache it + err = modRegStore.Cache(sourceAddr, modVersion, inputs, outputs) + if err != nil { + // A different job which ran in parallel for a different module block + // with the same source may have already cached the same module. + existsError := &state.AlreadyExistsError{} + if errors.As(err, &existsError) { + continue + } + + errs = multierror.Append(errs, err) + continue + } + } + + return errs.ErrorOrNil() +} diff --git a/internal/terraform/module/module_ops_mock_responses.go b/internal/terraform/module/module_ops_mock_responses.go new file mode 100644 index 00000000..b4978850 --- /dev/null +++ b/internal/terraform/module/module_ops_mock_responses.go @@ -0,0 +1,269 @@ +package module + +// moduleVersionsMockResponse represents response from https://registry.terraform.io/v1/modules/puppetlabs/deployment/ec/versions +var moduleVersionsMockResponse = `{ + "modules": [ + { + "source": "puppetlabs/deployment/ec", + "versions": [ + { + "version": "0.0.5", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.6", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.8", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.2", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.1", + "root": { + "providers": [], + "dependencies": [] + }, + "submodules": [ + { + "path": "modules/ec-deployment", + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + } + ] + }, + { + "version": "0.0.4", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.3", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + }, + { + "version": "0.0.7", + "root": { + "providers": [ + { + "name": "ec", + "namespace": "", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "dependencies": [] + }, + "submodules": [] + } + ] + } + ] +}` + +// moduleDataMockResponse represents response from https://registry.terraform.io/v1/modules/puppetlabs/deployment/ec/0.0.8 +var moduleDataMockResponse = `{ + "id": "puppetlabs/deployment/ec/0.0.8", + "owner": "mattkirby", + "namespace": "puppetlabs", + "name": "deployment", + "version": "0.0.8", + "provider": "ec", + "provider_logo_url": "/images/providers/generic.svg?2", + "description": "", + "source": "https://github.com/puppetlabs/terraform-ec-deployment", + "tag": "v0.0.8", + "published_at": "2021-08-05T00:26:33.501756Z", + "downloads": 3059237, + "verified": false, + "root": { + "path": "", + "name": "deployment", + "readme": "# EC project Terraform module\n\nTerraform module which creates a Elastic Cloud project.\n\n## Usage\n\nDetails coming soon\n", + "empty": false, + "inputs": [ + { + "name": "autoscale", + "type": "string", + "description": "Enable autoscaling of elasticsearch", + "default": "\"true\"", + "required": false + }, + { + "name": "ec_stack_version", + "type": "string", + "description": "Version of Elastic Cloud stack to deploy", + "default": "\"\"", + "required": false + }, + { + "name": "name", + "type": "string", + "description": "Name of resources", + "default": "\"ecproject\"", + "required": false + }, + { + "name": "traffic_filter_sourceip", + "type": "string", + "description": "traffic filter source IP", + "default": "\"\"", + "required": false + }, + { + "name": "ec_region", + "type": "string", + "description": "cloud provider region", + "default": "\"gcp-us-west1\"", + "required": false + }, + { + "name": "deployment_templateid", + "type": "string", + "description": "ID of Elastic Cloud deployment type", + "default": "\"gcp-io-optimized\"", + "required": false + } + ], + "outputs": [ + { + "name": "elasticsearch_password", + "description": "elasticsearch password" + }, + { + "name": "deployment_id", + "description": "Elastic Cloud deployment ID" + }, + { + "name": "elasticsearch_version", + "description": "Stack version deployed" + }, + { + "name": "elasticsearch_cloud_id", + "description": "Elastic Cloud project deployment ID" + }, + { + "name": "elasticsearch_https_endpoint", + "description": "elasticsearch https endpoint" + }, + { + "name": "elasticsearch_username", + "description": "elasticsearch username" + } + ], + "dependencies": [], + "provider_dependencies": [ + { + "name": "ec", + "namespace": "elastic", + "source": "elastic/ec", + "version": "0.2.1" + } + ], + "resources": [ + { + "name": "ecproject", + "type": "ec_deployment" + }, + { + "name": "gcp_vpc_nat", + "type": "ec_deployment_traffic_filter" + }, + { + "name": "ec_tf_association", + "type": "ec_deployment_traffic_filter_association" + } + ] + }, + "submodules": [], + "examples": [], + "providers": [ + "ec" + ], + "versions": [ + "0.0.1", + "0.0.2", + "0.0.3", + "0.0.4", + "0.0.5", + "0.0.6", + "0.0.7", + "0.0.8" + ] +}` diff --git a/internal/terraform/module/module_ops_test.go b/internal/terraform/module/module_ops_test.go new file mode 100644 index 00000000..3079d66a --- /dev/null +++ b/internal/terraform/module/module_ops_test.go @@ -0,0 +1,328 @@ +package module + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl-lang/lang" + "github.com/hashicorp/terraform-ls/internal/filesystem" + "github.com/hashicorp/terraform-ls/internal/registry" + "github.com/hashicorp/terraform-ls/internal/state" + tfaddr "github.com/hashicorp/terraform-registry-address" + tfregistry "github.com/hashicorp/terraform-schema/registry" + "github.com/zclconf/go-cty-debug/ctydebug" + "github.com/zclconf/go-cty/cty" +) + +func TestGetModuleDataFromRegistry_singleModule(t *testing.T) { + ctx := context.Background() + ss, err := state.NewStateStore() + if err != nil { + t.Fatal(err) + } + + testData, err := filepath.Abs("testdata") + if err != nil { + t.Fatal(err) + } + modPath := filepath.Join(testData, "uninitialized-external-module") + + err = ss.Modules.Add(modPath) + if err != nil { + t.Fatal(err) + } + + fs := filesystem.NewFilesystem(ss.DocumentStore) + err = ParseModuleConfiguration(fs, ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + err = LoadModuleMetadata(ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + regClient := registry.NewClient() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/versions" { + w.Write([]byte(moduleVersionsMockResponse)) + return + } + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/0.0.8" { + w.Write([]byte(moduleDataMockResponse)) + return + } + http.Error(w, fmt.Sprintf("unexpected request: %q", r.RequestURI), 400) + })) + regClient.BaseURL = srv.URL + t.Cleanup(srv.Close) + + err = GetModuleDataFromRegistry(ctx, regClient, ss.Modules, ss.RegistryModules, modPath) + if err != nil { + t.Fatal(err) + } + + addr, err := tfaddr.ParseRawModuleSourceRegistry("puppetlabs/deployment/ec") + if err != nil { + t.Fatal(err) + } + cons := version.MustConstraints(version.NewConstraint("0.0.8")) + + exists, err := ss.RegistryModules.Exists(addr, cons) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected cached metadata to exist for %q %q", addr, cons) + } + + meta, err := ss.Modules.RegistryModuleMeta(addr, cons) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectedModuleData, meta, ctydebug.CmpOptions); diff != "" { + t.Fatalf("metadata mismatch: %s", diff) + } +} + +func TestGetModuleDataFromRegistry_moduleNotFound(t *testing.T) { + ctx := context.Background() + ss, err := state.NewStateStore() + if err != nil { + t.Fatal(err) + } + + testData, err := filepath.Abs("testdata") + if err != nil { + t.Fatal(err) + } + modPath := filepath.Join(testData, "uninitialized-multiple-external-modules") + + err = ss.Modules.Add(modPath) + if err != nil { + t.Fatal(err) + } + + fs := filesystem.NewFilesystem(ss.DocumentStore) + err = ParseModuleConfiguration(fs, ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + err = LoadModuleMetadata(ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + regClient := registry.NewClient() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/versions" { + w.Write([]byte(moduleVersionsMockResponse)) + return + } + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/0.0.8" { + w.Write([]byte(moduleDataMockResponse)) + return + } + if r.RequestURI == "/v1/modules/terraform-aws-modules/eks/aws/versions" { + http.Error(w, `{"errors":["Not Found"]}`, 404) + return + } + http.Error(w, fmt.Sprintf("unexpected request: %q", r.RequestURI), 400) + })) + regClient.BaseURL = srv.URL + t.Cleanup(srv.Close) + + err = GetModuleDataFromRegistry(ctx, regClient, ss.Modules, ss.RegistryModules, modPath) + if err == nil { + t.Fatal("expected module data obtaining to return error") + } + + // Verify that 2nd module is still cached even if + // obtaining data for the other one errored out + + addr, err := tfaddr.ParseRawModuleSourceRegistry("puppetlabs/deployment/ec") + if err != nil { + t.Fatal(err) + } + cons := version.MustConstraints(version.NewConstraint("0.0.8")) + + exists, err := ss.RegistryModules.Exists(addr, cons) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected cached metadata to exist for %q %q", addr, cons) + } + + meta, err := ss.Modules.RegistryModuleMeta(addr, cons) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectedModuleData, meta, ctydebug.CmpOptions); diff != "" { + t.Fatalf("metadata mismatch: %s", diff) + } +} + +func TestGetModuleDataFromRegistry_apiTimeout(t *testing.T) { + ctx := context.Background() + ss, err := state.NewStateStore() + if err != nil { + t.Fatal(err) + } + + testData, err := filepath.Abs("testdata") + if err != nil { + t.Fatal(err) + } + modPath := filepath.Join(testData, "uninitialized-multiple-external-modules") + + err = ss.Modules.Add(modPath) + if err != nil { + t.Fatal(err) + } + + fs := filesystem.NewFilesystem(ss.DocumentStore) + err = ParseModuleConfiguration(fs, ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + err = LoadModuleMetadata(ss.Modules, modPath) + if err != nil { + t.Fatal(err) + } + + regClient := registry.NewClient() + regClient.Timeout = 500 * time.Millisecond + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/versions" { + w.Write([]byte(moduleVersionsMockResponse)) + return + } + if r.RequestURI == "/v1/modules/puppetlabs/deployment/ec/0.0.8" { + w.Write([]byte(moduleDataMockResponse)) + return + } + if r.RequestURI == "/v1/modules/terraform-aws-modules/eks/aws/versions" { + // trigger timeout + time.Sleep(1 * time.Second) + return + } + http.Error(w, fmt.Sprintf("unexpected request: %q", r.RequestURI), 400) + })) + regClient.BaseURL = srv.URL + t.Cleanup(srv.Close) + + err = GetModuleDataFromRegistry(ctx, regClient, ss.Modules, ss.RegistryModules, modPath) + if err == nil { + t.Fatal("expected module data obtaining to return error") + } + + // Verify that 2nd module is still cached even if + // obtaining data for the other one timed out + + addr, err := tfaddr.ParseRawModuleSourceRegistry("puppetlabs/deployment/ec") + if err != nil { + t.Fatal(err) + } + cons := version.MustConstraints(version.NewConstraint("0.0.8")) + + exists, err := ss.RegistryModules.Exists(addr, cons) + if err != nil { + t.Fatal(err) + } + if !exists { + t.Fatalf("expected cached metadata to exist for %q %q", addr, cons) + } + + meta, err := ss.Modules.RegistryModuleMeta(addr, cons) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectedModuleData, meta, ctydebug.CmpOptions); diff != "" { + t.Fatalf("metadata mismatch: %s", diff) + } +} + +var expectedModuleData = &tfregistry.ModuleData{ + Version: version.Must(version.NewVersion("0.0.8")), + Inputs: []tfregistry.Input{ + { + Name: "autoscale", + Type: cty.String, + Default: cty.StringVal("true"), + Description: lang.Markdown("Enable autoscaling of elasticsearch"), + Required: false, + }, + { + Name: "ec_stack_version", + Type: cty.String, + Default: cty.StringVal(""), + Description: lang.Markdown("Version of Elastic Cloud stack to deploy"), + Required: false, + }, + { + Name: "name", + Type: cty.String, + Default: cty.StringVal("ecproject"), + Description: lang.Markdown("Name of resources"), + Required: false, + }, + { + Name: "traffic_filter_sourceip", + Type: cty.String, + Default: cty.StringVal(""), + Description: lang.Markdown("traffic filter source IP"), + Required: false, + }, + { + Name: "ec_region", + Type: cty.String, + Default: cty.StringVal("gcp-us-west1"), + Description: lang.Markdown("cloud provider region"), + Required: false, + }, + { + Name: "deployment_templateid", + Type: cty.String, + Default: cty.StringVal("gcp-io-optimized"), + Description: lang.Markdown("ID of Elastic Cloud deployment type"), + Required: false, + }, + }, + Outputs: []tfregistry.Output{ + { + Name: "elasticsearch_password", + Description: lang.Markdown("elasticsearch password"), + }, + { + Name: "deployment_id", + Description: lang.Markdown("Elastic Cloud deployment ID"), + }, + { + Name: "elasticsearch_version", + Description: lang.Markdown("Stack version deployed"), + }, + { + Name: "elasticsearch_cloud_id", + Description: lang.Markdown("Elastic Cloud project deployment ID"), + }, + { + Name: "elasticsearch_https_endpoint", + Description: lang.Markdown("elasticsearch https endpoint"), + }, + { + Name: "elasticsearch_username", + Description: lang.Markdown("elasticsearch username"), + }, + }, +} diff --git a/internal/terraform/module/operation/op_type_string.go b/internal/terraform/module/operation/op_type_string.go index 1b79bac2..3f2eb1c1 100644 --- a/internal/terraform/module/operation/op_type_string.go +++ b/internal/terraform/module/operation/op_type_string.go @@ -18,11 +18,12 @@ func _() { _ = x[OpTypeDecodeReferenceTargets-7] _ = x[OpTypeDecodeReferenceOrigins-8] _ = x[OpTypeDecodeVarsReferences-9] + _ = x[OpTypeGetModuleDataFromRegistry-10] } -const _OpType_name = "OpTypeUnknownOpTypeGetTerraformVersionOpTypeObtainSchemaOpTypeParseModuleConfigurationOpTypeParseVariablesOpTypeParseModuleManifestOpTypeLoadModuleMetadataOpTypeDecodeReferenceTargetsOpTypeDecodeReferenceOriginsOpTypeDecodeVarsReferences" +const _OpType_name = "OpTypeUnknownOpTypeGetTerraformVersionOpTypeObtainSchemaOpTypeParseModuleConfigurationOpTypeParseVariablesOpTypeParseModuleManifestOpTypeLoadModuleMetadataOpTypeDecodeReferenceTargetsOpTypeDecodeReferenceOriginsOpTypeDecodeVarsReferencesOpTypeGetModuleDataFromRegistry" -var _OpType_index = [...]uint8{0, 13, 38, 56, 86, 106, 131, 155, 183, 211, 237} +var _OpType_index = [...]uint16{0, 13, 38, 56, 86, 106, 131, 155, 183, 211, 237, 268} func (i OpType) String() string { if i >= OpType(len(_OpType_index)-1) { diff --git a/internal/terraform/module/operation/operation.go b/internal/terraform/module/operation/operation.go index 761f74fe..5f0919f3 100644 --- a/internal/terraform/module/operation/operation.go +++ b/internal/terraform/module/operation/operation.go @@ -24,4 +24,5 @@ const ( OpTypeDecodeReferenceTargets OpTypeDecodeReferenceOrigins OpTypeDecodeVarsReferences + OpTypeGetModuleDataFromRegistry ) diff --git a/internal/terraform/module/testdata/uninitialized-external-module/main.tf b/internal/terraform/module/testdata/uninitialized-external-module/main.tf new file mode 100644 index 00000000..9a3bb20a --- /dev/null +++ b/internal/terraform/module/testdata/uninitialized-external-module/main.tf @@ -0,0 +1,5 @@ +module "ec" { + source = "puppetlabs/deployment/ec" + version = "0.0.8" + +} diff --git a/internal/terraform/module/testdata/uninitialized-multiple-external-modules/main.tf b/internal/terraform/module/testdata/uninitialized-multiple-external-modules/main.tf new file mode 100644 index 00000000..01a10892 --- /dev/null +++ b/internal/terraform/module/testdata/uninitialized-multiple-external-modules/main.tf @@ -0,0 +1,11 @@ +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "18.23.0" + +} + +module "ec" { + source = "puppetlabs/deployment/ec" + version = "0.0.8" + +}