Provide completion, hover and docs links for uninitialized Registry modules

go.mod

@@ -6,9 +6,9 @@ require (
 	github.com/agext/levenshtein v1.2.2 // indirect
 	github.com/apparentlymart/go-textseg v1.0.0
 	github.com/creachadair/jrpc2 v0.41.0
-	github.com/fsnotify/fsnotify v1.5.4
 	github.com/google/go-cmp v0.5.8
 	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-memdb v1.3.3
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-uuid v1.0.3
@@ -19,13 +19,13 @@ require (
 	github.com/hashicorp/terraform-exec v0.16.1
 	github.com/hashicorp/terraform-json v0.14.0
 	github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473
-	github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531
+	github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.4
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/otiai10/copy v1.7.0 // indirect
+	github.com/otiai10/copy v1.7.0
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546

go.sum

@@ -347,8 +347,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e
 github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473 h1:Vp3YMcnM+TvVMV5TplAhGeuzz3A0562AywL32y71y3Y=
 github.com/hashicorp/terraform-registry-address v0.0.0-20220422093245-eb7bcc2ff473/go.mod h1:bdLC+qQlJIBHKbCMA6GipcuaKjmjcvZlnVdpU583z3Y=
-github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531 h1:CVBByNVwgdRBKz6hdrL547Rw6RU4QF7sDnxvISdoBxM=
-github.com/hashicorp/terraform-schema v0.0.0-20220509053855-1e3acbcfd531/go.mod h1:rLQP6aOmOcA+C68h3Ea7utboW/UWwgn5m8i/pE5rm28=
+github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb h1:ZVpBMZlvIwHzFfSfNCspuH5WG8p5N8fn1Rjqayoxbmc=
+github.com/hashicorp/terraform-schema v0.0.0-20220617163605-9aece33a9bbb/go.mod h1:zphXkRtXhZ1vC60ytqhUBSnKDyYHBaV321zNgYuLKxs=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -473,8 +473,10 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
 github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=

internal/decoder/path_reader.go

@@ -4,18 +4,22 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl-lang/decoder"
 	"github.com/hashicorp/hcl-lang/lang"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
 	"github.com/hashicorp/terraform-ls/internal/state"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
 	tfmod "github.com/hashicorp/terraform-schema/module"
+	"github.com/hashicorp/terraform-schema/registry"
 )
 
 type ModuleReader interface {
 	ModuleByPath(modPath string) (*state.Module, error)
 	List() ([]*state.Module, error)
 	ModuleCalls(modPath string) (tfmod.ModuleCalls, error)
-	ModuleMeta(modPath string) (*tfmod.Meta, error)
+	LocalModuleMeta(modPath string) (*tfmod.Meta, error)
+	RegistryModuleMeta(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*registry.ModuleData, error)
 }
 
 type PathReader struct {

internal/langserver/handlers/did_open.go

@@ -159,6 +159,19 @@ func (svc *service) decodeModule(ctx context.Context, modHandle document.DirHand
 			}
 			ids = append(ids, id)
 
+			id, err = svc.stateStore.JobStore.EnqueueJob(job.Job{
+				Dir: modHandle,
+				Func: func(ctx context.Context) error {
+					return module.GetModuleDataFromRegistry(svc.srvCtx, svc.registryClient,
+						svc.modStore, svc.stateStore.RegistryModules, modHandle.Path())
+				},
+				Type: op.OpTypeGetModuleDataFromRegistry.String(),
+			})
+			if err != nil {
+				return
+			}
+			ids = append(ids, id)
+
 			return
 		},
 	})

internal/langserver/handlers/service.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/terraform-ls/internal/langserver/session"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
+	"github.com/hashicorp/terraform-ls/internal/registry"
 	"github.com/hashicorp/terraform-ls/internal/scheduler"
 	"github.com/hashicorp/terraform-ls/internal/schemas"
 	"github.com/hashicorp/terraform-ls/internal/settings"
@@ -46,19 +47,21 @@ type service struct {
 	closedDirWalker *module.Walker
 	openDirWalker   *module.Walker
 
-	fs            *filesystem.Filesystem
-	modStore      *state.ModuleStore
-	schemaStore   *state.ProviderSchemaStore
-	tfDiscoFunc   discovery.DiscoveryFunc
-	tfExecFactory exec.ExecutorFactory
-	tfExecOpts    *exec.ExecutorOpts
-	telemetry     telemetry.Sender
-	decoder       *decoder.Decoder
-	stateStore    *state.StateStore
-	server        session.Server
-	diagsNotifier *diagnostics.Notifier
-	notifier      *notifier.Notifier
-	indexer       *module.Indexer
+	fs               *filesystem.Filesystem
+	modStore         *state.ModuleStore
+	schemaStore      *state.ProviderSchemaStore
+	regMetadataStore *state.RegistryModuleStore
+	tfDiscoFunc      discovery.DiscoveryFunc
+	tfExecFactory    exec.ExecutorFactory
+	tfExecOpts       *exec.ExecutorOpts
+	telemetry        telemetry.Sender
+	decoder          *decoder.Decoder
+	stateStore       *state.StateStore
+	server           session.Server
+	diagsNotifier    *diagnostics.Notifier
+	notifier         *notifier.Notifier
+	indexer          *module.Indexer
+	registryClient   registry.Client
 
 	walkerCollector    *module.WalkerCollector
 	additionalHandlers map[string]rpch.Func
@@ -73,13 +76,14 @@ func NewSession(srvCtx context.Context) session.Session {
 
 	sessCtx, stopSession := context.WithCancel(srvCtx)
 	return &service{
-		logger:        discardLogs,
-		srvCtx:        srvCtx,
-		sessCtx:       sessCtx,
-		stopSession:   stopSession,
-		tfDiscoFunc:   d.LookPath,
-		tfExecFactory: exec.NewExecutor,
-		telemetry:     &telemetry.NoopSender{},
+		logger:         discardLogs,
+		srvCtx:         srvCtx,
+		sessCtx:        sessCtx,
+		stopSession:    stopSession,
+		tfDiscoFunc:    d.LookPath,
+		tfExecFactory:  exec.NewExecutor,
+		telemetry:      &telemetry.NoopSender{},
+		registryClient: registry.NewClient(),
 	}
 }
 

internal/langserver/handlers/session_mock_test.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"io/ioutil"
 	"log"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"sync"
 	"testing"
 
 	"github.com/creachadair/jrpc2/handler"
 	"github.com/hashicorp/terraform-ls/internal/langserver/session"
+	"github.com/hashicorp/terraform-ls/internal/registry"
 	"github.com/hashicorp/terraform-ls/internal/state"
 	"github.com/hashicorp/terraform-ls/internal/terraform/discovery"
 	"github.com/hashicorp/terraform-ls/internal/terraform/exec"
@@ -21,10 +24,12 @@ type MockSessionInput struct {
 	AdditionalHandlers map[string]handler.Func
 	StateStore         *state.StateStore
 	WalkerCollector    *module.WalkerCollector
+	RegistryServer     *httptest.Server
 }
 
 type mockSession struct {
-	mockInput *MockSessionInput
+	mockInput      *MockSessionInput
+	registryServer *httptest.Server
 
 	stopFunc     func()
 	stopCalled   bool
@@ -42,6 +47,7 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session {
 		stateStore = ms.mockInput.StateStore
 		walkerCollector = ms.mockInput.WalkerCollector
 		handlers = ms.mockInput.AdditionalHandlers
+		ms.registryServer = ms.mockInput.RegistryServer
 	}
 
 	var tfCalls *exec.TerraformMockCalls
@@ -53,6 +59,14 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session {
 		Path: "tf-mock",
 	}
 
+	regClient := registry.NewClient()
+	if ms.registryServer == nil {
+		ms.registryServer = defaultRegistryServer()
+	}
+	ms.registryServer.Start()
+
+	regClient.BaseURL = ms.registryServer.URL
+
 	svc := &service{
 		logger:             testLogger(),
 		srvCtx:             srvCtx,
@@ -63,11 +77,18 @@ func (ms *mockSession) new(srvCtx context.Context) session.Session {
 		additionalHandlers: handlers,
 		stateStore:         stateStore,
 		walkerCollector:    walkerCollector,
+		registryClient:     regClient,
 	}
 
 	return svc
 }
 
+func defaultRegistryServer() *httptest.Server {
+	return httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "unexpected Registry API request", 500)
+	}))
+}
+
 func testLogger() *log.Logger {
 	if testing.Verbose() {
 		return log.New(os.Stdout, "", log.LstdFlags|log.Lshortfile)
@@ -77,6 +98,8 @@ func testLogger() *log.Logger {
 }
 
 func (ms *mockSession) stop() {
+	ms.registryServer.Close()
+
 	ms.stopCalledMu.Lock()
 	defer ms.stopCalledMu.Unlock()
 

internal/registry/registry.go

@@ -0,0 +1,118 @@
+package registry
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"time"
+
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-version"
+	tfaddr "github.com/hashicorp/terraform-registry-address"
+)
+
+const (
+	defaultBaseURL = "https://registry.terraform.io"
+	defaultTimeout = 5 * time.Second
+)
+
+type Client struct {
+	BaseURL string
+	Timeout time.Duration
+}
+
+func NewClient() Client {
+	return Client{
+		BaseURL: defaultBaseURL,
+		Timeout: defaultTimeout,
+	}
+}
+
+func (c Client) GetModuleData(addr tfaddr.ModuleSourceRegistry, cons version.Constraints) (*ModuleResponse, error) {
+	var response ModuleResponse
+
+	v, err := c.GetMatchingModuleVersion(addr, cons)
+	if err != nil {
+		return nil, err
+	}
+
+	client := cleanhttp.DefaultClient()
+	client.Timeout = defaultTimeout
+
+	url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/%s", c.BaseURL,
+		addr.PackageAddr.Namespace,
+		addr.PackageAddr.Name,
+		addr.PackageAddr.TargetSystem,
+		v.String())
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		bodyBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		return nil, fmt.Errorf("unexpected response %s: %s", resp.Status, string(bodyBytes))
+	}
+
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	return &response, nil
+}
+
+func (c Client) GetMatchingModuleVersion(addr tfaddr.ModuleSourceRegistry, con version.Constraints) (*version.Version, error) {
+	url := fmt.Sprintf("%s/v1/modules/%s/%s/%s/versions", c.BaseURL,
+		addr.PackageAddr.Namespace,
+		addr.PackageAddr.Name,
+		addr.PackageAddr.TargetSystem)
+
+	client := cleanhttp.DefaultClient()
+	client.Timeout = defaultTimeout
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		bodyBytes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		return nil, fmt.Errorf("unexpected response %s: %s", resp.Status, string(bodyBytes))
+	}
+
+	var response ModuleVersionsResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	var foundVersions version.Collection
+	for _, module := range response.Modules {
+		for _, entry := range module.Versions {
+			ver, err := version.NewVersion(entry.Version)
+			if err == nil {
+				foundVersions = append(foundVersions, ver)
+			}
+		}
+	}
+
+	sort.Sort(foundVersions)
+
+	for _, fv := range foundVersions {
+		if con.Check(fv) {
+			return fv, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no suitable version found for %q %q", addr, con)
+}