walker: Use memdb for queued paths

internal/cmd/inspect_module_command.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	ictx "github.com/hashicorp/terraform-ls/internal/context"
+	"github.com/hashicorp/terraform-ls/internal/document"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 	"github.com/hashicorp/terraform-ls/internal/logging"
 	"github.com/hashicorp/terraform-ls/internal/state"
@@ -94,14 +95,19 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
 
 	ctx := context.Background()
 
-	walker := module.SyncWalker(fs, ss.DocumentStore, ss.Modules, ss.ProviderSchemas, ss.JobStore, exec.NewExecutor)
+	pa := state.NewPathAwaiter(ss.WalkerPaths, false)
+	walker := module.SyncWalker(fs, pa, ss.Modules, ss.ProviderSchemas, ss.JobStore, exec.NewExecutor)
 	walker.SetLogger(c.logger)
 
 	ctx, cancel := ictx.WithSignalCancel(context.Background(),
 		c.logger, os.Interrupt, syscall.SIGTERM)
 	defer cancel()
 
-	walker.EnqueuePath(rootPath)
+	dir := document.DirHandleFromPath(rootPath)
+	err = ss.WalkerPaths.EnqueueDir(dir)
+	if err != nil {
+		return err
+	}
 	err = walker.StartWalking(ctx)
 	if err != nil {
 		return err

internal/context/context.go

@@ -23,7 +23,6 @@ var (
 	ctxTfExecLogPath        = &contextKey{"terraform executor log path"}
 	ctxTfExecTimeout        = &contextKey{"terraform execution timeout"}
 	ctxWatcher              = &contextKey{"watcher"}
-	ctxModuleWalker         = &contextKey{"module walker"}
 	ctxRootDir              = &contextKey{"root directory"}
 	ctxCommandPrefix        = &contextKey{"command prefix"}
 	ctxDiagsNotifier        = &contextKey{"diagnostics notifier"}
@@ -119,18 +118,6 @@ func CommandPrefix(ctx context.Context) (string, bool) {
 	return *commandPrefix, true
 }
 
-func WithModuleWalker(ctx context.Context, w *module.Walker) context.Context {
-	return context.WithValue(ctx, ctxModuleWalker, w)
-}
-
-func ModuleWalker(ctx context.Context) (*module.Walker, error) {
-	w, ok := ctx.Value(ctxModuleWalker).(*module.Walker)
-	if !ok {
-		return nil, missingContextErr(ctxModuleWalker)
-	}
-	return w, nil
-}
-
 func WithDiagnosticsNotifier(ctx context.Context, diags *diagnostics.Notifier) context.Context {
 	return context.WithValue(ctx, ctxDiagsNotifier, diags)
 }

internal/langserver/handlers/did_change_workspace_folders.go

@@ -8,7 +8,6 @@ import (
 	lsctx "github.com/hashicorp/terraform-ls/internal/context"
 	"github.com/hashicorp/terraform-ls/internal/document"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
-	"github.com/hashicorp/terraform-ls/internal/uri"
 )
 
 func (svc *service) DidChangeWorkspaceFolders(ctx context.Context, params lsp.DidChangeWorkspaceFoldersParams) error {
@@ -17,13 +16,10 @@ func (svc *service) DidChangeWorkspaceFolders(ctx context.Context, params lsp.Di
 		return err
 	}
 
-	walker, err := lsctx.ModuleWalker(ctx)
-	if err != nil {
-		return err
-	}
-
 	for _, removed := range params.Event.Removed {
-		modPath, err := uri.PathFromURI(removed.URI)
+		modHandle := document.DirHandleFromURI(removed.URI)
+
+		err := svc.stateStore.WalkerPaths.DequeueDir(modHandle)
 		if err != nil {
 			jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{
 				Type: lsp.Warning,
@@ -32,34 +28,34 @@ func (svc *service) DidChangeWorkspaceFolders(ctx context.Context, params lsp.Di
 			})
 			continue
 		}
-		walker.RemovePathFromQueue(modPath)
 
-		err = watcher.RemoveModule(modPath)
+		err = watcher.RemoveModule(modHandle.Path())
 		if err != nil {
 			svc.logger.Printf("failed to remove module from watcher: %s", err)
 			continue
 		}
 
-		modHandle := document.DirHandleFromPath(modPath)
 		err = svc.stateStore.JobStore.DequeueJobsForDir(modHandle)
 		if err != nil {
 			svc.logger.Printf("failed to dequeue jobs for module: %s", err)
 			continue
 		}
 
-		callers, err := svc.modStore.CallersOfModule(modPath)
+		callers, err := svc.modStore.CallersOfModule(modHandle.Path())
 		if err != nil {
 			svc.logger.Printf("failed to remove module from watcher: %s", err)
 			continue
 		}
 		if len(callers) == 0 {
-			err = svc.modStore.Remove(modPath)
+			err = svc.modStore.Remove(modHandle.Path())
 			svc.logger.Printf("failed to remove module: %s", err)
 		}
 	}
 
 	for _, added := range params.Event.Added {
-		modPath, err := uri.PathFromURI(added.URI)
+		modHandle := document.DirHandleFromURI(added.URI)
+
+		err = svc.stateStore.WalkerPaths.EnqueueDir(modHandle)
 		if err != nil {
 			jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{
 				Type: lsp.Warning,
@@ -68,13 +64,12 @@ func (svc *service) DidChangeWorkspaceFolders(ctx context.Context, params lsp.Di
 			})
 			continue
 		}
-		err = watcher.AddModule(modPath)
+
+		err = watcher.AddModule(modHandle.Path())
 		if err != nil {
 			svc.logger.Printf("failed to add module to watcher: %s", err)
 			continue
 		}
-
-		walker.EnqueuePath(modPath)
 	}
 
 	return nil

internal/langserver/handlers/initialize.go

@@ -198,13 +198,16 @@ func (svc *service) Initialize(ctx context.Context, params lsp.InitializeParams)
 		excludeModulePaths = append(excludeModulePaths, modPath)
 	}
 
-	svc.walker.SetIgnoreDirectoryNames(cfgOpts.IgnoreDirectoryNames)
-	svc.walker.SetExcludeModulePaths(excludeModulePaths)
-	svc.walker.EnqueuePath(root.Path())
+	err = svc.stateStore.WalkerPaths.EnqueueDir(root)
+	if err != nil {
+		return serverCaps, err
+	}
 
 	if len(params.WorkspaceFolders) > 0 {
 		for _, folderPath := range params.WorkspaceFolders {
-			modPath, err := uri.PathFromURI(folderPath.URI)
+			modPath := document.DirHandleFromURI(folderPath.URI)
+
+			err := svc.stateStore.WalkerPaths.EnqueueDir(modPath)
 			if err != nil {
 				jrpc2.ServerFromContext(ctx).Notify(ctx, "window/showMessage", &lsp.ShowMessageParams{
 					Type: lsp.Warning,
@@ -213,15 +216,22 @@ func (svc *service) Initialize(ctx context.Context, params lsp.InitializeParams)
 				})
 				continue
 			}
-
-			svc.walker.EnqueuePath(modPath)
 		}
 	}
 
-	// Walker runs asynchronously so we're intentionally *not*
+	svc.closedDirWalker.SetIgnoreDirectoryNames(cfgOpts.IgnoreDirectoryNames)
+	svc.closedDirWalker.SetExcludeModulePaths(excludeModulePaths)
+	svc.openDirWalker.SetIgnoreDirectoryNames(cfgOpts.IgnoreDirectoryNames)
+	svc.openDirWalker.SetExcludeModulePaths(excludeModulePaths)
+
+	// Walkers run asynchronously so we're intentionally *not*
 	// passing the request context here
 	walkerCtx := context.Background()
-	err = svc.walker.StartWalking(walkerCtx)
+	err = svc.closedDirWalker.StartWalking(walkerCtx)
+	if err != nil {
+		return serverCaps, err
+	}
+	err = svc.openDirWalker.StartWalking(walkerCtx)
 	if err != nil {
 		return serverCaps, err
 	}

internal/langserver/handlers/service.go

@@ -42,11 +42,13 @@ type service struct {
 	closedDirIndexer *scheduler.Scheduler
 	openDirIndexer   *scheduler.Scheduler
 
+	closedDirWalker *module.Walker
+	openDirWalker   *module.Walker
+
 	fs            *filesystem.Filesystem
 	modStore      *state.ModuleStore
 	schemaStore   *state.ProviderSchemaStore
 	watcher       module.Watcher
-	walker        *module.Walker
 	newWatcher    module.WatcherFactory
 	newWalker     module.WalkerFactory
 	tfDiscoFunc   discovery.DiscoveryFunc
@@ -280,7 +282,6 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) {
 				return nil, err
 			}
 
-			ctx = lsctx.WithModuleWalker(ctx, svc.walker)
 			ctx = lsctx.WithWatcher(ctx, svc.watcher)
 
 			return handle(ctx, req, svc.DidChangeWorkspaceFolders)
@@ -300,7 +301,6 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) {
 			}
 
 			ctx = lsctx.WithCommandPrefix(ctx, &commandPrefix)
-			ctx = lsctx.WithModuleWalker(ctx, svc.walker)
 			ctx = lsctx.WithWatcher(ctx, svc.watcher)
 			ctx = lsctx.WithRootDirectory(ctx, &rootDir)
 			ctx = lsctx.WithDiagnosticsNotifier(ctx, svc.diagsNotifier)
@@ -463,8 +463,12 @@ func (svc *service) configureSessionDependencies(ctx context.Context, cfgOpts *s
 		return err
 	}
 
-	svc.walker = svc.newWalker(svc.fs, svc.stateStore.DocumentStore, svc.modStore, svc.schemaStore, svc.stateStore.JobStore, svc.tfExecFactory)
-	svc.walker.SetLogger(svc.logger)
+	closedPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, false)
+	svc.closedDirWalker = svc.newWalker(svc.fs, closedPa, svc.modStore, svc.schemaStore, svc.stateStore.JobStore, svc.tfExecFactory)
+	svc.closedDirWalker.SetLogger(svc.logger)
+	opendPa := state.NewPathAwaiter(svc.stateStore.WalkerPaths, true)
+	svc.openDirWalker = svc.newWalker(svc.fs, opendPa, svc.modStore, svc.schemaStore, svc.stateStore.JobStore, svc.tfExecFactory)
+	svc.openDirWalker.SetLogger(svc.logger)
 
 	ww, err := svc.newWatcher(svc.fs, svc.modStore, svc.stateStore.ProviderSchemas, svc.stateStore.JobStore, svc.tfExecFactory)
 	if err != nil {
@@ -500,10 +504,15 @@ func (svc *service) Finish(_ jrpc2.Assigner, status jrpc2.ServerStatus) {
 }
 
 func (svc *service) shutdown() {
-	if svc.walker != nil {
-		svc.logger.Printf("stopping walker for session ...")
-		svc.walker.Stop()
-		svc.logger.Printf("walker stopped")
+	if svc.closedDirWalker != nil {
+		svc.logger.Printf("stopping closedDirWalker for session ...")
+		svc.closedDirWalker.Stop()
+		svc.logger.Printf("closedDirWalker stopped")
+	}
+	if svc.openDirWalker != nil {
+		svc.logger.Printf("stopping openDirWalker for session ...")
+		svc.openDirWalker.Stop()
+		svc.logger.Printf("openDirWalker stopped")
 	}
 
 	if svc.watcher != nil {

internal/state/documents.go

@@ -52,6 +52,10 @@ func (s *DocumentStore) OpenDocument(dh document.Handle, langId string, version
 	if err != nil {
 		return err
 	}
+	err = updateWalkerDirOpenMark(txn, dh.Dir, true)
+	if err != nil {
+		return err
+	}
 
 	txn.Commit()
 	return nil
@@ -133,6 +137,11 @@ func (s *DocumentStore) CloseDocument(dh document.Handle) error {
 		return err
 	}
 
+	err = updateWalkerDirOpenMark(txn, dh.Dir, false)
+	if err != nil {
+		return err
+	}
+
 	txn.Commit()
 	return nil
 }

internal/state/state.go

@@ -20,6 +20,7 @@ const (
 	moduleIdsTableName      = "module_ids"
 	providerSchemaTableName = "provider_schema"
 	providerIdsTableName    = "provider_ids"
+	walkerPathsTableName    = "walker_paths"
 )
 
 var dbSchema = &memdb.DBSchema{
@@ -141,6 +142,20 @@ var dbSchema = &memdb.DBSchema{
 				},
 			},
 		},
+		walkerPathsTableName: {
+			Name: walkerPathsTableName,
+			Indexes: map[string]*memdb.IndexSchema{
+				"id": {
+					Name:    "id",
+					Unique:  true,
+					Indexer: &DirHandleFieldIndexer{Field: "Dir"},
+				},
+				"is_dir_open": {
+					Name:    "is_dir_open",
+					Indexer: &memdb.BoolFieldIndex{Field: "IsDirOpen"},
+				},
+			},
+		},
 	},
 }
 
@@ -149,6 +164,7 @@ type StateStore struct {
 	JobStore        *JobStore
 	Modules         *ModuleStore
 	ProviderSchemas *ProviderSchemaStore
+	WalkerPaths     *WalkerPathStore
 
 	db *memdb.MemDB
 }
@@ -213,6 +229,11 @@ func NewStateStore() (*StateStore, error) {
 			tableName: providerSchemaTableName,
 			logger:    defaultLogger,
 		},
+		WalkerPaths: &WalkerPathStore{
+			db:        db,
+			tableName: walkerPathsTableName,
+			logger:    defaultLogger,
+		},
 	}, nil
 }
 
@@ -221,6 +242,7 @@ func (s *StateStore) SetLogger(logger *log.Logger) {
 	s.JobStore.logger = logger
 	s.Modules.logger = logger
 	s.ProviderSchemas.logger = logger
+	s.WalkerPaths.logger = logger
 }
 
 var defaultLogger = log.New(ioutil.Discard, "", 0)