Add basic language server support for explicit symbol imports (#3282)

This PR adds straightforward "go to definition" support for Motoko's pattern-style import syntax. 

For example, `import { v|als } "mo:base/Array"` and `v|als(x)` (where `|` represents the cursor) will correctly navigate to the corresponding `Array.vals` declaration. 

As with existing language server functionality, these changes do not yet account for variable shadowing. However, the updated `parse_module_header` function already parses import patterns such as (`import {alias = field} "..."`) based on the proposed generalized import syntax (#3076). 

Please let me know if additional explanation would be helpful for any of these changes. I also included four test cases to cover the newly expected behavior of `definition_handler` and `parse_module_header`. 

Tested in VSCode on WSL using the official Motoko extension.

Fixes #3078
Supersedes #3263

CC @kritzcreek, @crusso, @matthewhammer

src/languageServer/completion.ml

@@ -136,11 +136,16 @@ let completions index project_root file_path file_contents line column =
   match find_completion_prefix file_contents line column with
   | None ->
       (* If we don't have any prefix to work with, just suggest the
-         imported module aliases, as well as top-level definitions in
+         imported module aliases/fields, as well as top-level definitions in
          the current file *)
       let decls = List.map item_of_ide_decl toplevel_decls in
       decls
-      @ List.map (fun (alias, _) -> module_alias_completion_item alias) imported
+      @ List.map
+          (function
+            | Source_file.AliasImport (ident, _)
+            | Source_file.FieldImport (_, ident, _) ->
+                module_alias_completion_item ident)
+          imported
   | Some ("", prefix) ->
       (* Without an alias but with a prefix we filter the toplevel
          identifiers of the current module *)
@@ -149,7 +154,12 @@ let completions index project_root file_path file_contents line column =
       |> List.map item_of_ide_decl
   | Some (alias, prefix) -> (
       let module_path =
-        imported |> List.find_opt (fun (mn, _) -> String.equal mn alias)
+        imported
+        |> List.map (function
+               | Source_file.AliasImport (ident, path)
+               | Source_file.FieldImport (_, ident, path)
+               -> (ident, path))
+        |> List.find_opt (fun (ident, _) -> String.equal alias ident)
       in
       match module_path with
       | Some mp -> (

src/languageServer/definition.ml

@@ -40,6 +40,7 @@ let definition_handler index position file_contents project_root file_path =
     let* path =
       match ident with
       | Alias (_, path) -> Some path
+      | Field (_, path) -> Some path
       | Unresolved _ -> None
       | Resolved { path; _ } -> Some path
       | Ident _ -> Lib.FilePath.relative_to project_root file_path
@@ -52,10 +53,8 @@ let definition_handler index position file_contents project_root file_path =
       match
         let* decl_ident =
           match ident with
-          | Alias _ -> None
-          | Unresolved { ident; _ } -> Some ident (* Currently unreachable *)
-          | Resolved { ident; _ } -> Some ident
-          | Ident ident -> Some ident
+          | Alias _ | Unresolved _ -> None
+          | Field (ident, _) | Resolved { ident; _ } | Ident ident -> Some ident
         in
         (* Note: ignoring `path` output value from `find_named` *)
         let* _, region = find_named decl_ident module_ in

src/languageServer/hover.ml

@@ -40,6 +40,9 @@ let hover_handler (index : DI.t) (position : Lsp.position)
     match ident_target with
     | Source_file.Alias (_, path) ->
         Some Lsp.{ hover_result_contents = markup_content path }
+    | Source_file.Field (_, path) ->
+        (* TODO: add field-specific information *)
+        Some Lsp.{ hover_result_contents = markup_content path }
     | Source_file.Resolved resolved ->
         let* _, decls =
           lookup_module project_root resolved.Source_file.path index

src/languageServer/source_file.ml

@@ -75,17 +75,23 @@ let import_relative_to_project_root root module_path dependency =
         |> Lib.FilePath.normalise
         |> Option.some
 
+type header_part =
+  (* import <alias> "<path>" *)
+  | AliasImport of string * string
+  (* import { <alias> = <field> } "<path>" *)
+  | FieldImport of string * string * string
+
 (* Given the source of a module, figure out under what names what
    modules have been imported. Normalizes the imported modules
-   filepaths relative to the project root *)
+   filepaths relative to the project root. *)
 let parse_module_header project_root current_file_path file =
   let lexbuf = Lexing.from_string file in
   let tokenizer, _ = Lexer.tokenizer Lexer.mode lexbuf in
   let next () =
     let t, _, _ = tokenizer () in
     t
   in
-  let res = ref [] in
+  let header_parts = ref [] in
   let rec loop = function
     | Parser.IMPORT -> (
         match next () with
@@ -97,15 +103,49 @@ let parse_module_header project_root current_file_path file =
                     path
                 in
                 (match path with
-                | Some path -> res := (alias, path) :: !res
+                | Some path ->
+                    header_parts := AliasImport (alias, path) :: !header_parts
                 | None -> ());
                 loop (next ())
             | tkn -> loop tkn)
+        | Parser.LCURLY -> loop_fields [] (next ())
         | tkn -> loop tkn)
-    | Parser.EOF -> List.rev !res
+    | Parser.EOF -> ()
     | tkn -> loop (next ())
+  (* Account for basic object pattern syntax *)
+  and loop_fields fields = function
+    (* Slightly lenient for new users *)
+    | Parser.SEMICOLON | Parser.COMMA | Parser.EQ ->
+        loop_fields fields (next ())
+    | Parser.ID field -> (
+        match next () with
+        | Parser.EQ -> (
+            match next () with
+            | Parser.ID alias ->
+                loop_fields ((field, alias) :: fields) (next ())
+            | tkn -> loop_fields ((field, field) :: fields) tkn)
+        | tkn -> loop_fields ((field, field) :: fields) tkn)
+    | Parser.RCURLY -> (
+        match next () with
+        | Parser.TEXT path ->
+            let path =
+              import_relative_to_project_root project_root current_file_path
+                path
+            in
+            fields
+            |> List.rev
+            |> List.iter (fun (field, alias) ->
+                   match path with
+                   | Some path ->
+                       header_parts :=
+                         FieldImport (field, alias, path) :: !header_parts
+                   | None -> ());
+            loop (next ())
+        | tkn -> loop tkn)
+    | tkn -> loop tkn
   in
-  try loop (next ()) with _ -> List.rev !res
+  (try loop (next ()) with _ -> ());
+  List.rev !header_parts
 
 type unresolved_target = { qualifier : string; ident : string }
 
@@ -114,18 +154,40 @@ type resolved_target = { qualifier : string; ident : string; path : string }
 type identifier_target =
   | Ident of string
   | Alias of string * string
+  | Field of string * string
   | Unresolved of unresolved_target
   | Resolved of resolved_target
 
 let identifier_at_pos project_root file_path file_contents position =
-  let imported = parse_module_header project_root file_path file_contents in
+  let header_parts = parse_module_header project_root file_path file_contents in
   cursor_target_at_pos position file_contents
   |> Option.map (function
-       | CIdent s -> (
-           match List.find_opt (fun (alias, _) -> alias = s) imported with
-           | None -> Ident s
-           | Some (alias, path) -> Alias (alias, path))
+       | CIdent ident -> (
+           match
+             List.find_map
+               (function
+                 | AliasImport (alias, path) ->
+                     if alias = ident then Some (Alias (alias, path)) else None
+                 | FieldImport (field, alias, path) ->
+                     if alias = ident then Some (Field (field, path)) else None)
+               header_parts
+           with
+           | None -> Ident ident
+           | Some x -> x)
        | CQualified (qual, ident) -> (
-           match List.find_opt (fun (alias, _) -> alias = qual) imported with
+           match
+             List.find_map
+               (function
+                 | AliasImport (alias, path) ->
+                     if alias = qual then
+                       Some (Resolved { qualifier = qual; ident; path })
+                     else None
+                 | FieldImport (field, alias, path) ->
+                     if alias = qual then
+                       (* TODO: find the qualified record / object key definition when possible *)
+                       Some (Field (field, path))
+                     else None)
+               header_parts
+           with
            | None -> Unresolved { qualifier = qual; ident }
-           | Some (alias, path) -> Resolved { qualifier = qual; ident; path }))
+           | Some x -> x))

src/languageServer/source_file_tests.ml

@@ -32,12 +32,14 @@ let hovered_identifier_test_case file expected =
 
 let parse_module_header_test_case project_root current_file file expected =
   let actual = Source_file.parse_module_header project_root current_file file in
-  let display_result (alias, path) = Printf.sprintf "%s => \"%s\"" alias path in
-  let result =
-    Lib.List.equal
-      (fun (x, y) (x', y') -> String.equal x x' && String.equal y y')
-      actual expected
+  let display_result = function
+    | Source_file.AliasImport (alias, path) ->
+        Printf.sprintf "%s => \"%s\"" alias path
+    | Source_file.FieldImport (field, alias, path) ->
+        if alias = field then Printf.sprintf "{ %s } => \"%s\"" field path
+        else Printf.sprintf "{ %s = %s } => \"%s\"" field alias path
   in
+  let result = Lib.List.equal ( = ) actual expected in
   if not result then
     Printf.printf "\nExpected: %s\nActual:   %s"
       (Completion.string_of_list display_result expected)
@@ -58,16 +60,17 @@ let%test "it finds a qualified identifier" =
 let%test "it parses a simple module header" =
   parse_module_header_test_case "/project" "/project/src/Main.mo"
     "import P \"lib/prelude\""
-    [ ("P", "src/lib/prelude") ]
+    [ Source_file.AliasImport ("P", "src/lib/prelude") ]
 
 let%test "it parses a simple module header that contains a prim import" =
   parse_module_header_test_case "/project" "/project/src/Main.mo"
-    "import Prim \"mo:⛔\"" [ ("Prim", "mo:⛔") ]
+    "import Prim \"mo:⛔\""
+    [ Source_file.AliasImport ("Prim", "mo:⛔") ]
 
 let%test "it parses a simple module header with package paths" =
   parse_module_header_test_case "/project" "/project/src/Main.mo"
     "import P \"mo:stdlib/prelude\""
-    [ ("P", "mo:stdlib/prelude") ]
+    [ Source_file.AliasImport ("P", "mo:stdlib/prelude") ]
 
 let%test "it parses a simple module header" =
   parse_module_header_test_case "/project" "/project/Main.mo"
@@ -89,4 +92,33 @@ func singleton(x: Int): Stack =
   ListFuncs.doubleton<Int>(x, x);
 }
 |}
-    [ ("List", "lib/ListLib"); ("ListFuncs", "lib/ListFuncs") ]
+    [
+      Source_file.AliasImport ("List", "lib/ListLib");
+      Source_file.AliasImport ("ListFuncs", "lib/ListFuncs");
+    ]
+
+let%test "it parses a simple module header with explicit field imports" =
+  parse_module_header_test_case "/project" "/project/Main.mo"
+    {|
+import { List; nil; cons = next } "lib/ListLib";
+
+module {
+
+private import { doubleton } "lib/ListFuncs";
+
+func push(x: Int, s: Stack): Stack =
+  cons<Int>(x, s);
+
+func empty(): Stack =
+  nil<Int>();
+
+func singleton(x: Int): Stack =
+  doubleton<Int>(x, x);
+}
+|}
+    [
+      Source_file.FieldImport ("List", "List", "lib/ListLib");
+      Source_file.FieldImport ("nil", "nil", "lib/ListLib");
+      Source_file.FieldImport ("cons", "next", "lib/ListLib");
+      Source_file.FieldImport ("doubleton", "doubleton", "lib/ListFuncs");
+    ]

test/lsp-int-test-project/definitions.mo

@@ -1,9 +1,11 @@
 import MyDependency "mo:mydep/lib";
 import List "lib/list";
+import { nil; last = lastElement } "lib/list";
 
 module {
     public func myFunc() {
         let myClass = MyDependency.MyClass();
         let myNil = List.nil<Nat>();
+        assert lastElement(myNil) == null;
     }
 }

test/lsp-int/Main.hs

@@ -176,22 +176,43 @@ main = do
           definitionsTestCase
             project
             doc
-            (Position 6 25)
+            (Position 7 25)
             [("lib/list.mo", Range (Position 31 14) (Position 31 17))]
 
           log "Definition for a Class"
           definitionsTestCase
             project
             doc
-            (Position 5 31)
+            (Position 6 31)
             [("mydependency/lib.mo", Range (Position 5 17) (Position 5 24))]
 
+          log "Definition for a function via an explicit field import"
+          definitionsTestCase
+            project
+            doc
+            (Position 8 15)
+            [("lib/list.mo", Range (Position 56 14) (Position 56 18))]
+
           log "Definition for an imported module alias"
           definitionsTestCase
             project
             doc
-            (Position 1 10)
+            (Position 1 7)
             [("lib/list.mo", Range (Position 0 0) (Position 0 0))]
+
+          log "Definition for an imported field"
+          definitionsTestCase
+            project
+            doc
+            (Position 2 9)
+            [("lib/list.mo", Range (Position 31 14) (Position 31 17))]
+
+          log "Definition for an imported field alias"
+          definitionsTestCase
+            project
+            doc
+            (Position 2 21)
+            [("lib/list.mo", Range (Position 56 14) (Position 56 18))]
 
         log "Completion tests"
         log "Completing top level definitions"