feat(api): Accept structured imports [LNG-288] (#989)

* Refactor sources * Fix FuncCompiler compilation * Normalize imports * Add relative imports * Remove Prelude * Remove import * Add a log * Add more logs * Use snapshot of fs2 * Remove prints * Add comments * Savepoint * Rewrite imports resolution * Fix relative import * Add comment * Added comments * Fix comment * Add comments * Refactor * Refactor * Add tests * Fix tests * Update tests * Add comment * Lower number of tests * Comment, rename * Add comment * Add emptiness check
fluencelabs · Dec 13, 2023 · f7bfa83 · f7bfa83
1 parent 00252fe
commit f7bfa83
Show file tree

Hide file tree

Showing 20 changed files with 619 additions and 396 deletions.
diff --git a/api/api-npm/index.d.ts b/api/api-npm/index.d.ts
@@ -20,11 +20,47 @@ export declare class CompilationResult {
   warnings: string[];
   generatedSources: GeneratedSource[];
 }
+/**
+ * Imports configuration for the compiler.
+ * Structure:
+ * {
+ *  "<compiled-path-prefix-1>": {
+ *   "<import-path-prefix-1>": ["<import-path-1>", "<import-path-2>"],
+ *   "<import-path-prefix-2>": "<import-path-3>",
+ *   ...
+ *  }
+ *  ...
+ * }
+ * Import `import` written in file with path `path`
+ * is resolved as follows:
+ * 1. Try to resolve `import` as relative import from `path`
+ * 2. If relative resolution failed:
+ *  a. Find **the longest** <compiled-path-prefix>
+ *     that is a prefix of `path` in the imports configuration
+ *  b. In obtained map, find **the longest** <import-path-prefix>
+ *     that is a prefix of `import`
+ *  c. Replace prefix in `import` with <import-path>
+ *  d. Try to resolve import with obtained path
+ *     (try a few paths if array was provided)
+ *
+ * WARNING: <compiled-path-prefix> in 2.a is compared with
+ *          absolute normalized path of `path`, so <compiled-path-prefix>
+ *          should be absolute normalized path as well
+ * NOTE: <import-path-prefix> could be empty string,
+ *       in which case it will match any import
+ * NOTE: passing just an array of strings is a shorthand for
+ * {
+ *   "/": {
+ *     "": <array>
+ *   }
+ * }
+ */
+type Imports = Record<string, Record<string, string[] | string>> | string[];
 
 /** Common arguments for all compile functions */
 type CommonArgs = {
-  /** Paths to directories, which you want to import .aqua files from. Example: ["./path/to/dir"] */
-  imports?: string[] | undefined;
+  /** Imports */
+  imports?: Imports | undefined;
   /** Constants to be passed to the compiler. Example: ["CONSTANT1=1", "CONSTANT2=2"] */
   constants?: string[] | undefined;
   /** Set log level for the compiler. Must be one of: Must be one of: all, trace, debug, info, warn, error, off. Default: info */

diff --git a/api/api-npm/index.js b/api/api-npm/index.js
@@ -24,6 +24,33 @@ function getConfig({
   );
 }
 
+function normalizeImports(imports) {
+  if (imports === undefined || imports === null) {
+    return {}; // No imports
+  }
+
+  if (Array.isArray(imports)) {
+    return {
+      "/": {
+        "": imports,
+      },
+    };
+  }
+
+  // Transform each inner string into an array
+  return Object.fromEntries(
+    Object.entries(imports).map(([pathPrefix, info]) => [
+      pathPrefix,
+      Object.fromEntries(
+        Object.entries(info).map(([importPrefix, locations]) => [
+          importPrefix,
+          Array.isArray(locations) ? locations : [locations],
+        ]),
+      ),
+    ]),
+  );
+}
+
 async function compile(...args) {
   try {
     const res = await Aqua.compile(...args);
@@ -42,11 +69,19 @@ async function compile(...args) {
 }
 
 export function compileFromString({ code, imports = [], ...commonArgs }) {
-  return compile(new Input(code), imports, getConfig(commonArgs));
+  return compile(
+    new Input(code),
+    normalizeImports(imports),
+    getConfig(commonArgs),
+  );
 }
 
 export function compileFromPath({ filePath, imports = [], ...commonArgs }) {
-  return compile(new Path(filePath), imports, getConfig(commonArgs));
+  return compile(
+    new Path(filePath),
+    normalizeImports(imports),
+    getConfig(commonArgs),
+  );
 }
 
 export function compileAquaCallFromString({
@@ -58,7 +93,7 @@ export function compileAquaCallFromString({
 }) {
   return compile(
     new Call(funcCall, data, new Input(code)),
-    imports,
+    normalizeImports(imports),
     getConfig(commonArgs),
   );
 }
@@ -72,7 +107,7 @@ export function compileAquaCallFromPath({
 }) {
   return compile(
     new Call(funcCall, data, new Input(filePath)),
-    imports,
+    normalizeImports(imports),
     getConfig(commonArgs),
   );
 }
diff --git a/api/api/.js/src/main/scala/api/AquaAPI.scala b/api/api/.js/src/main/scala/api/AquaAPI.scala
@@ -1,84 +1,95 @@
 package api
 
-import api.types.{AquaConfig, AquaFunction, CompilationResult, GeneratedSource, Input}
 import aqua.Rendering.given
-import aqua.raw.value.ValueRaw
-import aqua.api.{APICompilation, APIResult, AquaAPIConfig}
+import aqua.SpanParser
 import aqua.api.TargetType.*
+import aqua.api.{APICompilation, APIResult, AquaAPIConfig, Imports}
 import aqua.backend.air.AirBackend
+import aqua.backend.api.APIBackend
+import aqua.backend.js.JavaScriptBackend
+import aqua.backend.ts.TypeScriptBackend
 import aqua.backend.{AirFunction, Backend, Generated}
 import aqua.compiler.*
-import aqua.files.{AquaFileSources, AquaFilesIO, FileModuleId}
-import aqua.logging.{LogFormatter, LogLevels}
 import aqua.constants.Constants
+import aqua.definitions.FunctionDef
+import aqua.files.{AquaFileSources, AquaFilesIO, FileModuleId}
 import aqua.io.*
-import aqua.raw.ops.Call
-import aqua.run.{CliFunc, FuncCompiler}
+import aqua.js.{FunctionDefJs, ServiceDefJs, VarJson}
+import aqua.logging.{LogFormatter, LogLevels}
+import aqua.model.AquaContext
+import aqua.model.transform.{Transform, TransformConfig}
 import aqua.parser.lexer.{LiteralToken, Token}
 import aqua.parser.lift.FileSpan.F
 import aqua.parser.lift.{FileSpan, Span}
 import aqua.parser.{ArrowReturnError, BlockIndentError, LexerError, ParserError}
-import aqua.{AquaIO, SpanParser}
-import aqua.model.transform.{Transform, TransformConfig}
-import aqua.backend.api.APIBackend
-import aqua.backend.js.JavaScriptBackend
-import aqua.backend.ts.TypeScriptBackend
-import aqua.definitions.FunctionDef
-import aqua.js.{FunctionDefJs, ServiceDefJs, VarJson}
-import aqua.model.AquaContext
+import aqua.raw.ops.Call
 import aqua.raw.ops.CallArrowRawTag
+import aqua.raw.value.ValueRaw
 import aqua.raw.value.{LiteralRaw, VarRaw}
 import aqua.res.AquaRes
 
+import api.types.{AquaConfig, AquaFunction, CompilationResult, GeneratedSource, Input}
 import cats.Applicative
+import cats.data.Validated.{Invalid, Valid, invalidNec, validNec}
 import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec}
-import cats.data.Validated.{invalidNec, validNec, Invalid, Valid}
+import cats.effect.IO
+import cats.effect.unsafe.implicits.global
 import cats.syntax.applicative.*
 import cats.syntax.apply.*
+import cats.syntax.either.*
 import cats.syntax.flatMap.*
 import cats.syntax.functor.*
-import cats.effect.IO
-import cats.effect.unsafe.implicits.global
 import cats.syntax.show.*
 import cats.syntax.traverse.*
-import cats.syntax.either.*
 import fs2.io.file.{Files, Path}
-import scribe.Logging
-
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
-import scala.scalajs.js.{|, undefined, Promise, UndefOr}
 import scala.scalajs.js
 import scala.scalajs.js.JSConverters.*
 import scala.scalajs.js.annotation.*
+import scala.scalajs.js.{Promise, UndefOr, undefined, |}
+import scribe.Logging
 
 @JSExportTopLevel("Aqua")
 object AquaAPI extends App with Logging {
 
+  // See api-npm package for description of imports config
+  type ImportsJS = js.Dictionary[
+    js.Dictionary[js.Array[String]]
+  ]
+
   /**
    * All-in-one function that support different inputs and backends
    * @param input can be a path to aqua file, string with a code or a function call
-   * @param imports list of paths
+   * @param imports imports configuration
    * @param aquaConfigJS compiler config
-   * @return compiler results depends on input and config
+   * @return compiler results depending on input and config
    */
   @JSExport
   def compile(
     input: types.Input | types.Path | types.Call,
-    imports: js.Array[String],
+    imports: ImportsJS,
     aquaConfigJS: js.UndefOr[AquaConfig]
   ): Promise[CompilationResult] = {
     aquaConfigJS.toOption
       .map(AquaConfig.fromJS)
       .getOrElse(validNec(AquaAPIConfig()))
       .traverse { config =>
-        val importsList = imports.toList
+        val apiImports = Imports.fromMap(
+          imports.view
+            .mapValues(
+              _.toMap.view
+                .mapValues(_.toList)
+                .toMap
+            )
+            .toMap
+        )
 
         input match {
           case i: (types.Input | types.Path) =>
-            compileAll(i, importsList, config)
+            compileAll(i, apiImports, config)
           case c: types.Call =>
-            compileCall(c, importsList, config)
+            compileCall(c, apiImports, config)
 
         }
       }
@@ -90,7 +101,7 @@ object AquaAPI extends App with Logging {
   // Compile all non-call inputs
   private def compileAll(
     input: types.Input | types.Path,
-    imports: List[String],
+    imports: Imports,
     config: AquaAPIConfig
   ): IO[CompilationResult] = {
     val backend: Backend = config.targetType match {
@@ -138,7 +149,7 @@ object AquaAPI extends App with Logging {
   // Compile a function call
   private def compileCall(
     call: types.Call,
-    imports: List[String],
+    imports: Imports,
     config: AquaAPIConfig
   ): IO[CompilationResult] = {
     val path = call.input match {

diff --git a/api/api/.jvm/src/main/scala/aqua/api/Test.scala b/api/api/.jvm/src/main/scala/aqua/api/Test.scala
@@ -17,7 +17,7 @@ object Test extends IOApp.Simple {
     APICompilation
       .compilePath(
         "./aqua-src/antithesis.aqua",
-        "./aqua" :: Nil,
+        Imports.fromMap(Map("/" -> Map("" -> List("./aqua")))),
         AquaAPIConfig(targetType = TypeScriptType),
         TypeScriptBackend(false, "IFluenceClient$$")
       )