From e3670f8bee80a0fc76e484a578dcb9bef3757d9e Mon Sep 17 00:00:00 2001 From: InversionSpaces Date: Tue, 13 Jun 2023 11:50:22 +0000 Subject: [PATCH 1/9] WIP --- build.sbt | 7 +- .../main/scala/aqua/compiler/Library.scala | 81 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 compiler-native-lib/src/main/scala/aqua/compiler/Library.scala diff --git a/build.sbt b/build.sbt index edce76fa3..f3545f375 100644 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,7 @@ name := "aqua-hll" val commons = Seq( version := { val aquaSnapshot = sys.env.getOrElse("SNAPSHOT", "") - if (aquaSnapshot.isEmpty()) aquaVersion else aquaVersion + "-" + aquaSnapshot, + if (aquaSnapshot.isEmpty()) aquaVersion else aquaVersion + "-" + aquaSnapshot }, scalaVersion := scalaV, libraryDependencies ++= Seq( @@ -212,6 +212,11 @@ lazy val compiler = crossProject(JVMPlatform, JSPlatform) .settings(commons) .dependsOn(semantics, linker, backend, transform % "test->test", res % "test->test") +lazy val `compiler-native-lib` = project + .in(file("compiler-native-lib")) + .settings(commons: _*) + .dependsOn(compiler.jvm, io.jvm, transform.jvm, `backend-air`.jvm) + lazy val backend = crossProject(JVMPlatform, JSPlatform) .withoutSuffixFor(JVMPlatform) .crossType(CrossType.Pure) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala new file mode 100644 index 000000000..f1afa48ca --- /dev/null +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -0,0 +1,81 @@ +package aqua.compiler + +import aqua.AquaIO +import org.graalvm.nativeimage.IsolateThread +import org.graalvm.nativeimage.c.function.CEntryPoint +import org.graalvm.nativeimage.c.`type`.CCharPointer +import org.graalvm.nativeimage.c.`type`.CTypeConversion +import aqua.io.AquaFileError +import aqua.files.{AquaFileSources, FileModuleId} +import cats.{Applicative, Functor, Monad} +import cats.data.{Chain, ValidatedNec} +import cats.data.Validated.Valid +import cats.data.Validated.validNec +import cats.effect.IO +import fs2.io.file.{Files, Path} +import cats.implicits.* +import aqua.SpanParser +import aqua.model.transform.TransformConfig +import aqua.model.AquaContext +import aqua.model.transform.Transform +import aqua.backend.Backend +import aqua.backend.AirFunction +import aqua.backend.Generated +import aqua.res.AquaRes +import aqua.parser.lift.FileSpan +import aqua.backend.air.AirBackend +import aqua.files.AquaFilesIO.summon +import cats.effect.unsafe.implicits.global +object Library { + + + private final val path = Path("") + + private class RawAquaSource[F[_] : AquaIO : Monad : Files](input: String, imports: List[String]) extends AquaFileSources[F](path, imports.map(Path.apply)): + override def sources: F[ValidatedNec[AquaFileError, Chain[(FileModuleId, String)]]] = { + Applicative[F].pure(Valid(Chain.one((FileModuleId(path), input)))) + } + + + private class LocalBackendTransform(transformConfig: TransformConfig) extends Backend.Transform: + override def transform(ex: AquaContext): AquaRes = + Transform.contextRes(ex, transformConfig) + + override def generate(aqua: AquaRes): Seq[Generated] = AirBackend.generate(aqua) + + + private class LocalAirValidator[F[_] : Applicative]() extends AirValidator[F]: + + override def init(): F[Unit] = Applicative[F].pure(()) + + override def validate(airs: List[AirFunction]): F[ValidatedNec[String, Unit]] = + Applicative[F].pure(validNec(())) + + private def compileF[F[_] : AquaIO : Monad : Files](input: String, imports: List[String]): F[ValidatedNec[AquaError[FileModuleId, AquaFileError, FileSpan.F], Chain[AquaCompiled[FileModuleId]]]] = for { + sources <- new RawAquaSource[F](input, imports).pure + transformConfig <- TransformConfig().pure + backendTransform <- new LocalBackendTransform(transformConfig).pure + validator <- new LocalAirValidator[F]().pure + result <- CompilerAPI + .compile[F, AquaFileError, FileModuleId, FileSpan.F]( + sources, + SpanParser.parser, + validator, + backendTransform, + AquaCompilerConf(transformConfig.constantsList)) + } yield result + + @CEntryPoint(name = "compile") def compile(thread: IsolateThread, codePointer: CCharPointer): Int = { + val code = CTypeConversion.toJavaString(codePointer) + + val program = compileF[IO](code, List.empty) + val result = program.unsafeRunSync() + + if (result.isValid) { + 0 + } else { + 1 + } + } + +} From e56f5df8351402e0ebf278b0acc57da71ef01dfc Mon Sep 17 00:00:00 2001 From: InversionSpaces Date: Tue, 13 Jun 2023 15:48:27 +0000 Subject: [PATCH 2/9] Rewrite library, set up native image --- build.sbt | 15 ++- .../main/scala/aqua/compiler/Library.scala | 110 +++++++----------- 2 files changed, 58 insertions(+), 67 deletions(-) diff --git a/build.sbt b/build.sbt index f3545f375..4cbd2459c 100644 --- a/build.sbt +++ b/build.sbt @@ -214,8 +214,21 @@ lazy val compiler = crossProject(JVMPlatform, JSPlatform) lazy val `compiler-native-lib` = project .in(file("compiler-native-lib")) + .enablePlugins(GraalVMNativeImagePlugin) .settings(commons: _*) - .dependsOn(compiler.jvm, io.jvm, transform.jvm, `backend-air`.jvm) + .settings( + Compile / mainClass := Some("aqua.compiler.Library"), + graalVMNativeImageOptions ++= Seq( + "--verbose", + "--no-fallback", + // Uncomment next lines to use llvm backend + // and obtain bitcode files + // "-H:CompilerBackend=llvm", + // "-H:TempDirectory=temp", // Directory with bc files + "--shared" // Produce shared library + ) + ) + .dependsOn(`aqua-api`.jvm) lazy val backend = crossProject(JVMPlatform, JSPlatform) .withoutSuffixFor(JVMPlatform) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala index f1afa48ca..14985917e 100644 --- a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -1,81 +1,59 @@ package aqua.compiler -import aqua.AquaIO import org.graalvm.nativeimage.IsolateThread import org.graalvm.nativeimage.c.function.CEntryPoint -import org.graalvm.nativeimage.c.`type`.CCharPointer -import org.graalvm.nativeimage.c.`type`.CTypeConversion -import aqua.io.AquaFileError -import aqua.files.{AquaFileSources, FileModuleId} -import cats.{Applicative, Functor, Monad} -import cats.data.{Chain, ValidatedNec} -import cats.data.Validated.Valid -import cats.data.Validated.validNec -import cats.effect.IO -import fs2.io.file.{Files, Path} -import cats.implicits.* -import aqua.SpanParser -import aqua.model.transform.TransformConfig -import aqua.model.AquaContext -import aqua.model.transform.Transform -import aqua.backend.Backend -import aqua.backend.AirFunction -import aqua.backend.Generated -import aqua.res.AquaRes -import aqua.parser.lift.FileSpan -import aqua.backend.air.AirBackend -import aqua.files.AquaFilesIO.summon -import cats.effect.unsafe.implicits.global -object Library { - - - private final val path = Path("") - - private class RawAquaSource[F[_] : AquaIO : Monad : Files](input: String, imports: List[String]) extends AquaFileSources[F](path, imports.map(Path.apply)): - override def sources: F[ValidatedNec[AquaFileError, Chain[(FileModuleId, String)]]] = { - Applicative[F].pure(Valid(Chain.one((FileModuleId(path), input)))) - } - +import org.graalvm.nativeimage.c.`type`.{CCharPointer, CCharPointerPointer, CTypeConversion} - private class LocalBackendTransform(transformConfig: TransformConfig) extends Backend.Transform: - override def transform(ex: AquaContext): AquaRes = - Transform.contextRes(ex, transformConfig) +import scala.annotation.static - override def generate(aqua: AquaRes): Seq[Generated] = AirBackend.generate(aqua) - - - private class LocalAirValidator[F[_] : Applicative]() extends AirValidator[F]: +import cats.effect.unsafe.implicits.global - override def init(): F[Unit] = Applicative[F].pure(()) +import aqua.api.{APICompilation, AquaAPIConfig} +import aqua.backend.api.APIBackend - override def validate(airs: List[AirFunction]): F[ValidatedNec[String, Unit]] = - Applicative[F].pure(validNec(())) +// This is neede for @static to work in object +class Library {} - private def compileF[F[_] : AquaIO : Monad : Files](input: String, imports: List[String]): F[ValidatedNec[AquaError[FileModuleId, AquaFileError, FileSpan.F], Chain[AquaCompiled[FileModuleId]]]] = for { - sources <- new RawAquaSource[F](input, imports).pure - transformConfig <- TransformConfig().pure - backendTransform <- new LocalBackendTransform(transformConfig).pure - validator <- new LocalAirValidator[F]().pure - result <- CompilerAPI - .compile[F, AquaFileError, FileModuleId, FileSpan.F]( - sources, - SpanParser.parser, - validator, - backendTransform, - AquaCompilerConf(transformConfig.constantsList)) - } yield result +object Library { - @CEntryPoint(name = "compile") def compile(thread: IsolateThread, codePointer: CCharPointer): Int = { + @CEntryPoint(name = "compile") + @static + def compile( + thread: IsolateThread, + codePointer: CCharPointer, + resultPointer: CCharPointerPointer, + errorsPointer: CCharPointerPointer + ): Int = { val code = CTypeConversion.toJavaString(codePointer) - val program = compileF[IO](code, List.empty) - val result = program.unsafeRunSync() - - if (result.isValid) { - 0 - } else { - 1 - } + val result = APICompilation + .compileString( + code, + imports = Nil, + aquaConfig = AquaAPIConfig(), + backend = APIBackend + ) + .unsafeRunSync() + + result.fold( + errors => + errors.toChain.toList.zipWithIndex.foreach { case (error, i) => + errorsPointer.write(i, CTypeConversion.toCString(error).get()) + } + + 1 + , + compiled => + compiled.toList.flatMap(_.compiled).flatMap(_.air).map(_.air).zipWithIndex.foreach { + case (air, i) => resultPointer.write(i, CTypeConversion.toCString(code).get()) + } + + 0 + ) } + // Without main native-image refuses to work + @static + def main(args: Array[String]): Unit = () + } From eebb35fa558febedc2f03a71d90d0a9bf16b2403 Mon Sep 17 00:00:00 2001 From: InversionSpaces Date: Tue, 13 Jun 2023 16:51:11 +0000 Subject: [PATCH 3/9] Use sbt-native-image --- build.sbt | 8 +++++--- .../src/main/scala/aqua/compiler/Library.scala | 3 +++ project/plugins.sbt | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 4cbd2459c..889161b36 100644 --- a/build.sbt +++ b/build.sbt @@ -214,18 +214,20 @@ lazy val compiler = crossProject(JVMPlatform, JSPlatform) lazy val `compiler-native-lib` = project .in(file("compiler-native-lib")) - .enablePlugins(GraalVMNativeImagePlugin) + .enablePlugins(NativeImagePlugin) .settings(commons: _*) .settings( Compile / mainClass := Some("aqua.compiler.Library"), - graalVMNativeImageOptions ++= Seq( + nativeImageVersion := "22.1.0", + nativeImageOptions ++= Seq( "--verbose", "--no-fallback", + "--shared", // Produce shared library + "--initialize-at-run-time=aqua.logging.LogFormatter$" // Uncomment next lines to use llvm backend // and obtain bitcode files // "-H:CompilerBackend=llvm", // "-H:TempDirectory=temp", // Directory with bc files - "--shared" // Produce shared library ) ) .dependsOn(`aqua-api`.jvm) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala index 14985917e..6a8086900 100644 --- a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -10,6 +10,7 @@ import cats.effect.unsafe.implicits.global import aqua.api.{APICompilation, AquaAPIConfig} import aqua.backend.api.APIBackend +import aqua.logging.LogFormatter // This is neede for @static to work in object class Library {} @@ -26,6 +27,8 @@ object Library { ): Int = { val code = CTypeConversion.toJavaString(codePointer) + LogFormatter.initLogger(Some(scribe.Level.Info)) + val result = APICompilation .compileString( code, diff --git a/project/plugins.sbt b/project/plugins.sbt index 7044eb9b2..d7360d4d7 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,4 +2,5 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.15.0") addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.3.2") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") +addSbtPlugin("org.scalameta" % "sbt-native-image" % "0.3.4") addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.10.0") From db9cea18414cb9b4dff18473672947b83ca89162 Mon Sep 17 00:00:00 2001 From: InversionSpaces Date: Fri, 6 Oct 2023 08:37:33 +0000 Subject: [PATCH 4/9] Fix compilation --- compiler-native-lib/src/main/scala/aqua/compiler/Library.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala index 6a8086900..c60c13b84 100644 --- a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -38,7 +38,7 @@ object Library { ) .unsafeRunSync() - result.fold( + result.value.value.fold( errors => errors.toChain.toList.zipWithIndex.foreach { case (error, i) => errorsPointer.write(i, CTypeConversion.toCString(error).get()) From 9069876fa179df979a414a7d23ed326984ee6de9 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 6 Oct 2023 12:56:48 +0300 Subject: [PATCH 5/9] Change graalvm version --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 889161b36..5e2700bce 100644 --- a/build.sbt +++ b/build.sbt @@ -218,7 +218,8 @@ lazy val `compiler-native-lib` = project .settings(commons: _*) .settings( Compile / mainClass := Some("aqua.compiler.Library"), - nativeImageVersion := "22.1.0", + nativeImageJvm := "graalvm-java20", + nativeImageVersion:="20.0.2", nativeImageOptions ++= Seq( "--verbose", "--no-fallback", From 000e3ed734cf3bb860c8bf7a3f591bb775a445d0 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 6 Oct 2023 14:07:40 +0300 Subject: [PATCH 6/9] Change graalvm version --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 5e2700bce..b1d442745 100644 --- a/build.sbt +++ b/build.sbt @@ -217,6 +217,7 @@ lazy val `compiler-native-lib` = project .enablePlugins(NativeImagePlugin) .settings(commons: _*) .settings( + name := "libaqua", Compile / mainClass := Some("aqua.compiler.Library"), nativeImageJvm := "graalvm-java20", nativeImageVersion:="20.0.2", @@ -224,7 +225,7 @@ lazy val `compiler-native-lib` = project "--verbose", "--no-fallback", "--shared", // Produce shared library - "--initialize-at-run-time=aqua.logging.LogFormatter$" + "--initialize-at-run-time=aqua.logging.LogFormatter$", // Uncomment next lines to use llvm backend // and obtain bitcode files // "-H:CompilerBackend=llvm", From 5924486895a5878fb75f98e08dd4cf1ee26fb4cf Mon Sep 17 00:00:00 2001 From: InversionSpaces Date: Fri, 6 Oct 2023 11:14:11 +0000 Subject: [PATCH 7/9] Init empty logger --- .../src/main/scala/aqua/compiler/Library.scala | 2 +- .../logging/src/main/scala/aqua/logging/LogFormatter.scala | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala index c60c13b84..9bf3e6f66 100644 --- a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -27,7 +27,7 @@ object Library { ): Int = { val code = CTypeConversion.toJavaString(codePointer) - LogFormatter.initLogger(Some(scribe.Level.Info)) + LogFormatter.initEmptyLogger() val result = APICompilation .compileString( diff --git a/utils/logging/src/main/scala/aqua/logging/LogFormatter.scala b/utils/logging/src/main/scala/aqua/logging/LogFormatter.scala index 62cacf1be..3845ab81f 100644 --- a/utils/logging/src/main/scala/aqua/logging/LogFormatter.scala +++ b/utils/logging/src/main/scala/aqua/logging/LogFormatter.scala @@ -18,4 +18,10 @@ object LogFormatter { .withHandler(formatter = formatter, minimumLevel = level) .replace() } + + def initEmptyLogger(): Logger = + scribe.Logger.root + .clearHandlers() + .clearModifiers() + .replace() } From 5966cc5dd2cf53b1a96d7a8a1eb8df0b4045cdc5 Mon Sep 17 00:00:00 2001 From: Nick Pavlov Date: Wed, 1 May 2024 11:07:29 +0300 Subject: [PATCH 8/9] Fix graal --- api/api/src/main/scala/aqua/api/Imports.scala | 1 + build.sbt | 10 ++++++---- .../src/main/scala/aqua/compiler/Library.scala | 6 ++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/api/src/main/scala/aqua/api/Imports.scala b/api/api/src/main/scala/aqua/api/Imports.scala index 880dad2fd..f2ff23e30 100644 --- a/api/api/src/main/scala/aqua/api/Imports.scala +++ b/api/api/src/main/scala/aqua/api/Imports.scala @@ -12,6 +12,7 @@ final case class Imports( settings: Map[Path, Imports.PathSettings] ) { + def toIO: IOImports = IOImports( settings.view diff --git a/build.sbt b/build.sbt index b1d442745..2bb3738a0 100644 --- a/build.sbt +++ b/build.sbt @@ -219,8 +219,7 @@ lazy val `compiler-native-lib` = project .settings( name := "libaqua", Compile / mainClass := Some("aqua.compiler.Library"), - nativeImageJvm := "graalvm-java20", - nativeImageVersion:="20.0.2", + nativeImageVersion:="22.3.1", nativeImageOptions ++= Seq( "--verbose", "--no-fallback", @@ -228,8 +227,11 @@ lazy val `compiler-native-lib` = project "--initialize-at-run-time=aqua.logging.LogFormatter$", // Uncomment next lines to use llvm backend // and obtain bitcode files - // "-H:CompilerBackend=llvm", - // "-H:TempDirectory=temp", // Directory with bc files + // "-H:CompilerBackend=llvm", + // "-H:TempDirectory=temp", // Directory with bc files + ), + libraryDependencies ++= Seq( + "org.graalvm.sdk" % "graal-sdk" % "24.0.1" ) ) .dependsOn(`aqua-api`.jvm) diff --git a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala index 9bf3e6f66..a57076179 100644 --- a/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala +++ b/compiler-native-lib/src/main/scala/aqua/compiler/Library.scala @@ -5,10 +5,8 @@ import org.graalvm.nativeimage.c.function.CEntryPoint import org.graalvm.nativeimage.c.`type`.{CCharPointer, CCharPointerPointer, CTypeConversion} import scala.annotation.static - import cats.effect.unsafe.implicits.global - -import aqua.api.{APICompilation, AquaAPIConfig} +import aqua.api.{APICompilation, AquaAPIConfig, Imports} import aqua.backend.api.APIBackend import aqua.logging.LogFormatter @@ -32,7 +30,7 @@ object Library { val result = APICompilation .compileString( code, - imports = Nil, + imports = Imports(Map.empty), aquaConfig = AquaAPIConfig(), backend = APIBackend ) From 443584513f18f8dacb3f86d846beb033dccfc986 Mon Sep 17 00:00:00 2001 From: Nick Pavlov Date: Wed, 1 May 2024 12:30:19 +0300 Subject: [PATCH 9/9] cleanup --- api/api/src/main/scala/aqua/api/Imports.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/api/src/main/scala/aqua/api/Imports.scala b/api/api/src/main/scala/aqua/api/Imports.scala index f2ff23e30..c9bca47b1 100644 --- a/api/api/src/main/scala/aqua/api/Imports.scala +++ b/api/api/src/main/scala/aqua/api/Imports.scala @@ -11,8 +11,7 @@ import fs2.io.file.Path final case class Imports( settings: Map[Path, Imports.PathSettings] ) { - - + def toIO: IOImports = IOImports( settings.view