From f7dbfc0e195cb62438ad24d57a119c7622c47233 Mon Sep 17 00:00:00 2001 From: "P. Oscar Boykin" Date: Tue, 4 Feb 2025 15:00:30 -1000 Subject: [PATCH] Add DecodedLibraryWithDeps (#1374) --- .../org/bykn/bosatsu/library/Command.scala | 80 +++++++------ .../bykn/bosatsu/library/DecodedLibrary.scala | 112 ++++++++++++++++-- 2 files changed, 144 insertions(+), 48 deletions(-) diff --git a/core/src/main/scala/org/bykn/bosatsu/library/Command.scala b/core/src/main/scala/org/bykn/bosatsu/library/Command.scala index 36cd590f8..ed002218a 100644 --- a/core/src/main/scala/org/bykn/bosatsu/library/Command.scala +++ b/core/src/main/scala/org/bykn/bosatsu/library/Command.scala @@ -597,21 +597,56 @@ object Command { libFromCas(dep) } .map { fetchedLibsOpt => - val fetchedDeps = fetchedLibsOpt.flatMap { + fetchedLibsOpt.flatMap { case None => Nil case Some(dep) => // we will find the transitivies by walking them - dep.arg.publicDependencies.toList ::: dep.arg.privateDependencies.toList + (dep.arg.publicDependencies.toList ::: dep.arg.privateDependencies.toList) + .filterNot { dep => + nextFetched.contains((dep.name, versionOf(dep))) + } } - fetchedDeps.filterNot { dep => - nextFetched.contains((dep.name, versionOf(dep))) - } } nextBatchF.map((nextFetched, _)) } + def showFetchState(fs: FetchState): F[Doc] = { + val depStr = if (fs.size == 1) "dependency" else "dependencies" + val header = Doc.text(s"fetched ${fs.size} transitive ${depStr}.") + + val resultDoc = header + Doc.line + Doc.intercalate( + Doc.hardLine, + fs.toList.map { case ((n, v), hashes) => + val sortedHashes = hashes.toList.sortBy(_._1.toIdent) + val hashDoc = Doc.intercalate( + Doc.comma + Doc.line, + sortedHashes.map { case (wh, msg) => + val ident = wh.toIdent + msg match { + case Right(true) => Doc.text(show"fetched $ident") + case Right(false) => Doc.text(show"cached $ident") + case Left(err) => + Doc.text(show"failed: $ident ${err.getMessage}") + } + } + ) + + Doc.text(show"$n $v:") + (Doc.line + hashDoc).nested(4).grouped + } + ) + + val success = fs.forall { case (_, dl) => + dl.forall { case (_, res) => res.isRight } + } + if (success) moduleIOMonad.pure(resultDoc) + else + moduleIOMonad.raiseError( + CliException("failed to fetch", err = resultDoc) + ) + } + moduleIOMonad .tailRecM((SortedMap.empty: FetchState, deps)) { case (fetched, deps) => step(fetched, deps).map { @@ -619,40 +654,7 @@ object Command { case next => Left(next) } } - .flatMap { fs => - val depStr = if (fs.size == 1) "dependency" else "dependencies" - val header = Doc.text(s"fetched ${fs.size} transitive ${depStr}.") - - val resultDoc = header + Doc.line + Doc.intercalate( - Doc.hardLine, - fs.toList.map { case ((n, v), hashes) => - val sortedHashes = hashes.toList.sortBy(_._1.toIdent) - val hashDoc = Doc.intercalate( - Doc.comma + Doc.line, - sortedHashes.map { case (wh, msg) => - val ident = wh.toIdent - msg match { - case Right(true) => Doc.text(show"fetched $ident") - case Right(false) => Doc.text(show"cached $ident") - case Left(err) => - Doc.text(show"failed: $ident ${err.getMessage}") - } - } - ) - - Doc.text(show"$n $v:") + (Doc.line + hashDoc).nested(4).grouped - } - ) - - val success = fs.forall { case (_, dl) => - dl.forall { case (_, res) => res.isRight } - } - if (success) moduleIOMonad.pure(resultDoc) - else - moduleIOMonad.raiseError( - CliException("failed to fetch", err = resultDoc) - ) - } + .flatMap(showFetchState(_)) } } } diff --git a/core/src/main/scala/org/bykn/bosatsu/library/DecodedLibrary.scala b/core/src/main/scala/org/bykn/bosatsu/library/DecodedLibrary.scala index 3e399bbc4..81763be9f 100644 --- a/core/src/main/scala/org/bykn/bosatsu/library/DecodedLibrary.scala +++ b/core/src/main/scala/org/bykn/bosatsu/library/DecodedLibrary.scala @@ -2,12 +2,17 @@ package org.bykn.bosatsu.library import _root_.bosatsu.{TypedAst => proto} import cats.MonadError +import cats.data.StateT import cats.syntax.all._ -import org.bykn.bosatsu.hashing.{Hashed, HashValue} +import org.bykn.bosatsu.hashing.{Algo, Hashed, HashValue} +import org.bykn.bosatsu.tool.CliException import org.bykn.bosatsu.{Package, PackageName, PackageMap, ProtoConverter} import scala.collection.immutable.{SortedMap, SortedSet} +import org.typelevel.paiges.Doc case class DecodedLibrary[A]( + name: Name, + version: Version, hashValue: HashValue[A], protoLib: proto.Library, interfaces: List[Package.Interface], @@ -18,7 +23,7 @@ case class DecodedLibrary[A]( } object DecodedLibrary { - def decode[F[_], A]( + def decode[F[_], A: Algo]( protoLib: Hashed[A, proto.Library] )(implicit F: MonadError[F, Throwable]): F[DecodedLibrary[A]] = F.fromTry( @@ -27,14 +32,103 @@ object DecodedLibrary { protoLib.arg.exportedIfaces, protoLib.arg.internalPackages ) - ).map { case (ifs, impls) => + ).flatMap { case (ifs, impls) => // TODO: should verify somewhere that all the package names are distinct, but since this is presumed to be // a good library maybe that's a waste - DecodedLibrary[A]( - protoLib.hash, - protoLib.arg, - ifs, - PackageMap(impls.iterator.map(pack => (pack.name, pack)).to(SortedMap)) - ) + + protoLib.arg.descriptor.flatMap(_.version) match { + case Some(protoV) => + F.pure( + DecodedLibrary[A]( + Name(protoLib.arg.name), + Version.fromProto(protoV), + protoLib.hash, + protoLib.arg, + ifs, + PackageMap( + impls.iterator.map(pack => (pack.name, pack)).to(SortedMap) + ) + ) + ) + case None => + F.raiseError( + CliException( + "missing version", + Doc.text( + show"while decoding library ${protoLib.arg.name} with hash ${protoLib.hash.toIdent} has missing version." + ) + ) + ) + } + } +} + +case class DecodedLibraryWithDeps[A]( + lib: DecodedLibrary[A], + deps: SortedMap[(Name, Version), DecodedLibraryWithDeps[A]] +) { + def name: Name = lib.name + def version: Version = lib.version +} + +object DecodedLibraryWithDeps { + def decodeAll[F[_]]( + protoLib: Hashed[Algo.Blake3, proto.Library] + )(load: proto.LibDependency => F[Hashed[Algo.Blake3, proto.Library]])(implicit + F: MonadError[F, Throwable] + ): F[DecodedLibraryWithDeps[Algo.Blake3]] = { + type Key = (Name, Version) + type Value = DecodedLibraryWithDeps[Algo.Blake3] + type S = Map[Key, Value] + type Cached[T] = StateT[F, S, T] + + val getS: Cached[S] = StateT.get + + def get(k: Key): Cached[Option[Value]] = + getS.map(_.get(k)) + + def decodeAndStore( + protoLib: Hashed[Algo.Blake3, proto.Library] + ): Cached[Value] = + for { + root <- StateT.liftF(DecodedLibrary.decode[F, Algo.Blake3](protoLib)) + deps <- + (protoLib.arg.privateDependencies.toList ::: protoLib.arg.publicDependencies.toList) + .traverse(fetchDep(_)) + depMap = deps.iterator + .map(dec => (dec.name, dec.version) -> dec) + .to(SortedMap) + result = DecodedLibraryWithDeps(root, depMap) + _ <- StateT.modify[F, S]( + _.updated((root.name, root.version), result) + ) + } yield result + + def fetchDep(dep: proto.LibDependency): Cached[Value] = { + val fV = dep.desc.flatMap(_.version) match { + case Some(v) => F.pure(Version.fromProto(v)) + case None => + F.raiseError[Version]( + CliException( + "missing version", + Doc.text( + show"while loading dependency ${dep.name} with hashes=${dep.desc.toList.flatMap(_.hashes).mkString(",")}: missing version." + ) + ) + ) + } + + for { + v <- StateT.liftF[F, S, Version](fV) + cached <- get((Name(dep.name), v)) + res <- cached match { + case Some(d) => StateT.pure[F, S, Value](d) + case None => + StateT.liftF(load(dep)).flatMap(decodeAndStore(_)) + } + } yield res } + + decodeAndStore(protoLib).runA(Map.empty) + } }