From 6bf4b348d1729de5131e07770f12a434940b143b Mon Sep 17 00:00:00 2001 From: DieMyst Date: Mon, 30 Jan 2023 14:59:08 +0400 Subject: [PATCH 1/4] Par in make struct and copy struct --- .../main/scala/aqua/model/inline/Inline.scala | 30 +++++++- .../model/inline/MakeStructRawInliner.scala | 2 +- .../aqua/model/inline/RawValueInliner.scala | 2 +- .../inline/raw/ApplyIntoCopyRawInliner.scala | 2 +- .../raw/ApplyPropertiesRawInliner.scala | 2 +- .../aqua/model/inline/CopyInlinerSpec.scala | 71 +++++++++++++++++ .../model/inline/MakeStructInlinerSpec.scala | 76 +++++++++++++++++++ .../model/inline/RawValueInlinerSpec.scala | 1 - 8 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 model/inline/src/test/scala/aqua/model/inline/CopyInlinerSpec.scala create mode 100644 model/inline/src/test/scala/aqua/model/inline/MakeStructInlinerSpec.scala diff --git a/model/inline/src/main/scala/aqua/model/inline/Inline.scala b/model/inline/src/main/scala/aqua/model/inline/Inline.scala index 1db49e46f..baf3b0f25 100644 --- a/model/inline/src/main/scala/aqua/model/inline/Inline.scala +++ b/model/inline/src/main/scala/aqua/model/inline/Inline.scala @@ -13,7 +13,6 @@ object SeqMode extends MergeMode object ParMode extends MergeMode /** - * * @param flattenValues values that need to be resolved before `predo`. * ListMap for keeping order of values (mostly for debugging purposes) * @param predo operations tree @@ -23,7 +22,34 @@ private[inline] case class Inline( flattenValues: ListMap[String, ValueRaw] = ListMap.empty, predo: Chain[OpModel.Tree] = Chain.empty, mergeMode: MergeMode = ParMode -) +) { + + def desugar: Inline = { + val desugaredPredo = + predo.toList match { + case Nil => Chain.empty + case x :: Nil => + Chain.one(x) + case l => + mergeMode match + case SeqMode => + Chain.one(SeqModel.wrap(l: _*)) + case ParMode => Chain.one(ParModel.wrap(l: _*)) + } + + Inline( + flattenValues, + desugaredPredo + ) + } + + def mergeWith(inline: Inline, mode: MergeMode): Inline = { + val left = desugar + val right = inline.desugar + + Inline(left.flattenValues ++ right.flattenValues, left.predo ++ right.predo, mode) + } +} // TODO may not be needed there private[inline] object Inline { diff --git a/model/inline/src/main/scala/aqua/model/inline/MakeStructRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/MakeStructRawInliner.scala index 5c3653a27..a797c1a79 100644 --- a/model/inline/src/main/scala/aqua/model/inline/MakeStructRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/MakeStructRawInliner.scala @@ -53,7 +53,7 @@ object MakeStructRawInliner extends RawInliner[MakeStructRaw] { name <- Mangler[S].findAndForbidName(raw.structType.name + "_obj") foldedFields <- raw.fields.nonEmptyTraverse(unfold(_)) varModel = VarModel(name, raw.baseType) - valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _) + valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar fields = foldedFields.map(_._1) objCreation <- createObj(fields, varModel) } yield { diff --git a/model/inline/src/main/scala/aqua/model/inline/RawValueInliner.scala b/model/inline/src/main/scala/aqua/model/inline/RawValueInliner.scala index ea9b31ad2..8586d866e 100644 --- a/model/inline/src/main/scala/aqua/model/inline/RawValueInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/RawValueInliner.scala @@ -94,7 +94,7 @@ object RawValueInliner extends Logging { } } - private def toModel[S: Mangler: Exports: Arrows]( + private[inline] def toModel[S: Mangler: Exports: Arrows]( unfoldF: State[S, (ValueModel, Inline)] ): State[S, (ValueModel, Option[OpModel.Tree])] = for { diff --git a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyIntoCopyRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyIntoCopyRawInliner.scala index 05a98fc76..8f661a1fe 100644 --- a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyIntoCopyRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyIntoCopyRawInliner.scala @@ -53,7 +53,7 @@ object ApplyIntoCopyRawInliner extends Logging { name <- Mangler[S].findAndForbidName(value.name + "_obj_copy") foldedFields <- intoCopy.fields.nonEmptyTraverse(unfold(_)) varModel = VarModel(name, value.baseType) - valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _) + valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar fields = foldedFields.map(_._1) objCopy <- copyObj(value, fields, varModel) } yield { diff --git a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala index 8ede56c54..96e2216c1 100644 --- a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala @@ -184,7 +184,7 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi ((raw, properties.headOption) match { case (vr @ VarRaw(_, st @ StreamType(_)), Some(IntoIndexRaw(idx, _))) => unfold(vr).flatMap { - case (vm @ VarModel(nameVM, _, _), inl) => + case (VarModel(nameVM, _, _), inl) => val gateRaw = ApplyGateRaw(nameVM, st, idx) unfold(gateRaw).flatMap { case (gateResVal: VarModel, gateResInline) => diff --git a/model/inline/src/test/scala/aqua/model/inline/CopyInlinerSpec.scala b/model/inline/src/test/scala/aqua/model/inline/CopyInlinerSpec.scala new file mode 100644 index 000000000..1779f7559 --- /dev/null +++ b/model/inline/src/test/scala/aqua/model/inline/CopyInlinerSpec.scala @@ -0,0 +1,71 @@ +package aqua.model.inline + +import aqua.model.* +import aqua.model.inline.raw.ApplyIntoCopyRawInliner +import aqua.model.inline.state.InliningState +import aqua.raw.ops.* +import aqua.raw.value.* +import aqua.types.* +import cats.data.{Chain, NonEmptyList, NonEmptyMap} +import cats.syntax.show.* +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class CopyInlinerSpec extends AnyFlatSpec with Matchers { + + "copy inliner" should "unfold values in parallel" in { + + val structType = StructType( + "struct_type", + NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string) + ) + + val arrType = ArrayType(ScalarType.string) + val length = FunctorRaw("length", ScalarType.u32) + val lengthValue = VarRaw("l", arrType).withProperty(length) + + val getField = CallArrowRaw(None, "get_field", Nil, ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)), Option(LiteralRaw("\"serv\"", ScalarType.string))) + + val copyRaw = + IntoCopyRaw(structType, NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField)) + val varName = "some_struct" + val varRaw = VarRaw(varName, structType).withProperty(copyRaw) + + val (model, tree) = + RawValueInliner.valueToModel[InliningState](varRaw, false).run(InliningState()).value._2 + + val result = VarModel(varName + "_obj_copy", structType) + model shouldBe result + + val lengthModel = FunctorModel("length", ScalarType.u32) + + tree.get.equalsOrShowDiff( + SeqModel.wrap( + ParModel.wrap( + SeqModel.wrap( + FlattenModel(VarModel("l", arrType), "l_to_functor").leaf, + FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf, + ), + CallServiceModel( + "serv", + "get_field", + Nil, + VarModel("get_field", ScalarType.string) + ).leaf + ), + CallServiceModel( + "json", + "puts", + VarModel(varName, structType) :: LiteralModel.fromRaw( + LiteralRaw.quote("field1") + ) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw( + LiteralRaw.quote("field2") + ) :: VarModel("get_field", ScalarType.string) :: Nil, + result + ).leaf + ) + ) shouldBe true + + } + +} diff --git a/model/inline/src/test/scala/aqua/model/inline/MakeStructInlinerSpec.scala b/model/inline/src/test/scala/aqua/model/inline/MakeStructInlinerSpec.scala new file mode 100644 index 000000000..19bc926cf --- /dev/null +++ b/model/inline/src/test/scala/aqua/model/inline/MakeStructInlinerSpec.scala @@ -0,0 +1,76 @@ +package aqua.model.inline + +import aqua.model.* +import aqua.model.inline.raw.ApplyIntoCopyRawInliner +import aqua.model.inline.state.InliningState +import aqua.raw.ops.* +import aqua.raw.value.* +import aqua.types.* +import cats.data.{Chain, NonEmptyList, NonEmptyMap} +import cats.syntax.show.* +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class MakeStructInlinerSpec extends AnyFlatSpec with Matchers { + + "make struct inliner" should "unfold values in parallel" in { + + val structType = StructType( + "struct_type", + NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string) + ) + + val arrType = ArrayType(ScalarType.string) + val length = FunctorRaw("length", ScalarType.u32) + val lengthValue = VarRaw("l", arrType).withProperty(length) + + val getField = CallArrowRaw( + None, + "get_field", + Nil, + ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)), + Option(LiteralRaw("\"serv\"", ScalarType.string)) + ) + + val makeStruct = + MakeStructRaw(NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField), structType) + val varName = structType.name + + val (model, tree) = + RawValueInliner.valueToModel[InliningState](makeStruct, false).run(InliningState()).value._2 + + val result = VarModel(varName + "_obj", structType) + model shouldBe result + + val lengthModel = FunctorModel("length", ScalarType.u32) + + tree.get.equalsOrShowDiff( + SeqModel.wrap( + ParModel.wrap( + SeqModel.wrap( + FlattenModel(VarModel("l", arrType), "l_to_functor").leaf, + FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf + ), + CallServiceModel( + "serv", + "get_field", + Nil, + VarModel("get_field", ScalarType.string) + ).leaf + ), + CallServiceModel( + "json", + "obj", + LiteralModel.fromRaw( + LiteralRaw.quote("field1") + ) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw( + LiteralRaw.quote("field2") + ) :: VarModel("get_field", ScalarType.string) :: Nil, + result + ).leaf + ) + ) shouldBe true + + } + +} diff --git a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala index 69b048d31..8a75a77a5 100644 --- a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala +++ b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala @@ -325,7 +325,6 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers { ) ) ) - println(resTree) } "raw value inliner" should "desugarize stream with length" in { From 1e523d8cf16ff83167b9e795073ade62aa1aefcd Mon Sep 17 00:00:00 2001 From: DieMyst Date: Tue, 31 Jan 2023 14:40:34 +0400 Subject: [PATCH 2/4] optimize properties --- .../raw/ApplyPropertiesRawInliner.scala | 85 +++++++++++++------ .../model/inline/RawValueInlinerSpec.scala | 1 + 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala index 96e2216c1..3ccafdc0f 100644 --- a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala @@ -22,7 +22,7 @@ import aqua.model.{ XorModel } import aqua.model.inline.Inline -import aqua.model.inline.SeqMode +import aqua.model.inline.{ParMode, SeqMode} import aqua.model.inline.RawValueInliner.unfold import aqua.model.inline.state.{Arrows, Exports, Mangler} import aqua.raw.value.{ @@ -42,6 +42,7 @@ import aqua.types.{ArrayType, CanonStreamType, ScalarType, StreamType, Type} import cats.Eval import cats.data.{Chain, IndexedStateT, State} import cats.syntax.monoid.* +import cats.syntax.traverse.* import cats.instances.list.* import scribe.Logging @@ -140,8 +141,37 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi mergeMode = SeqMode ) } + } + + case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel]) + private def optimizeProperties[S: Mangler: Exports: Arrows]( + properties: Chain[PropertyRaw] + ): State[S, (Chain[PropertyRawWithModel], Inline)] = { + properties.map { + case iir @ IntoIndexRaw(vr, t) => + unfold(vr, propertiesAllowed = false).flatMap { + case (vm@VarModel(_, _, _), inline) if vm.properties.nonEmpty => + removeProperties(vm).map { case (vf, inlf) => + PropertyRawWithModel(iir, Option(IntoIndexModel(vf.name, t))) -> Inline( + inline.flattenValues ++ inlf.flattenValues, + inline.predo ++ inlf.predo, + mergeMode = SeqMode + ) + } + case (VarModel(name, _, _), inline) => + State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(name, t))) -> inline) + case (LiteralModel(literal, _), inline) => + State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(literal, t))) -> inline) + } + + case p => State.pure(PropertyRawWithModel(p, None) -> Inline.empty) + }.sequence.map { (propsWithInline: Chain[(PropertyRawWithModel, Inline)]) => + val fullInline = propsWithInline.map(_._2).foldLeft(Inline.empty)(_ |+| _) + val props = propsWithInline.map(_._1) + (props, fullInline) } + } private def unfoldProperties[S: Mangler: Exports: Arrows]( prevInline: Inline, @@ -149,31 +179,38 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi properties: Chain[PropertyRaw], propertiesAllowed: Boolean ): State[S, (VarModel, Inline)] = { - properties - .foldLeft[State[S, (VarModel, Inline)]]( - State.pure((vm, prevInline)) - ) { case (state, property) => - state.flatMap { case (vm, leftInline) => - unfoldProperty(vm, property).flatMap { - case (v, i) if !propertiesAllowed && v.properties.nonEmpty => - removeProperties(v).map { case (vf, inlf) => - vf -> Inline( - leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues, - leftInline.predo ++ i.predo ++ inlf.predo, - mergeMode = SeqMode - ) - } - case (v, i) => - State.pure( - v -> Inline( - leftInline.flattenValues ++ i.flattenValues, - leftInline.predo ++ i.predo, - mergeMode = SeqMode - ) - ) + optimizeProperties(properties).flatMap { case (optimizedProps, optimizationInline) => + optimizedProps + .foldLeft[State[S, (VarModel, Inline)]]( + State.pure((vm, prevInline.mergeWith(optimizationInline, ParMode))) + ) { case (state, property) => + state.flatMap { case (vm, leftInline) => + property match { + case PropertyRawWithModel(_, Some(model)) => + State.pure(vm.copy(properties = vm.properties :+ model) -> leftInline) + case PropertyRawWithModel(raw, _) => + unfoldProperty(vm, raw).flatMap { + case (v, i) if !propertiesAllowed && v.properties.nonEmpty => + removeProperties(v).map { case (vf, inlf) => + vf -> Inline( + leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues, + leftInline.predo ++ i.predo ++ inlf.predo, + mergeMode = SeqMode + ) + } + case (v, i) => + State.pure( + v -> Inline( + leftInline.flattenValues ++ i.flattenValues, + leftInline.predo ++ i.predo, + mergeMode = SeqMode + ) + ) + } + } } } - } + } } private def unfoldRawWithProperties[S: Mangler: Exports: Arrows]( diff --git a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala index 8a75a77a5..591796661 100644 --- a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala +++ b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala @@ -17,6 +17,7 @@ import aqua.raw.value.{ApplyPropertyRaw, FunctorRaw, IntoIndexRaw, LiteralRaw, V import aqua.types.* import cats.data.NonEmptyMap import cats.data.Chain +import cats.syntax.show.* import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers From 720865a05254da6096762372ab31d9e434e8d41c Mon Sep 17 00:00:00 2001 From: DieMyst Date: Tue, 31 Jan 2023 14:51:57 +0400 Subject: [PATCH 3/4] fix tests --- .../aqua/model/inline/raw/ApplyPropertiesRawInliner.scala | 2 +- .../test/scala/aqua/model/inline/RawValueInlinerSpec.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala index 3ccafdc0f..8b1226354 100644 --- a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala @@ -182,7 +182,7 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi optimizeProperties(properties).flatMap { case (optimizedProps, optimizationInline) => optimizedProps .foldLeft[State[S, (VarModel, Inline)]]( - State.pure((vm, prevInline.mergeWith(optimizationInline, ParMode))) + State.pure((vm, prevInline.mergeWith(optimizationInline, SeqMode))) ) { case (state, property) => state.flatMap { case (vm, leftInline) => property match { diff --git a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala index 591796661..8e1d40f27 100644 --- a/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala +++ b/model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala @@ -208,7 +208,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers { resTree.isEmpty should be(false) resTree.get.equalsOrShowDiff( - SeqModel.wrap( + ParModel.wrap( SeqModel.wrap( FlattenModel( VarModel( @@ -285,7 +285,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers { resTree.isEmpty should be(false) resTree.get.equalsOrShowDiff( - SeqModel.wrap( + ParModel.wrap( FlattenModel( VarModel( "ys", @@ -367,7 +367,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers { resTree.isEmpty should be(false) resTree.get.equalsOrShowDiff( - SeqModel.wrap( + ParModel.wrap( FlattenModel( VarModel( "ys", From 69ecee71d4c2c6ac23b7ffcedbe0da2eb0c43a12 Mon Sep 17 00:00:00 2001 From: DieMyst Date: Wed, 1 Feb 2023 11:21:23 +0400 Subject: [PATCH 4/4] PR fixes --- .../aqua/model/inline/raw/ApplyPropertiesRawInliner.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala index 8b1226354..5e31a0a60 100644 --- a/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala +++ b/model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala @@ -143,8 +143,10 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi } } - case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel]) + // Helper for `optimizeProperties` + private case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel]) + // Unfold properties that we can process in parallel private def optimizeProperties[S: Mangler: Exports: Arrows]( properties: Chain[PropertyRaw] ): State[S, (Chain[PropertyRawWithModel], Inline)] = {