mirror of
https://github.com/fluencelabs/aqua.git
synced 2025-03-15 11:40:50 +00:00
perf: Unfold variables in parallel where it is possible (fixes LNG-109 ) (#656)
This commit is contained in:
parent
50c1da3b2f
commit
439f2cde03
@ -13,7 +13,6 @@ object SeqMode extends MergeMode
|
||||
object ParMode extends MergeMode
|
||||
|
||||
/**
|
||||
*
|
||||
* @param flattenValues values that need to be resolved before `predo`.
|
||||
* ListMap for keeping order of values (mostly for debugging purposes)
|
||||
* @param predo operations tree
|
||||
@ -23,7 +22,34 @@ private[inline] case class Inline(
|
||||
flattenValues: ListMap[String, ValueRaw] = ListMap.empty,
|
||||
predo: Chain[OpModel.Tree] = Chain.empty,
|
||||
mergeMode: MergeMode = ParMode
|
||||
)
|
||||
) {
|
||||
|
||||
def desugar: Inline = {
|
||||
val desugaredPredo =
|
||||
predo.toList match {
|
||||
case Nil => Chain.empty
|
||||
case x :: Nil =>
|
||||
Chain.one(x)
|
||||
case l =>
|
||||
mergeMode match
|
||||
case SeqMode =>
|
||||
Chain.one(SeqModel.wrap(l: _*))
|
||||
case ParMode => Chain.one(ParModel.wrap(l: _*))
|
||||
}
|
||||
|
||||
Inline(
|
||||
flattenValues,
|
||||
desugaredPredo
|
||||
)
|
||||
}
|
||||
|
||||
def mergeWith(inline: Inline, mode: MergeMode): Inline = {
|
||||
val left = desugar
|
||||
val right = inline.desugar
|
||||
|
||||
Inline(left.flattenValues ++ right.flattenValues, left.predo ++ right.predo, mode)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO may not be needed there
|
||||
private[inline] object Inline {
|
||||
|
@ -53,7 +53,7 @@ object MakeStructRawInliner extends RawInliner[MakeStructRaw] {
|
||||
name <- Mangler[S].findAndForbidName(raw.structType.name + "_obj")
|
||||
foldedFields <- raw.fields.nonEmptyTraverse(unfold(_))
|
||||
varModel = VarModel(name, raw.baseType)
|
||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
|
||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
|
||||
fields = foldedFields.map(_._1)
|
||||
objCreation <- createObj(fields, varModel)
|
||||
} yield {
|
||||
|
@ -94,7 +94,7 @@ object RawValueInliner extends Logging {
|
||||
}
|
||||
}
|
||||
|
||||
private def toModel[S: Mangler: Exports: Arrows](
|
||||
private[inline] def toModel[S: Mangler: Exports: Arrows](
|
||||
unfoldF: State[S, (ValueModel, Inline)]
|
||||
): State[S, (ValueModel, Option[OpModel.Tree])] =
|
||||
for {
|
||||
|
@ -53,7 +53,7 @@ object ApplyIntoCopyRawInliner extends Logging {
|
||||
name <- Mangler[S].findAndForbidName(value.name + "_obj_copy")
|
||||
foldedFields <- intoCopy.fields.nonEmptyTraverse(unfold(_))
|
||||
varModel = VarModel(name, value.baseType)
|
||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
|
||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
|
||||
fields = foldedFields.map(_._1)
|
||||
objCopy <- copyObj(value, fields, varModel)
|
||||
} yield {
|
||||
|
@ -22,7 +22,7 @@ import aqua.model.{
|
||||
XorModel
|
||||
}
|
||||
import aqua.model.inline.Inline
|
||||
import aqua.model.inline.SeqMode
|
||||
import aqua.model.inline.{ParMode, SeqMode}
|
||||
import aqua.model.inline.RawValueInliner.unfold
|
||||
import aqua.model.inline.state.{Arrows, Exports, Mangler}
|
||||
import aqua.raw.value.{
|
||||
@ -42,6 +42,7 @@ import aqua.types.{ArrayType, CanonStreamType, ScalarType, StreamType, Type}
|
||||
import cats.Eval
|
||||
import cats.data.{Chain, IndexedStateT, State}
|
||||
import cats.syntax.monoid.*
|
||||
import cats.syntax.traverse.*
|
||||
import cats.instances.list.*
|
||||
import scribe.Logging
|
||||
|
||||
@ -140,7 +141,38 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
||||
mergeMode = SeqMode
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Helper for `optimizeProperties`
|
||||
private case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel])
|
||||
|
||||
// Unfold properties that we can process in parallel
|
||||
private def optimizeProperties[S: Mangler: Exports: Arrows](
|
||||
properties: Chain[PropertyRaw]
|
||||
): State[S, (Chain[PropertyRawWithModel], Inline)] = {
|
||||
properties.map {
|
||||
case iir @ IntoIndexRaw(vr, t) =>
|
||||
unfold(vr, propertiesAllowed = false).flatMap {
|
||||
case (vm@VarModel(_, _, _), inline) if vm.properties.nonEmpty =>
|
||||
removeProperties(vm).map { case (vf, inlf) =>
|
||||
PropertyRawWithModel(iir, Option(IntoIndexModel(vf.name, t))) -> Inline(
|
||||
inline.flattenValues ++ inlf.flattenValues,
|
||||
inline.predo ++ inlf.predo,
|
||||
mergeMode = SeqMode
|
||||
)
|
||||
}
|
||||
case (VarModel(name, _, _), inline) =>
|
||||
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(name, t))) -> inline)
|
||||
case (LiteralModel(literal, _), inline) =>
|
||||
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(literal, t))) -> inline)
|
||||
}
|
||||
|
||||
case p => State.pure(PropertyRawWithModel(p, None) -> Inline.empty)
|
||||
}.sequence.map { (propsWithInline: Chain[(PropertyRawWithModel, Inline)]) =>
|
||||
val fullInline = propsWithInline.map(_._2).foldLeft(Inline.empty)(_ |+| _)
|
||||
val props = propsWithInline.map(_._1)
|
||||
(props, fullInline)
|
||||
}
|
||||
}
|
||||
|
||||
private def unfoldProperties[S: Mangler: Exports: Arrows](
|
||||
@ -149,12 +181,17 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
||||
properties: Chain[PropertyRaw],
|
||||
propertiesAllowed: Boolean
|
||||
): State[S, (VarModel, Inline)] = {
|
||||
properties
|
||||
optimizeProperties(properties).flatMap { case (optimizedProps, optimizationInline) =>
|
||||
optimizedProps
|
||||
.foldLeft[State[S, (VarModel, Inline)]](
|
||||
State.pure((vm, prevInline))
|
||||
State.pure((vm, prevInline.mergeWith(optimizationInline, SeqMode)))
|
||||
) { case (state, property) =>
|
||||
state.flatMap { case (vm, leftInline) =>
|
||||
unfoldProperty(vm, property).flatMap {
|
||||
property match {
|
||||
case PropertyRawWithModel(_, Some(model)) =>
|
||||
State.pure(vm.copy(properties = vm.properties :+ model) -> leftInline)
|
||||
case PropertyRawWithModel(raw, _) =>
|
||||
unfoldProperty(vm, raw).flatMap {
|
||||
case (v, i) if !propertiesAllowed && v.properties.nonEmpty =>
|
||||
removeProperties(v).map { case (vf, inlf) =>
|
||||
vf -> Inline(
|
||||
@ -175,6 +212,8 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def unfoldRawWithProperties[S: Mangler: Exports: Arrows](
|
||||
raw: ValueRaw,
|
||||
@ -184,7 +223,7 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
||||
((raw, properties.headOption) match {
|
||||
case (vr @ VarRaw(_, st @ StreamType(_)), Some(IntoIndexRaw(idx, _))) =>
|
||||
unfold(vr).flatMap {
|
||||
case (vm @ VarModel(nameVM, _, _), inl) =>
|
||||
case (VarModel(nameVM, _, _), inl) =>
|
||||
val gateRaw = ApplyGateRaw(nameVM, st, idx)
|
||||
unfold(gateRaw).flatMap {
|
||||
case (gateResVal: VarModel, gateResInline) =>
|
||||
|
@ -0,0 +1,71 @@
|
||||
package aqua.model.inline
|
||||
|
||||
import aqua.model.*
|
||||
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
|
||||
import aqua.model.inline.state.InliningState
|
||||
import aqua.raw.ops.*
|
||||
import aqua.raw.value.*
|
||||
import aqua.types.*
|
||||
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
|
||||
import cats.syntax.show.*
|
||||
import org.scalatest.flatspec.AnyFlatSpec
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
|
||||
class CopyInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
|
||||
"copy inliner" should "unfold values in parallel" in {
|
||||
|
||||
val structType = StructType(
|
||||
"struct_type",
|
||||
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
|
||||
)
|
||||
|
||||
val arrType = ArrayType(ScalarType.string)
|
||||
val length = FunctorRaw("length", ScalarType.u32)
|
||||
val lengthValue = VarRaw("l", arrType).withProperty(length)
|
||||
|
||||
val getField = CallArrowRaw(None, "get_field", Nil, ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)), Option(LiteralRaw("\"serv\"", ScalarType.string)))
|
||||
|
||||
val copyRaw =
|
||||
IntoCopyRaw(structType, NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField))
|
||||
val varName = "some_struct"
|
||||
val varRaw = VarRaw(varName, structType).withProperty(copyRaw)
|
||||
|
||||
val (model, tree) =
|
||||
RawValueInliner.valueToModel[InliningState](varRaw, false).run(InliningState()).value._2
|
||||
|
||||
val result = VarModel(varName + "_obj_copy", structType)
|
||||
model shouldBe result
|
||||
|
||||
val lengthModel = FunctorModel("length", ScalarType.u32)
|
||||
|
||||
tree.get.equalsOrShowDiff(
|
||||
SeqModel.wrap(
|
||||
ParModel.wrap(
|
||||
SeqModel.wrap(
|
||||
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
|
||||
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf,
|
||||
),
|
||||
CallServiceModel(
|
||||
"serv",
|
||||
"get_field",
|
||||
Nil,
|
||||
VarModel("get_field", ScalarType.string)
|
||||
).leaf
|
||||
),
|
||||
CallServiceModel(
|
||||
"json",
|
||||
"puts",
|
||||
VarModel(varName, structType) :: LiteralModel.fromRaw(
|
||||
LiteralRaw.quote("field1")
|
||||
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
|
||||
LiteralRaw.quote("field2")
|
||||
) :: VarModel("get_field", ScalarType.string) :: Nil,
|
||||
result
|
||||
).leaf
|
||||
)
|
||||
) shouldBe true
|
||||
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,76 @@
|
||||
package aqua.model.inline
|
||||
|
||||
import aqua.model.*
|
||||
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
|
||||
import aqua.model.inline.state.InliningState
|
||||
import aqua.raw.ops.*
|
||||
import aqua.raw.value.*
|
||||
import aqua.types.*
|
||||
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
|
||||
import cats.syntax.show.*
|
||||
import org.scalatest.flatspec.AnyFlatSpec
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
|
||||
class MakeStructInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
|
||||
"make struct inliner" should "unfold values in parallel" in {
|
||||
|
||||
val structType = StructType(
|
||||
"struct_type",
|
||||
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
|
||||
)
|
||||
|
||||
val arrType = ArrayType(ScalarType.string)
|
||||
val length = FunctorRaw("length", ScalarType.u32)
|
||||
val lengthValue = VarRaw("l", arrType).withProperty(length)
|
||||
|
||||
val getField = CallArrowRaw(
|
||||
None,
|
||||
"get_field",
|
||||
Nil,
|
||||
ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)),
|
||||
Option(LiteralRaw("\"serv\"", ScalarType.string))
|
||||
)
|
||||
|
||||
val makeStruct =
|
||||
MakeStructRaw(NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField), structType)
|
||||
val varName = structType.name
|
||||
|
||||
val (model, tree) =
|
||||
RawValueInliner.valueToModel[InliningState](makeStruct, false).run(InliningState()).value._2
|
||||
|
||||
val result = VarModel(varName + "_obj", structType)
|
||||
model shouldBe result
|
||||
|
||||
val lengthModel = FunctorModel("length", ScalarType.u32)
|
||||
|
||||
tree.get.equalsOrShowDiff(
|
||||
SeqModel.wrap(
|
||||
ParModel.wrap(
|
||||
SeqModel.wrap(
|
||||
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
|
||||
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf
|
||||
),
|
||||
CallServiceModel(
|
||||
"serv",
|
||||
"get_field",
|
||||
Nil,
|
||||
VarModel("get_field", ScalarType.string)
|
||||
).leaf
|
||||
),
|
||||
CallServiceModel(
|
||||
"json",
|
||||
"obj",
|
||||
LiteralModel.fromRaw(
|
||||
LiteralRaw.quote("field1")
|
||||
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
|
||||
LiteralRaw.quote("field2")
|
||||
) :: VarModel("get_field", ScalarType.string) :: Nil,
|
||||
result
|
||||
).leaf
|
||||
)
|
||||
) shouldBe true
|
||||
|
||||
}
|
||||
|
||||
}
|
@ -17,6 +17,7 @@ import aqua.raw.value.{ApplyPropertyRaw, FunctorRaw, IntoIndexRaw, LiteralRaw, V
|
||||
import aqua.types.*
|
||||
import cats.data.NonEmptyMap
|
||||
import cats.data.Chain
|
||||
import cats.syntax.show.*
|
||||
import org.scalatest.flatspec.AnyFlatSpec
|
||||
import org.scalatest.matchers.should.Matchers
|
||||
|
||||
@ -207,7 +208,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
resTree.isEmpty should be(false)
|
||||
|
||||
resTree.get.equalsOrShowDiff(
|
||||
SeqModel.wrap(
|
||||
ParModel.wrap(
|
||||
SeqModel.wrap(
|
||||
FlattenModel(
|
||||
VarModel(
|
||||
@ -284,7 +285,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
resTree.isEmpty should be(false)
|
||||
|
||||
resTree.get.equalsOrShowDiff(
|
||||
SeqModel.wrap(
|
||||
ParModel.wrap(
|
||||
FlattenModel(
|
||||
VarModel(
|
||||
"ys",
|
||||
@ -325,7 +326,6 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
)
|
||||
)
|
||||
)
|
||||
println(resTree)
|
||||
}
|
||||
|
||||
"raw value inliner" should "desugarize stream with length" in {
|
||||
@ -367,7 +367,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
||||
resTree.isEmpty should be(false)
|
||||
|
||||
resTree.get.equalsOrShowDiff(
|
||||
SeqModel.wrap(
|
||||
ParModel.wrap(
|
||||
FlattenModel(
|
||||
VarModel(
|
||||
"ys",
|
||||
|
Loading…
x
Reference in New Issue
Block a user