mirror of
https://github.com/fluencelabs/aqua.git
synced 2025-03-15 11:40:50 +00:00
perf: Unfold variables in parallel where it is possible (fixes LNG-109 ) (#656)
This commit is contained in:
parent
50c1da3b2f
commit
439f2cde03
@ -13,7 +13,6 @@ object SeqMode extends MergeMode
|
|||||||
object ParMode extends MergeMode
|
object ParMode extends MergeMode
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
|
||||||
* @param flattenValues values that need to be resolved before `predo`.
|
* @param flattenValues values that need to be resolved before `predo`.
|
||||||
* ListMap for keeping order of values (mostly for debugging purposes)
|
* ListMap for keeping order of values (mostly for debugging purposes)
|
||||||
* @param predo operations tree
|
* @param predo operations tree
|
||||||
@ -23,7 +22,34 @@ private[inline] case class Inline(
|
|||||||
flattenValues: ListMap[String, ValueRaw] = ListMap.empty,
|
flattenValues: ListMap[String, ValueRaw] = ListMap.empty,
|
||||||
predo: Chain[OpModel.Tree] = Chain.empty,
|
predo: Chain[OpModel.Tree] = Chain.empty,
|
||||||
mergeMode: MergeMode = ParMode
|
mergeMode: MergeMode = ParMode
|
||||||
)
|
) {
|
||||||
|
|
||||||
|
def desugar: Inline = {
|
||||||
|
val desugaredPredo =
|
||||||
|
predo.toList match {
|
||||||
|
case Nil => Chain.empty
|
||||||
|
case x :: Nil =>
|
||||||
|
Chain.one(x)
|
||||||
|
case l =>
|
||||||
|
mergeMode match
|
||||||
|
case SeqMode =>
|
||||||
|
Chain.one(SeqModel.wrap(l: _*))
|
||||||
|
case ParMode => Chain.one(ParModel.wrap(l: _*))
|
||||||
|
}
|
||||||
|
|
||||||
|
Inline(
|
||||||
|
flattenValues,
|
||||||
|
desugaredPredo
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
def mergeWith(inline: Inline, mode: MergeMode): Inline = {
|
||||||
|
val left = desugar
|
||||||
|
val right = inline.desugar
|
||||||
|
|
||||||
|
Inline(left.flattenValues ++ right.flattenValues, left.predo ++ right.predo, mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TODO may not be needed there
|
// TODO may not be needed there
|
||||||
private[inline] object Inline {
|
private[inline] object Inline {
|
||||||
|
@ -53,7 +53,7 @@ object MakeStructRawInliner extends RawInliner[MakeStructRaw] {
|
|||||||
name <- Mangler[S].findAndForbidName(raw.structType.name + "_obj")
|
name <- Mangler[S].findAndForbidName(raw.structType.name + "_obj")
|
||||||
foldedFields <- raw.fields.nonEmptyTraverse(unfold(_))
|
foldedFields <- raw.fields.nonEmptyTraverse(unfold(_))
|
||||||
varModel = VarModel(name, raw.baseType)
|
varModel = VarModel(name, raw.baseType)
|
||||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
|
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
|
||||||
fields = foldedFields.map(_._1)
|
fields = foldedFields.map(_._1)
|
||||||
objCreation <- createObj(fields, varModel)
|
objCreation <- createObj(fields, varModel)
|
||||||
} yield {
|
} yield {
|
||||||
|
@ -94,7 +94,7 @@ object RawValueInliner extends Logging {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def toModel[S: Mangler: Exports: Arrows](
|
private[inline] def toModel[S: Mangler: Exports: Arrows](
|
||||||
unfoldF: State[S, (ValueModel, Inline)]
|
unfoldF: State[S, (ValueModel, Inline)]
|
||||||
): State[S, (ValueModel, Option[OpModel.Tree])] =
|
): State[S, (ValueModel, Option[OpModel.Tree])] =
|
||||||
for {
|
for {
|
||||||
|
@ -53,7 +53,7 @@ object ApplyIntoCopyRawInliner extends Logging {
|
|||||||
name <- Mangler[S].findAndForbidName(value.name + "_obj_copy")
|
name <- Mangler[S].findAndForbidName(value.name + "_obj_copy")
|
||||||
foldedFields <- intoCopy.fields.nonEmptyTraverse(unfold(_))
|
foldedFields <- intoCopy.fields.nonEmptyTraverse(unfold(_))
|
||||||
varModel = VarModel(name, value.baseType)
|
varModel = VarModel(name, value.baseType)
|
||||||
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
|
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
|
||||||
fields = foldedFields.map(_._1)
|
fields = foldedFields.map(_._1)
|
||||||
objCopy <- copyObj(value, fields, varModel)
|
objCopy <- copyObj(value, fields, varModel)
|
||||||
} yield {
|
} yield {
|
||||||
|
@ -22,7 +22,7 @@ import aqua.model.{
|
|||||||
XorModel
|
XorModel
|
||||||
}
|
}
|
||||||
import aqua.model.inline.Inline
|
import aqua.model.inline.Inline
|
||||||
import aqua.model.inline.SeqMode
|
import aqua.model.inline.{ParMode, SeqMode}
|
||||||
import aqua.model.inline.RawValueInliner.unfold
|
import aqua.model.inline.RawValueInliner.unfold
|
||||||
import aqua.model.inline.state.{Arrows, Exports, Mangler}
|
import aqua.model.inline.state.{Arrows, Exports, Mangler}
|
||||||
import aqua.raw.value.{
|
import aqua.raw.value.{
|
||||||
@ -42,6 +42,7 @@ import aqua.types.{ArrayType, CanonStreamType, ScalarType, StreamType, Type}
|
|||||||
import cats.Eval
|
import cats.Eval
|
||||||
import cats.data.{Chain, IndexedStateT, State}
|
import cats.data.{Chain, IndexedStateT, State}
|
||||||
import cats.syntax.monoid.*
|
import cats.syntax.monoid.*
|
||||||
|
import cats.syntax.traverse.*
|
||||||
import cats.instances.list.*
|
import cats.instances.list.*
|
||||||
import scribe.Logging
|
import scribe.Logging
|
||||||
|
|
||||||
@ -140,40 +141,78 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
|||||||
mergeMode = SeqMode
|
mergeMode = SeqMode
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper for `optimizeProperties`
|
||||||
|
private case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel])
|
||||||
|
|
||||||
|
// Unfold properties that we can process in parallel
|
||||||
|
private def optimizeProperties[S: Mangler: Exports: Arrows](
|
||||||
|
properties: Chain[PropertyRaw]
|
||||||
|
): State[S, (Chain[PropertyRawWithModel], Inline)] = {
|
||||||
|
properties.map {
|
||||||
|
case iir @ IntoIndexRaw(vr, t) =>
|
||||||
|
unfold(vr, propertiesAllowed = false).flatMap {
|
||||||
|
case (vm@VarModel(_, _, _), inline) if vm.properties.nonEmpty =>
|
||||||
|
removeProperties(vm).map { case (vf, inlf) =>
|
||||||
|
PropertyRawWithModel(iir, Option(IntoIndexModel(vf.name, t))) -> Inline(
|
||||||
|
inline.flattenValues ++ inlf.flattenValues,
|
||||||
|
inline.predo ++ inlf.predo,
|
||||||
|
mergeMode = SeqMode
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case (VarModel(name, _, _), inline) =>
|
||||||
|
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(name, t))) -> inline)
|
||||||
|
case (LiteralModel(literal, _), inline) =>
|
||||||
|
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(literal, t))) -> inline)
|
||||||
|
}
|
||||||
|
|
||||||
|
case p => State.pure(PropertyRawWithModel(p, None) -> Inline.empty)
|
||||||
|
}.sequence.map { (propsWithInline: Chain[(PropertyRawWithModel, Inline)]) =>
|
||||||
|
val fullInline = propsWithInline.map(_._2).foldLeft(Inline.empty)(_ |+| _)
|
||||||
|
val props = propsWithInline.map(_._1)
|
||||||
|
(props, fullInline)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private def unfoldProperties[S: Mangler: Exports: Arrows](
|
private def unfoldProperties[S: Mangler: Exports: Arrows](
|
||||||
prevInline: Inline,
|
prevInline: Inline,
|
||||||
vm: VarModel,
|
vm: VarModel,
|
||||||
properties: Chain[PropertyRaw],
|
properties: Chain[PropertyRaw],
|
||||||
propertiesAllowed: Boolean
|
propertiesAllowed: Boolean
|
||||||
): State[S, (VarModel, Inline)] = {
|
): State[S, (VarModel, Inline)] = {
|
||||||
properties
|
optimizeProperties(properties).flatMap { case (optimizedProps, optimizationInline) =>
|
||||||
.foldLeft[State[S, (VarModel, Inline)]](
|
optimizedProps
|
||||||
State.pure((vm, prevInline))
|
.foldLeft[State[S, (VarModel, Inline)]](
|
||||||
) { case (state, property) =>
|
State.pure((vm, prevInline.mergeWith(optimizationInline, SeqMode)))
|
||||||
state.flatMap { case (vm, leftInline) =>
|
) { case (state, property) =>
|
||||||
unfoldProperty(vm, property).flatMap {
|
state.flatMap { case (vm, leftInline) =>
|
||||||
case (v, i) if !propertiesAllowed && v.properties.nonEmpty =>
|
property match {
|
||||||
removeProperties(v).map { case (vf, inlf) =>
|
case PropertyRawWithModel(_, Some(model)) =>
|
||||||
vf -> Inline(
|
State.pure(vm.copy(properties = vm.properties :+ model) -> leftInline)
|
||||||
leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues,
|
case PropertyRawWithModel(raw, _) =>
|
||||||
leftInline.predo ++ i.predo ++ inlf.predo,
|
unfoldProperty(vm, raw).flatMap {
|
||||||
mergeMode = SeqMode
|
case (v, i) if !propertiesAllowed && v.properties.nonEmpty =>
|
||||||
)
|
removeProperties(v).map { case (vf, inlf) =>
|
||||||
}
|
vf -> Inline(
|
||||||
case (v, i) =>
|
leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues,
|
||||||
State.pure(
|
leftInline.predo ++ i.predo ++ inlf.predo,
|
||||||
v -> Inline(
|
mergeMode = SeqMode
|
||||||
leftInline.flattenValues ++ i.flattenValues,
|
)
|
||||||
leftInline.predo ++ i.predo,
|
}
|
||||||
mergeMode = SeqMode
|
case (v, i) =>
|
||||||
)
|
State.pure(
|
||||||
)
|
v -> Inline(
|
||||||
|
leftInline.flattenValues ++ i.flattenValues,
|
||||||
|
leftInline.predo ++ i.predo,
|
||||||
|
mergeMode = SeqMode
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private def unfoldRawWithProperties[S: Mangler: Exports: Arrows](
|
private def unfoldRawWithProperties[S: Mangler: Exports: Arrows](
|
||||||
@ -184,7 +223,7 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
|
|||||||
((raw, properties.headOption) match {
|
((raw, properties.headOption) match {
|
||||||
case (vr @ VarRaw(_, st @ StreamType(_)), Some(IntoIndexRaw(idx, _))) =>
|
case (vr @ VarRaw(_, st @ StreamType(_)), Some(IntoIndexRaw(idx, _))) =>
|
||||||
unfold(vr).flatMap {
|
unfold(vr).flatMap {
|
||||||
case (vm @ VarModel(nameVM, _, _), inl) =>
|
case (VarModel(nameVM, _, _), inl) =>
|
||||||
val gateRaw = ApplyGateRaw(nameVM, st, idx)
|
val gateRaw = ApplyGateRaw(nameVM, st, idx)
|
||||||
unfold(gateRaw).flatMap {
|
unfold(gateRaw).flatMap {
|
||||||
case (gateResVal: VarModel, gateResInline) =>
|
case (gateResVal: VarModel, gateResInline) =>
|
||||||
|
@ -0,0 +1,71 @@
|
|||||||
|
package aqua.model.inline
|
||||||
|
|
||||||
|
import aqua.model.*
|
||||||
|
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
|
||||||
|
import aqua.model.inline.state.InliningState
|
||||||
|
import aqua.raw.ops.*
|
||||||
|
import aqua.raw.value.*
|
||||||
|
import aqua.types.*
|
||||||
|
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
|
||||||
|
import cats.syntax.show.*
|
||||||
|
import org.scalatest.flatspec.AnyFlatSpec
|
||||||
|
import org.scalatest.matchers.should.Matchers
|
||||||
|
|
||||||
|
class CopyInlinerSpec extends AnyFlatSpec with Matchers {
|
||||||
|
|
||||||
|
"copy inliner" should "unfold values in parallel" in {
|
||||||
|
|
||||||
|
val structType = StructType(
|
||||||
|
"struct_type",
|
||||||
|
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
|
||||||
|
)
|
||||||
|
|
||||||
|
val arrType = ArrayType(ScalarType.string)
|
||||||
|
val length = FunctorRaw("length", ScalarType.u32)
|
||||||
|
val lengthValue = VarRaw("l", arrType).withProperty(length)
|
||||||
|
|
||||||
|
val getField = CallArrowRaw(None, "get_field", Nil, ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)), Option(LiteralRaw("\"serv\"", ScalarType.string)))
|
||||||
|
|
||||||
|
val copyRaw =
|
||||||
|
IntoCopyRaw(structType, NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField))
|
||||||
|
val varName = "some_struct"
|
||||||
|
val varRaw = VarRaw(varName, structType).withProperty(copyRaw)
|
||||||
|
|
||||||
|
val (model, tree) =
|
||||||
|
RawValueInliner.valueToModel[InliningState](varRaw, false).run(InliningState()).value._2
|
||||||
|
|
||||||
|
val result = VarModel(varName + "_obj_copy", structType)
|
||||||
|
model shouldBe result
|
||||||
|
|
||||||
|
val lengthModel = FunctorModel("length", ScalarType.u32)
|
||||||
|
|
||||||
|
tree.get.equalsOrShowDiff(
|
||||||
|
SeqModel.wrap(
|
||||||
|
ParModel.wrap(
|
||||||
|
SeqModel.wrap(
|
||||||
|
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
|
||||||
|
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf,
|
||||||
|
),
|
||||||
|
CallServiceModel(
|
||||||
|
"serv",
|
||||||
|
"get_field",
|
||||||
|
Nil,
|
||||||
|
VarModel("get_field", ScalarType.string)
|
||||||
|
).leaf
|
||||||
|
),
|
||||||
|
CallServiceModel(
|
||||||
|
"json",
|
||||||
|
"puts",
|
||||||
|
VarModel(varName, structType) :: LiteralModel.fromRaw(
|
||||||
|
LiteralRaw.quote("field1")
|
||||||
|
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
|
||||||
|
LiteralRaw.quote("field2")
|
||||||
|
) :: VarModel("get_field", ScalarType.string) :: Nil,
|
||||||
|
result
|
||||||
|
).leaf
|
||||||
|
)
|
||||||
|
) shouldBe true
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,76 @@
|
|||||||
|
package aqua.model.inline
|
||||||
|
|
||||||
|
import aqua.model.*
|
||||||
|
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
|
||||||
|
import aqua.model.inline.state.InliningState
|
||||||
|
import aqua.raw.ops.*
|
||||||
|
import aqua.raw.value.*
|
||||||
|
import aqua.types.*
|
||||||
|
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
|
||||||
|
import cats.syntax.show.*
|
||||||
|
import org.scalatest.flatspec.AnyFlatSpec
|
||||||
|
import org.scalatest.matchers.should.Matchers
|
||||||
|
|
||||||
|
class MakeStructInlinerSpec extends AnyFlatSpec with Matchers {
|
||||||
|
|
||||||
|
"make struct inliner" should "unfold values in parallel" in {
|
||||||
|
|
||||||
|
val structType = StructType(
|
||||||
|
"struct_type",
|
||||||
|
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
|
||||||
|
)
|
||||||
|
|
||||||
|
val arrType = ArrayType(ScalarType.string)
|
||||||
|
val length = FunctorRaw("length", ScalarType.u32)
|
||||||
|
val lengthValue = VarRaw("l", arrType).withProperty(length)
|
||||||
|
|
||||||
|
val getField = CallArrowRaw(
|
||||||
|
None,
|
||||||
|
"get_field",
|
||||||
|
Nil,
|
||||||
|
ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)),
|
||||||
|
Option(LiteralRaw("\"serv\"", ScalarType.string))
|
||||||
|
)
|
||||||
|
|
||||||
|
val makeStruct =
|
||||||
|
MakeStructRaw(NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField), structType)
|
||||||
|
val varName = structType.name
|
||||||
|
|
||||||
|
val (model, tree) =
|
||||||
|
RawValueInliner.valueToModel[InliningState](makeStruct, false).run(InliningState()).value._2
|
||||||
|
|
||||||
|
val result = VarModel(varName + "_obj", structType)
|
||||||
|
model shouldBe result
|
||||||
|
|
||||||
|
val lengthModel = FunctorModel("length", ScalarType.u32)
|
||||||
|
|
||||||
|
tree.get.equalsOrShowDiff(
|
||||||
|
SeqModel.wrap(
|
||||||
|
ParModel.wrap(
|
||||||
|
SeqModel.wrap(
|
||||||
|
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
|
||||||
|
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf
|
||||||
|
),
|
||||||
|
CallServiceModel(
|
||||||
|
"serv",
|
||||||
|
"get_field",
|
||||||
|
Nil,
|
||||||
|
VarModel("get_field", ScalarType.string)
|
||||||
|
).leaf
|
||||||
|
),
|
||||||
|
CallServiceModel(
|
||||||
|
"json",
|
||||||
|
"obj",
|
||||||
|
LiteralModel.fromRaw(
|
||||||
|
LiteralRaw.quote("field1")
|
||||||
|
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
|
||||||
|
LiteralRaw.quote("field2")
|
||||||
|
) :: VarModel("get_field", ScalarType.string) :: Nil,
|
||||||
|
result
|
||||||
|
).leaf
|
||||||
|
)
|
||||||
|
) shouldBe true
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -17,6 +17,7 @@ import aqua.raw.value.{ApplyPropertyRaw, FunctorRaw, IntoIndexRaw, LiteralRaw, V
|
|||||||
import aqua.types.*
|
import aqua.types.*
|
||||||
import cats.data.NonEmptyMap
|
import cats.data.NonEmptyMap
|
||||||
import cats.data.Chain
|
import cats.data.Chain
|
||||||
|
import cats.syntax.show.*
|
||||||
import org.scalatest.flatspec.AnyFlatSpec
|
import org.scalatest.flatspec.AnyFlatSpec
|
||||||
import org.scalatest.matchers.should.Matchers
|
import org.scalatest.matchers.should.Matchers
|
||||||
|
|
||||||
@ -207,7 +208,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
|||||||
resTree.isEmpty should be(false)
|
resTree.isEmpty should be(false)
|
||||||
|
|
||||||
resTree.get.equalsOrShowDiff(
|
resTree.get.equalsOrShowDiff(
|
||||||
SeqModel.wrap(
|
ParModel.wrap(
|
||||||
SeqModel.wrap(
|
SeqModel.wrap(
|
||||||
FlattenModel(
|
FlattenModel(
|
||||||
VarModel(
|
VarModel(
|
||||||
@ -284,7 +285,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
|||||||
resTree.isEmpty should be(false)
|
resTree.isEmpty should be(false)
|
||||||
|
|
||||||
resTree.get.equalsOrShowDiff(
|
resTree.get.equalsOrShowDiff(
|
||||||
SeqModel.wrap(
|
ParModel.wrap(
|
||||||
FlattenModel(
|
FlattenModel(
|
||||||
VarModel(
|
VarModel(
|
||||||
"ys",
|
"ys",
|
||||||
@ -325,7 +326,6 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
println(resTree)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
"raw value inliner" should "desugarize stream with length" in {
|
"raw value inliner" should "desugarize stream with length" in {
|
||||||
@ -367,7 +367,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
|
|||||||
resTree.isEmpty should be(false)
|
resTree.isEmpty should be(false)
|
||||||
|
|
||||||
resTree.get.equalsOrShowDiff(
|
resTree.get.equalsOrShowDiff(
|
||||||
SeqModel.wrap(
|
ParModel.wrap(
|
||||||
FlattenModel(
|
FlattenModel(
|
||||||
VarModel(
|
VarModel(
|
||||||
"ys",
|
"ys",
|
||||||
|
Loading…
x
Reference in New Issue
Block a user