fluencelabs/aqua
1
0
Fork 0
mirror of https://github.com/fluencelabs/aqua.git synced 2025-03-15 11:40:50 +00:00
Code Issues Projects Releases Wiki Activity

perf: Unfold variables in parallel where it is possible (fixes LNG-109 ) (#656)

Browse Source
This commit is contained in:
Dima 2023-02-01 14:50:43 +07:00 committed by GitHub
parent 50c1da3b2f
commit 439f2cde03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 247 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
model/inline/src/main/scala/aqua/model/inline/Inline.scala
View File

@ -13,7 +13,6 @@ object SeqMode extends MergeMode
object ParMode extends MergeMode
/**
*
* @param flattenValues values that need to be resolved before `predo`.
* ListMap for keeping order of values (mostly for debugging purposes)
* @param predo operations tree
@ -23,7 +22,34 @@ private[inline] case class Inline(
flattenValues: ListMap[String, ValueRaw] = ListMap.empty,
predo: Chain[OpModel.Tree] = Chain.empty,
mergeMode: MergeMode = ParMode
)
) {
def desugar: Inline = {
val desugaredPredo =
predo.toList match {
case Nil => Chain.empty
case x :: Nil =>
Chain.one(x)
case l =>
mergeMode match
case SeqMode =>
Chain.one(SeqModel.wrap(l: _*))
case ParMode => Chain.one(ParModel.wrap(l: _*))
}
Inline(
flattenValues,
desugaredPredo
)
}
def mergeWith(inline: Inline, mode: MergeMode): Inline = {
val left = desugar
val right = inline.desugar
Inline(left.flattenValues ++ right.flattenValues, left.predo ++ right.predo, mode)
}
}
// TODO may not be needed there
private[inline] object Inline {

2
model/inline/src/main/scala/aqua/model/inline/MakeStructRawInliner.scala
View File

@ -53,7 +53,7 @@ object MakeStructRawInliner extends RawInliner[MakeStructRaw] {
name <- Mangler[S].findAndForbidName(raw.structType.name + "_obj")
foldedFields <- raw.fields.nonEmptyTraverse(unfold(_))
varModel = VarModel(name, raw.baseType)
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
fields = foldedFields.map(_._1)
objCreation <- createObj(fields, varModel)
} yield {

2
model/inline/src/main/scala/aqua/model/inline/RawValueInliner.scala
View File

@ -94,7 +94,7 @@ object RawValueInliner extends Logging {
}
}
private def toModel[S: Mangler: Exports: Arrows](
private[inline] def toModel[S: Mangler: Exports: Arrows](
unfoldF: State[S, (ValueModel, Inline)]
): State[S, (ValueModel, Option[OpModel.Tree])] =
for {

2
model/inline/src/main/scala/aqua/model/inline/raw/ApplyIntoCopyRawInliner.scala
View File

@ -53,7 +53,7 @@ object ApplyIntoCopyRawInliner extends Logging {
name <- Mangler[S].findAndForbidName(value.name + "_obj_copy")
foldedFields <- intoCopy.fields.nonEmptyTraverse(unfold(_))
varModel = VarModel(name, value.baseType)
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _)
valsInline = foldedFields.toSortedMap.values.map(_._2).fold(Inline.empty)(_ |+| _).desugar
fields = foldedFields.map(_._1)
objCopy <- copyObj(value, fields, varModel)
} yield {

91
model/inline/src/main/scala/aqua/model/inline/raw/ApplyPropertiesRawInliner.scala
View File

@ -22,7 +22,7 @@ import aqua.model.{
XorModel
}
import aqua.model.inline.Inline
import aqua.model.inline.SeqMode
import aqua.model.inline.{ParMode, SeqMode}
import aqua.model.inline.RawValueInliner.unfold
import aqua.model.inline.state.{Arrows, Exports, Mangler}
import aqua.raw.value.{
@ -42,6 +42,7 @@ import aqua.types.{ArrayType, CanonStreamType, ScalarType, StreamType, Type}
import cats.Eval
import cats.data.{Chain, IndexedStateT, State}
import cats.syntax.monoid.*
import cats.syntax.traverse.*
import cats.instances.list.*
import scribe.Logging
@ -140,40 +141,78 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
mergeMode = SeqMode
)
}
}
// Helper for `optimizeProperties`
private case class PropertyRawWithModel(raw: PropertyRaw, model: Option[PropertyModel])
// Unfold properties that we can process in parallel
private def optimizeProperties[S: Mangler: Exports: Arrows](
properties: Chain[PropertyRaw]
): State[S, (Chain[PropertyRawWithModel], Inline)] = {
properties.map {
case iir @ IntoIndexRaw(vr, t) =>
unfold(vr, propertiesAllowed = false).flatMap {
case (vm@VarModel(_, _, _), inline) if vm.properties.nonEmpty =>
removeProperties(vm).map { case (vf, inlf) =>
PropertyRawWithModel(iir, Option(IntoIndexModel(vf.name, t))) -> Inline(
inline.flattenValues ++ inlf.flattenValues,
inline.predo ++ inlf.predo,
mergeMode = SeqMode
)
}
case (VarModel(name, _, _), inline) =>
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(name, t))) -> inline)
case (LiteralModel(literal, _), inline) =>
State.pure(PropertyRawWithModel(iir, Option(IntoIndexModel(literal, t))) -> inline)
}
case p => State.pure(PropertyRawWithModel(p, None) -> Inline.empty)
}.sequence.map { (propsWithInline: Chain[(PropertyRawWithModel, Inline)]) =>
val fullInline = propsWithInline.map(_._2).foldLeft(Inline.empty)(_ |+| _)
val props = propsWithInline.map(_._1)
(props, fullInline)
}
}
private def unfoldProperties[S: Mangler: Exports: Arrows](
prevInline: Inline,
vm: VarModel,
properties: Chain[PropertyRaw],
propertiesAllowed: Boolean
): State[S, (VarModel, Inline)] = {
properties
.foldLeft[State[S, (VarModel, Inline)]](
State.pure((vm, prevInline))
) { case (state, property) =>
state.flatMap { case (vm, leftInline) =>
unfoldProperty(vm, property).flatMap {
case (v, i) if !propertiesAllowed && v.properties.nonEmpty =>
removeProperties(v).map { case (vf, inlf) =>
vf -> Inline(
leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues,
leftInline.predo ++ i.predo ++ inlf.predo,
mergeMode = SeqMode
)
}
case (v, i) =>
State.pure(
v -> Inline(
leftInline.flattenValues ++ i.flattenValues,
leftInline.predo ++ i.predo,
mergeMode = SeqMode
)
)
optimizeProperties(properties).flatMap { case (optimizedProps, optimizationInline) =>
optimizedProps
.foldLeft[State[S, (VarModel, Inline)]](
State.pure((vm, prevInline.mergeWith(optimizationInline, SeqMode)))
) { case (state, property) =>
state.flatMap { case (vm, leftInline) =>
property match {
case PropertyRawWithModel(_, Some(model)) =>
State.pure(vm.copy(properties = vm.properties :+ model) -> leftInline)
case PropertyRawWithModel(raw, _) =>
unfoldProperty(vm, raw).flatMap {
case (v, i) if !propertiesAllowed && v.properties.nonEmpty =>
removeProperties(v).map { case (vf, inlf) =>
vf -> Inline(
leftInline.flattenValues ++ i.flattenValues ++ inlf.flattenValues,
leftInline.predo ++ i.predo ++ inlf.predo,
mergeMode = SeqMode
)
}
case (v, i) =>
State.pure(
v -> Inline(
leftInline.flattenValues ++ i.flattenValues,
leftInline.predo ++ i.predo,
mergeMode = SeqMode
)
)
}
}
}
}
}
}
}
private def unfoldRawWithProperties[S: Mangler: Exports: Arrows](
@ -184,7 +223,7 @@ object ApplyPropertiesRawInliner extends RawInliner[ApplyPropertyRaw] with Loggi
((raw, properties.headOption) match {
case (vr @ VarRaw(_, st @ StreamType(_)), Some(IntoIndexRaw(idx, _))) =>
unfold(vr).flatMap {
case (vm @ VarModel(nameVM, _, _), inl) =>
case (VarModel(nameVM, _, _), inl) =>
val gateRaw = ApplyGateRaw(nameVM, st, idx)
unfold(gateRaw).flatMap {
case (gateResVal: VarModel, gateResInline) =>

71
model/inline/src/test/scala/aqua/model/inline/CopyInlinerSpec.scala Normal file
View File

@ -0,0 +1,71 @@
package aqua.model.inline
import aqua.model.*
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
import aqua.model.inline.state.InliningState
import aqua.raw.ops.*
import aqua.raw.value.*
import aqua.types.*
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
import cats.syntax.show.*
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
class CopyInlinerSpec extends AnyFlatSpec with Matchers {
"copy inliner" should "unfold values in parallel" in {
val structType = StructType(
"struct_type",
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
)
val arrType = ArrayType(ScalarType.string)
val length = FunctorRaw("length", ScalarType.u32)
val lengthValue = VarRaw("l", arrType).withProperty(length)
val getField = CallArrowRaw(None, "get_field", Nil, ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)), Option(LiteralRaw("\"serv\"", ScalarType.string)))
val copyRaw =
IntoCopyRaw(structType, NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField))
val varName = "some_struct"
val varRaw = VarRaw(varName, structType).withProperty(copyRaw)
val (model, tree) =
RawValueInliner.valueToModel[InliningState](varRaw, false).run(InliningState()).value._2
val result = VarModel(varName + "_obj_copy", structType)
model shouldBe result
val lengthModel = FunctorModel("length", ScalarType.u32)
tree.get.equalsOrShowDiff(
SeqModel.wrap(
ParModel.wrap(
SeqModel.wrap(
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf,
),
CallServiceModel(
"serv",
"get_field",
Nil,
VarModel("get_field", ScalarType.string)
).leaf
),
CallServiceModel(
"json",
"puts",
VarModel(varName, structType) :: LiteralModel.fromRaw(
LiteralRaw.quote("field1")
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
LiteralRaw.quote("field2")
) :: VarModel("get_field", ScalarType.string) :: Nil,
result
).leaf
)
) shouldBe true
}
}

76
model/inline/src/test/scala/aqua/model/inline/MakeStructInlinerSpec.scala Normal file
View File

@ -0,0 +1,76 @@
package aqua.model.inline
import aqua.model.*
import aqua.model.inline.raw.ApplyIntoCopyRawInliner
import aqua.model.inline.state.InliningState
import aqua.raw.ops.*
import aqua.raw.value.*
import aqua.types.*
import cats.data.{Chain, NonEmptyList, NonEmptyMap}
import cats.syntax.show.*
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
class MakeStructInlinerSpec extends AnyFlatSpec with Matchers {
"make struct inliner" should "unfold values in parallel" in {
val structType = StructType(
"struct_type",
NonEmptyMap.of("field1" -> ScalarType.u32, "field2" -> ScalarType.string)
)
val arrType = ArrayType(ScalarType.string)
val length = FunctorRaw("length", ScalarType.u32)
val lengthValue = VarRaw("l", arrType).withProperty(length)
val getField = CallArrowRaw(
None,
"get_field",
Nil,
ArrowType(NilType, UnlabeledConsType(ScalarType.string, NilType)),
Option(LiteralRaw("\"serv\"", ScalarType.string))
)
val makeStruct =
MakeStructRaw(NonEmptyMap.of("field1" -> lengthValue, "field2" -> getField), structType)
val varName = structType.name
val (model, tree) =
RawValueInliner.valueToModel[InliningState](makeStruct, false).run(InliningState()).value._2
val result = VarModel(varName + "_obj", structType)
model shouldBe result
val lengthModel = FunctorModel("length", ScalarType.u32)
tree.get.equalsOrShowDiff(
SeqModel.wrap(
ParModel.wrap(
SeqModel.wrap(
FlattenModel(VarModel("l", arrType), "l_to_functor").leaf,
FlattenModel(VarModel("l_to_functor", arrType, Chain.one(lengthModel)), "l_length").leaf
),
CallServiceModel(
"serv",
"get_field",
Nil,
VarModel("get_field", ScalarType.string)
).leaf
),
CallServiceModel(
"json",
"obj",
LiteralModel.fromRaw(
LiteralRaw.quote("field1")
) :: VarModel("l_length", ScalarType.u32) :: LiteralModel.fromRaw(
LiteralRaw.quote("field2")
) :: VarModel("get_field", ScalarType.string) :: Nil,
result
).leaf
)
) shouldBe true
}
}

8
model/inline/src/test/scala/aqua/model/inline/RawValueInlinerSpec.scala
View File

@ -17,6 +17,7 @@ import aqua.raw.value.{ApplyPropertyRaw, FunctorRaw, IntoIndexRaw, LiteralRaw, V
import aqua.types.*
import cats.data.NonEmptyMap
import cats.data.Chain
import cats.syntax.show.*
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
@ -207,7 +208,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
resTree.isEmpty should be(false)
resTree.get.equalsOrShowDiff(
SeqModel.wrap(
ParModel.wrap(
SeqModel.wrap(
FlattenModel(
VarModel(
@ -284,7 +285,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
resTree.isEmpty should be(false)
resTree.get.equalsOrShowDiff(
SeqModel.wrap(
ParModel.wrap(
FlattenModel(
VarModel(
"ys",
@ -325,7 +326,6 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
)
)
)
println(resTree)
}
"raw value inliner" should "desugarize stream with length" in {
@ -367,7 +367,7 @@ class RawValueInlinerSpec extends AnyFlatSpec with Matchers {
resTree.isEmpty should be(false)
resTree.get.equalsOrShowDiff(
SeqModel.wrap(
ParModel.wrap(
FlattenModel(
VarModel(
"ys",