Merge pull request #119 from szeiger/perf-scopes

Some additional performance optimization
databricks · Jul 1, 2021 · d013527 · d013527
2 parents 59a8e7d + 972b41a
commit d013527
Show file tree

Hide file tree

Showing 45 changed files with 3,985 additions and 1,691 deletions.
diff --git a/bench/src/main/scala/sjsonnet/MainBenchmark.scala b/bench/src/main/scala/sjsonnet/MainBenchmark.scala
@@ -1,12 +1,35 @@
 package sjsonnet
 
-import java.io.{OutputStream, PrintStream}
+import java.io.{OutputStream, PrintStream, StringWriter}
 import java.util.concurrent.TimeUnit
 
 import org.openjdk.jmh.annotations._
 import org.openjdk.jmh.infra._
 
-import scala.collection.mutable.ArrayBuffer
+object MainBenchmark {
+  val mainArgs = Array[String](
+    "../../universe2/rulemanager/deploy/rulemanager.jsonnet",
+    "-J", "../../universe2",
+    "-J", "../../universe2/mt-shards/dev/az-westus-c2",
+  )
+
+  def findFiles(): (IndexedSeq[(Path, String)], EvalScope) = {
+    val parser = mainargs.ParserForClass[Config]
+    val config = parser.constructEither(MainBenchmark.mainArgs, autoPrintHelpAndExit = None).getOrElse(???)
+    val file = config.file
+    val wd = os.pwd
+    val path = OsPath(os.Path(file, wd))
+    val interp = new Interpreter(
+      Map.empty[String, ujson.Value],
+      Map.empty[String, ujson.Value],
+      OsPath(wd),
+      importer = SjsonnetMain.resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), None)
+    )
+    val renderer = new Renderer(new StringWriter, indent = 3)
+    interp.interpret0(interp.resolver.read(path).get, path, renderer).getOrElse(???)
+    (interp.parseCache.keySet.toIndexedSeq, interp.evaluator)
+  }
+}
 
 @BenchmarkMode(Array(Mode.AverageTime))
 @Fork(4)
@@ -17,12 +40,6 @@ import scala.collection.mutable.ArrayBuffer
 @State(Scope.Benchmark)
 class MainBenchmark {
 
-  val mainArgs = Array[String](
-    "../../universe2/rulemanager/deploy/rulemanager.jsonnet",
-    "-J", "../../universe2",
-    "-J", "../../universe2/mt-shards/dev/az-westus-c2",
-  )
-
   val dummyOut = new PrintStream(new OutputStream {
     def write(b: Int): Unit = ()
     override def write(b: Array[Byte]): Unit = ()
@@ -32,7 +49,7 @@ class MainBenchmark {
   @Benchmark
   def main(bh: Blackhole): Unit = {
     bh.consume(SjsonnetMain.main0(
-      mainArgs,
+      MainBenchmark.mainArgs,
       collection.mutable.HashMap.empty,
       System.in,
       dummyOut,

diff --git a/bench/src/main/scala/sjsonnet/MaterializerBenchmark.scala b/bench/src/main/scala/sjsonnet/MaterializerBenchmark.scala
@@ -0,0 +1,68 @@
+package sjsonnet
+
+import java.io.{StringWriter, Writer}
+import java.util.concurrent.TimeUnit
+
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra._
+import ujson.JsVisitor
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(2)
+@Threads(1)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+class MaterializerBenchmark {
+
+  private var interp: Interpreter = _
+  private var value: Val = _
+
+  @Setup
+  def setup(): Unit = {
+    val parser = mainargs.ParserForClass[Config]
+    val config = parser.constructEither(MainBenchmark.mainArgs, autoPrintHelpAndExit = None).getOrElse(???)
+    val file = config.file
+    val wd = os.pwd
+    val path = os.Path(file, wd)
+    var currentPos: Position = null
+    this.interp = new Interpreter(
+      Map.empty[String, ujson.Value],
+      Map.empty[String, ujson.Value],
+      OsPath(wd),
+      importer = SjsonnetMain.resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), None),
+    )
+    value = interp.evaluate(os.read(path), OsPath(path)).getOrElse(???)
+    assert(renderYaml() == oldRenderYaml())
+    val r1 = render()
+    assert(r1 == oldRender())
+    System.err.println("JSON length: "+r1.length)
+    assert(renderPython() == oldRenderPython())
+  }
+
+  @Benchmark def newRenderB(bh: Blackhole): Unit = bh.consume(render())
+  @Benchmark def oldRenderB(bh: Blackhole): Unit = bh.consume(oldRender())
+  @Benchmark def newRenderPythonB(bh: Blackhole): Unit = bh.consume(renderPython())
+  @Benchmark def oldRenderPythonB(bh: Blackhole): Unit = bh.consume(oldRenderPython())
+  @Benchmark def newRenderYamlB(bh: Blackhole): Unit = bh.consume(renderYaml())
+  @Benchmark def oldRenderYamlB(bh: Blackhole): Unit = bh.consume(oldRenderYaml())
+
+  @Benchmark def renderPrettyYamlB(bh: Blackhole): Unit = bh.consume(renderPrettyYaml())
+
+  private def render() = renderWith(new Renderer(_, indent=3))
+  private def renderPython() = renderWith(new PythonRenderer(_, indent=3))
+  private def renderYaml() = renderWith(new YamlRenderer(_, indent=3))
+  private def oldRender() = renderWith(new OldRenderer(_, indent=3))
+  private def oldRenderPython() = renderWith(new OldPythonRenderer(_, indent=3))
+  private def oldRenderYaml() = renderWith(new OldYamlRenderer(_, indent=3))
+
+  private def renderPrettyYaml() = renderWith(new PrettyYamlRenderer(_, indent=3, getCurrentPosition = () => null))
+
+  private def renderWith[T <: Writer](r: StringWriter => JsVisitor[T, T]): String = {
+    val writer = new StringWriter
+    val renderer = r(writer)
+    interp.materialize(value, renderer)
+    writer.toString
+  }
+}
diff --git a/bench/src/main/scala/sjsonnet/OptimizerBenchmark.scala b/bench/src/main/scala/sjsonnet/OptimizerBenchmark.scala
@@ -0,0 +1,119 @@
+package sjsonnet
+
+import java.io.StringWriter
+import java.util.concurrent.TimeUnit
+
+import fastparse.Parsed.Success
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra._
+
+import scala.collection.mutable
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(2)
+@Threads(1)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+class OptimizerBenchmark {
+
+  private var inputs: Iterable[(Expr, FileScope)] = _
+  private var allFiles: IndexedSeq[(Path, String)] = _
+  private var ev: EvalScope = _
+
+  @Setup
+  def setup(): Unit = {
+    val (allFiles, ev) = MainBenchmark.findFiles()
+    this.inputs = allFiles.map { case (p, s) =>
+      fastparse.parse(s, new Parser(p).document(_)) match {
+        case Success(v, _) => v
+      }
+    }
+    this.ev = ev
+    val static = inputs.map {
+      case (expr, fs) => ((new StaticOptimizer(ev)).optimize(expr), fs)
+    }
+    val countBefore, countStatic = new Counter
+    inputs.foreach(t => assert(countBefore.transform(t._1) eq t._1))
+    static.foreach(t => assert(countStatic.transform(t._1) eq t._1))
+    System.err.println(s"Documents: total=${inputs.size}")
+    System.err.println(s"Before: $countBefore")
+    System.err.println(s"Static: $countStatic")
+  }
+
+  @Benchmark
+  def main(bh: Blackhole): Unit = {
+    bh.consume(inputs.foreach { case (expr, fs) =>
+      bh.consume((new StaticOptimizer(ev)).optimize(expr))
+    })
+  }
+
+  class Counter extends ExprTransform {
+    var total, vals, exprs, arrVals, staticArrExprs, otherArrExprs, staticObjs, missedStaticObjs,
+      otherObjs, namedApplies, applies, arityApplies, builtin = 0
+    val applyArities = new mutable.LongMap[Int]()
+    val ifElseChains = new mutable.LongMap[Int]()
+    val selectChains = new mutable.LongMap[Int]()
+    def transform(e: Expr) = {
+      total += 1
+      if(e.isInstanceOf[Val]) vals += 1
+      else exprs += 1
+      e match {
+        case _: Val.Arr => arrVals += 1
+        case a: Expr.Arr =>
+          if(a.value.forall(_.isInstanceOf[Val])) staticArrExprs += 1
+          else otherArrExprs += 1
+        case _: Val.Obj => staticObjs += 1
+        case e: Expr.ObjBody.MemberList =>
+          if(e.binds == null && e.asserts == null && e.fields.forall(_.isStatic)) missedStaticObjs += 1
+          else otherObjs += 1
+        case e: Expr.Apply =>
+          if(e.namedNames == null) {
+            applies += 1
+            val a = e.args.length
+            applyArities.put(a.toLong, applyArities.getOrElse(a.toLong, 0) + 1)
+          } else namedApplies += 1
+
+        case _: Expr.Apply0 | _: Expr.Apply1 | _: Expr.Apply2 | _: Expr.Apply3 => arityApplies += 1
+        case _: Expr.ApplyBuiltin | _: Expr.ApplyBuiltin1 | _: Expr.ApplyBuiltin2 => builtin += 1
+        case _ =>
+      }
+      val ifElseCount = countIfElse(e)
+      if(ifElseCount > 0) {
+        ifElseChains.put(ifElseCount.toLong, ifElseChains.getOrElse(ifElseCount.toLong, 0) + 1)
+        if(ifElseCount > 1)
+          ifElseChains.put(ifElseCount.toLong-1L, ifElseChains.getOrElse(ifElseCount.toLong-1L, 0) - 1)
+      }
+      val selectCount = countSelectOnId(e)
+      if(selectCount >= 0) {
+        selectChains.put(selectCount.toLong, selectChains.getOrElse(selectCount.toLong, 0) + 1)
+        if(selectCount > 0)
+          selectChains.put(selectCount.toLong-1L, selectChains.getOrElse(selectCount.toLong-1L, 0) - 1)
+      }
+      rec(e)
+    }
+    def countIfElse(e: Expr): Int = e match {
+      case Expr.IfElse(_, _, _, else0) =>
+        countIfElse(else0) + 1
+      case _ => 0
+    }
+    def countSelectOnId(e: Expr): Int = e match {
+      case Expr.Select(_, x, _) =>
+       val c = countSelectOnId(x)
+        if(c == -1) -1 else c + 1
+      case _: Expr.ValidId => 0
+      case _ => -1
+    }
+    override def toString = {
+      val arities = applyArities.toSeq.sortBy(_._1).map { case (a,b) => s"$a: $b" }.mkString(", ")
+      val chains = ifElseChains.toSeq.sortBy(_._1).map { case (a,b) => s"$a: $b" }.mkString(", ")
+      val selChains = selectChains.toSeq.sortBy(_._1).map { case (a,b) => s"$a: $b" }.mkString(", ")
+      s"Total: $total, Val: $vals, Expr: $exprs, Val.Arr: $arrVals, static Expr.Arr: $staticArrExprs, "+
+        s"other Expr.Arr: $otherArrExprs, Val.Obj: $staticObjs, static MemberList: $missedStaticObjs, "+
+        s"other MemberList: $otherObjs, named Apply: $namedApplies, other Apply: $applies, "+
+        s"ApplyN: $arityApplies, ApplyBuiltin*: $builtin; Apply arities: {$arities}, "+
+        s"if/else chains: $chains, Select/ValidId chains: $selChains"
+    }
+  }
+}
diff --git a/bench/src/main/scala/sjsonnet/ParserBenchmark.scala b/bench/src/main/scala/sjsonnet/ParserBenchmark.scala
@@ -0,0 +1,32 @@
+package sjsonnet
+
+import java.util.concurrent.TimeUnit
+
+import fastparse.Parsed.Success
+import org.openjdk.jmh.annotations._
+import org.openjdk.jmh.infra._
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@Fork(2)
+@Threads(1)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+class ParserBenchmark {
+
+  private var allFiles: IndexedSeq[(Path, String)] = _
+  private var interp: Interpreter = _
+
+  @Setup
+  def setup(): Unit =
+    allFiles = MainBenchmark.findFiles()._1
+
+  @Benchmark
+  def main(bh: Blackhole): Unit = {
+    bh.consume(allFiles.foreach { case (p, s) =>
+      val res = fastparse.parse(s, new Parser(p).document(_))
+      bh.consume(res.asInstanceOf[Success[_]])
+    })
+  }
+}