diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..60ca04c8f --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,94 @@ +name: Run Benchmarks +on: + push: + branches: + - main + pull_request: + +env: + BENCHMARK_RESULTS: snake-kmp-benchmarks/build/reports/benchmarks + +concurrency: + group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + run-benchmark: + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + - os: macos-latest + additional-args: '-x jvmBenchmark -x jsBenchmark' + - os: macos-13 # for macosX64 + additional-args: '-x jvmBenchmark -x jsBenchmark' + - os: windows-latest + additional-args: '-x jvmBenchmark -x jsBenchmark' + name: Performance regression check on ${{ matrix.os }} runner + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: 'Set up JDK' + uses: 'actions/setup-java@v4' + with: + java-version: '11' + distribution: 'zulu' + cache: 'gradle' + - name: Validate Gradle Wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + with: + gradle-version: wrapper + - name: Run benchmarks + run: ./gradlew -p snake-kmp-benchmarks benchmark --no-parallel ${{ matrix.additional-args }} + - uses: actions/upload-artifact@v4 + with: + name: bench-results-${{ matrix.os }} + path: ${{ env.BENCHMARK_RESULTS }}/main/**/*.json + collect-benchmarks-results: + runs-on: ubuntu-latest + needs: + - run-benchmark + env: + RESULTS_DIR: bench-results + steps: + # without checkout step 'benchmark-action/github-action-benchmark' action won't work + - uses: actions/checkout@v4 + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + pattern: bench-results-* + path: ${{ env.RESULTS_DIR }} + merge-multiple: true + - name: Prepare and join benchmark reports + id: prep + run: | + for report in $(find ./${{ env.RESULTS_DIR }} -type f -name "*.json") + do + file_name=$(basename "$report") + platform="${file_name%.*}" + # Trim 'it.krzeminski.snakeyaml.engine.kmp.benchmark.' to make benchmark name more readable + jq "[ .[] | .benchmark |= \"${platform}.\" + ltrimstr(\"it.krzeminski.snakeyaml.engine.kmp.benchmark.\") | .params |= map_values(. |= match(\"data.+\"; \"g\").string) ]" $report > ${{ env.RESULTS_DIR }}/$platform.json + done + AGGREGATED_REPORT=aggregated.json + # Joined reports looks like this: [[{},{}], [{},{}]] + # We need to transform them into this: [{},{}] + ls ${{ env.RESULTS_DIR }}/*.json + jq -s '[ .[] | .[] ]' ${{ env.RESULTS_DIR }}/*.json > $AGGREGATED_REPORT + echo "report=$AGGREGATED_REPORT" >> $GITHUB_OUTPUT + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: SnakeKMP benchmarks + tool: 'jmh' + output-file-path: ${{ steps.prep.outputs.report }} + comment-on-alert: true + summary-always: true + alert-threshold: '150%' + fail-threshold: '200%' + gh-repository: github.com/krzema12/snakeyaml-engine-kmp-benchmarks + github-token: ${{ secrets.PUBLISH_BENCHMARK_RESULTS }} + # Push and deploy GitHub pages branch automatically only if run in main repo and not in PR + auto-push: ${{ github.repository == 'krzema12/snakeyaml-engine-kmp' && github.event_name != 'pull_request' }} diff --git a/build.gradle.kts b/build.gradle.kts index adafbb7fd..d3e9b8899 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,6 +11,10 @@ group = "it.krzeminski" version = "3.0.2-SNAPSHOT" description = "SnakeYAML Engine KMP" +apiValidation { + ignoredProjects += listOf("snake-kmp-benchmarks") +} + kotlin { sourceSets { commonMain { diff --git a/gradle/kotlin-js-store/yarn.lock b/gradle/kotlin-js-store/yarn.lock index f95deae7c..f6ecad051 100644 --- a/gradle/kotlin-js-store/yarn.lock +++ b/gradle/kotlin-js-store/yarn.lock @@ -321,6 +321,14 @@ base64id@2.0.0, base64id@~2.0.0: resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== +benchmark@*: + version "2.1.4" + resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" + integrity sha512-l9MlfN4M1K/H2fbhfMy3B7vJd6AGKJVQn2h6Sg/Yx+KckoUA7ewS5Vv6TjSq18ooE1kS9hhAlQRH3AkXIh/aOQ== + dependencies: + lodash "^4.17.4" + platform "^1.3.3" + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -1178,7 +1186,7 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" -lodash@^4.17.15, lodash@^4.17.21: +lodash@^4.17.15, lodash@^4.17.21, lodash@^4.17.4: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -1436,6 +1444,11 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +platform@^1.3.3: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" @@ -1653,7 +1666,7 @@ source-map-loader@5.0.0: iconv-lite "^0.6.3" source-map-js "^1.0.2" -source-map-support@0.5.21, source-map-support@~0.5.20: +source-map-support@*, source-map-support@0.5.21, source-map-support@~0.5.20: version "0.5.21" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== diff --git a/settings.gradle.kts b/settings.gradle.kts index 6546df6c1..64bfb31c0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -55,3 +55,6 @@ dependencyResolutionManagement { enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") enableFeaturePreview("STABLE_CONFIGURATION_CACHE") + + +include("snake-kmp-benchmarks") diff --git a/snake-kmp-benchmarks/build.gradle.kts b/snake-kmp-benchmarks/build.gradle.kts new file mode 100644 index 000000000..7514d7dbc --- /dev/null +++ b/snake-kmp-benchmarks/build.gradle.kts @@ -0,0 +1,97 @@ +plugins { + kotlin("multiplatform") + kotlin("plugin.allopen") version "2.0.0" + id("org.jetbrains.kotlinx.benchmark") version "0.4.11" +} + +allOpen { + // JMH requires all benchmark classes to be open + annotation("org.openjdk.jmh.annotations.State") +} + +kotlin { + jvmToolchain(11) + + //region JVM Targets + jvm() + //endregion + + //region JS target + js(IR) { + nodejs() + } + //endregion + + //region Native Targets + // According to https://kotlinlang.org/docs/native-target-support.html + // Tier 1 + macosX64() + macosArm64() + iosSimulatorArm64() + iosX64() + + // Tier 2 + linuxX64() + linuxArm64() + watchosSimulatorArm64() + watchosX64() + watchosArm32() + watchosArm64() + tvosSimulatorArm64() + tvosX64() + tvosArm64() + iosArm64() + + // Tier 3 + mingwX64() + //endregion + + sourceSets { + commonMain { + dependencies { + implementation(project.dependencies.platform("com.squareup.okio:okio-bom:3.9.0")) + implementation(projects.snakeyamlEngineKmp) + implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.11") + implementation("com.squareup.okio:okio") + } + } + + jsMain { + dependencies { + implementation("net.thauvin.erik.urlencoder:urlencoder-lib:1.5.0") { + because("https://github.com/Kotlin/kotlinx-benchmark/issues/185 - only compile dependnecies (declared as api) are using during benchmark compilation") + } + implementation("com.squareup.okio:okio-nodefilesystem") + } + } + } +} + +benchmark { + configurations { + getByName("main") { + iterations = 10 + iterationTime = 5 + iterationTimeUnit = "s" + param( + "openAiYamlPath", + // Absolute path is required by JS target. Otherwise, file cannot be found + layout.projectDirectory + .file("data/issues/kmp-issue-204-OpenAI-API.yaml") + .asFile + .absolutePath, + ) + } + } + targets { + register("jvm") + register("js") + register("macosX64") + register("macosArm64") + register("iosX64") + register("iosArm64") + register("iosSimulatorArm64") + register("linuxX64") + register("mingwX64") + } +} diff --git a/src/jvmTest/resources/issues/kmp-issue-204-OpenAI-API.yaml b/snake-kmp-benchmarks/data/issues/kmp-issue-204-OpenAI-API.yaml similarity index 100% rename from src/jvmTest/resources/issues/kmp-issue-204-OpenAI-API.yaml rename to snake-kmp-benchmarks/data/issues/kmp-issue-204-OpenAI-API.yaml diff --git a/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.kt b/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.kt new file mode 100644 index 000000000..459f3f210 --- /dev/null +++ b/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.kt @@ -0,0 +1,9 @@ +package it.krzeminski.snakeyaml.engine.kmp.benchmark + +import okio.FileSystem + +/** + * Because of JS (and Wasm) target it is required to have this method + * to access the file system in the common code + */ +expect fun fileSystem(): FileSystem diff --git a/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/LoadingTimeBenchmark.kt b/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/LoadingTimeBenchmark.kt new file mode 100644 index 000000000..3f6ac7663 --- /dev/null +++ b/snake-kmp-benchmarks/src/commonMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/LoadingTimeBenchmark.kt @@ -0,0 +1,60 @@ +package it.krzeminski.snakeyaml.engine.kmp.benchmark + +import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings +import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader +import it.krzeminski.snakeyaml.engine.kmp.composer.Composer +import it.krzeminski.snakeyaml.engine.kmp.constructor.BaseConstructor +import it.krzeminski.snakeyaml.engine.kmp.constructor.StandardConstructor +import it.krzeminski.snakeyaml.engine.kmp.parser.ParserImpl +import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader +import kotlinx.benchmark.* +import okio.FileSystem +import okio.Path.Companion.toPath +import okio.buffer +import okio.use + +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(BenchmarkTimeUnit.MILLISECONDS) +class LoadingTimeBenchmark { + @Param("") + var openAiYamlPath: String = "" + + private val loadSettings = LoadSettings.builder().build() + + private lateinit var constructor: BaseConstructor + + @Setup + fun setUp() { + constructor = StandardConstructor(loadSettings) + } + + @Benchmark + fun loadsOpenAiSchema(): Map<*, *> { + return with(FILE_SYSTEM) { + openReadOnly(openAiYamlPath.toPath(normalize = true)).use { handle -> + handle.source().buffer().use { source -> + // TODO: there is a Load class in JVM sources that can handle all of it + // but it is not available for common code. + // Probably, it should be moved from JVM sources to common sources. + val reader = StreamReader( + loadSettings = loadSettings, + stream = YamlUnicodeReader(source), + ) + val composer = Composer( + settings = loadSettings, + parser = ParserImpl( + settings = loadSettings, + reader = reader, + ) + ) + constructor.constructSingleDocument(composer.getSingleNode()) as Map<*, *> + } + } + } + } + + private companion object { + private val FILE_SYSTEM: FileSystem = fileSystem() + } +} diff --git a/snake-kmp-benchmarks/src/jsMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.js.kt b/snake-kmp-benchmarks/src/jsMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.js.kt new file mode 100644 index 000000000..86c5b45f1 --- /dev/null +++ b/snake-kmp-benchmarks/src/jsMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.js.kt @@ -0,0 +1,10 @@ +package it.krzeminski.snakeyaml.engine.kmp.benchmark + +import okio.FileSystem +import okio.NodeJsFileSystem + +/** + * Without JS and Wasm targets there is no need in this function + * but we can keep it so minimize change when those targets are added + */ +actual fun fileSystem(): FileSystem = NodeJsFileSystem diff --git a/snake-kmp-benchmarks/src/jvmMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.jvm.kt b/snake-kmp-benchmarks/src/jvmMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.jvm.kt new file mode 100644 index 000000000..041bbed68 --- /dev/null +++ b/snake-kmp-benchmarks/src/jvmMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.jvm.kt @@ -0,0 +1,5 @@ +package it.krzeminski.snakeyaml.engine.kmp.benchmark + +import okio.FileSystem + +actual fun fileSystem(): FileSystem = FileSystem.SYSTEM diff --git a/snake-kmp-benchmarks/src/nativeMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.native.kt b/snake-kmp-benchmarks/src/nativeMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.native.kt new file mode 100644 index 000000000..041bbed68 --- /dev/null +++ b/snake-kmp-benchmarks/src/nativeMain/kotlin/it/krzeminski/snakeyaml/engine/kmp/benchmark/FileSystem.native.kt @@ -0,0 +1,5 @@ +package it.krzeminski.snakeyaml.engine.kmp.benchmark + +import okio.FileSystem + +actual fun fileSystem(): FileSystem = FileSystem.SYSTEM diff --git a/src/jvmTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/KmpIssue204.kt b/src/jvmTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/KmpIssue204.kt deleted file mode 100644 index 1778f3a2e..000000000 --- a/src/jvmTest/kotlin/it/krzeminski/snakeyaml/engine/kmp/KmpIssue204.kt +++ /dev/null @@ -1,26 +0,0 @@ -package it.krzeminski.snakeyaml.engine.kmp - -import io.kotest.core.spec.style.FunSpec -import io.kotest.matchers.comparables.shouldBeLessThan -import io.kotest.matchers.maps.shouldHaveSize -import io.kotest.matchers.nulls.shouldNotBeNull -import io.kotest.matchers.resource.resourceAsString -import it.krzeminski.snakeyaml.engine.kmp.api.Load -import kotlin.time.Duration.Companion.seconds -import kotlin.time.measureTime - -class KmpIssue204 : FunSpec({ - test("large yaml should be parsed quickly") { - // quick and dirty performance test - // TODO https://github.com/krzema12/snakeyaml-engine-kmp/issues/208 replace with proper performance tests - - val openAiApiYaml = resourceAsString("/issues/kmp-issue-204-OpenAI-API.yaml") - val duration = measureTime { - val loader = Load() - val value = loader.loadOne(openAiApiYaml) as? Map<*, *> - value.shouldNotBeNull() - value shouldHaveSize 8 - } - duration shouldBeLessThan 15.seconds // should be must faster, but GitHub runners can be slow - } -})