From bae25e0401f1663b5b7d0ebe0cf4aaca76ba5014 Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Sat, 19 Aug 2023 01:58:07 -0700 Subject: [PATCH 1/9] add basic jmh profiler support through advanced config --- plugin/main/src/kotlinx/benchmark/gradle/Utils.kt | 6 ++++++ .../src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt index c07d4a11..b72e8674 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt @@ -228,6 +228,11 @@ private fun validateConfig(config: BenchmarkConfiguration) { "jsUseBridge" -> require(value is Boolean) { "Invalid value for 'jsUseBridge': '$value'. Expected a Boolean value." } + "jmhProfiler" -> { + require(value.toString() in ValidOptions.jmhProfilers) { + "Invalid value for 'jmhProfiler': '$value'. Accepted values: ${ValidOptions.jmhProfilers.joinToString(", ")}." + } + } else -> throw IllegalArgumentException("Invalid advanced option name: '$param'. Accepted options: \"nativeFork\", \"nativeGCAfterIteration\", \"jvmForks\", \"jsUseBridge\".") } } @@ -244,6 +249,7 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") + val jmhProfilers = setOf("stack", "gc") } internal val Gradle.isConfigurationCacheAvailable diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index d1b8a5ca..5409ec72 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -56,6 +56,13 @@ fun main(args: Array) { } } + val profilerName = config.advanced["jmhProfiler"] + when (profilerName) { + "gc" -> jmhOptions.addProfiler("gc") + "stack" -> jmhOptions.addProfiler("stack") + else -> throw IllegalArgumentException("Unknown JMH profiler: $profilerName") + } + val reportFormat = ResultFormatType.valueOf(config.reportFormat.uppercase()) val reporter = BenchmarkProgress.create(config.traceFormat) val output = JmhOutputFormat(reporter, config.name) From 454e2ad6ea167067068f9a40dfac09e7e87d8f2e Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Sat, 19 Aug 2023 02:02:05 -0700 Subject: [PATCH 2/9] rename to config option to jmhProfiler --- runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index 5409ec72..eea87ec7 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -56,7 +56,7 @@ fun main(args: Array) { } } - val profilerName = config.advanced["jmhProfiler"] + val profilerName = config.advanced["jvmProfiler"] when (profilerName) { "gc" -> jmhOptions.addProfiler("gc") "stack" -> jmhOptions.addProfiler("stack") From ab8fe5b2d20e86069d9abac04354cd82508a4c89 Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Fri, 25 Aug 2023 23:52:51 -0700 Subject: [PATCH 3/9] add test and update error message --- .../benchmark/integration/OptionsValidationTest.kt | 11 +++++++++++ plugin/main/src/kotlinx/benchmark/gradle/Utils.kt | 8 ++++---- .../src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 4 +++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt index 9cb63fdf..a3156a60 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt @@ -226,6 +226,13 @@ class OptionsValidationTest : GradleTest() { iterationTimeUnit = "ms" advanced("jsUseBridge", "x") } + + configuration("invalidjvmProfiler") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jvmProfiler", "x") + } } runner.runAndFail("blankAdvancedConfigNameBenchmark") { @@ -246,6 +253,9 @@ class OptionsValidationTest : GradleTest() { runner.runAndFail("invalidJsUseBridgeBenchmark") { assertOutputContains("Invalid value for 'jsUseBridge': 'x'. Expected a Boolean value.") } + runner.runAndFail("invalidjvmProfiler") { + assertOutputContains("Invalid value for 'jvmProfiler': 'x'. Accepted values: ${ValidOptions.jvmProfilers.joinToString(", ")}.") + } } } @@ -260,4 +270,5 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") + val jvmProfilers = setOf("stack", "gc", "cl", "comp") } \ No newline at end of file diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt index b72e8674..a43b7a24 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt @@ -228,9 +228,9 @@ private fun validateConfig(config: BenchmarkConfiguration) { "jsUseBridge" -> require(value is Boolean) { "Invalid value for 'jsUseBridge': '$value'. Expected a Boolean value." } - "jmhProfiler" -> { - require(value.toString() in ValidOptions.jmhProfilers) { - "Invalid value for 'jmhProfiler': '$value'. Accepted values: ${ValidOptions.jmhProfilers.joinToString(", ")}." + "jvmProfiler" -> { + require(value.toString() in ValidOptions.jvmProfilers) { + "Invalid value for 'jvmProfiler': '$value'. Accepted values: ${ValidOptions.jvmProfilers.joinToString(", ")}." } } else -> throw IllegalArgumentException("Invalid advanced option name: '$param'. Accepted options: \"nativeFork\", \"nativeGCAfterIteration\", \"jvmForks\", \"jsUseBridge\".") @@ -249,7 +249,7 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") - val jmhProfilers = setOf("stack", "gc") + val jvmProfilers = setOf("stack", "gc", "cl", "comp") } internal val Gradle.isConfigurationCacheAvailable diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index eea87ec7..8359bc43 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -60,7 +60,9 @@ fun main(args: Array) { when (profilerName) { "gc" -> jmhOptions.addProfiler("gc") "stack" -> jmhOptions.addProfiler("stack") - else -> throw IllegalArgumentException("Unknown JMH profiler: $profilerName") + "cl" -> jmhOptions.addProfiler("cl") + "comp" -> jmhOptions.addProfiler("comp") + else -> throw IllegalArgumentException("Invalid value for 'jvmProfiler': $profilerName. Accepted values: gc, stack, cl") } val reportFormat = ResultFormatType.valueOf(config.reportFormat.uppercase()) From 60e1bb151a66aa44632d81dd04526bfca9a7e9b1 Mon Sep 17 00:00:00 2001 From: Henok Woldesenbet <62161211+wldeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 18:59:08 -0700 Subject: [PATCH 4/9] update test name --- .../kotlinx/benchmark/integration/OptionsValidationTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt index a3156a60..8350a316 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt @@ -227,7 +227,7 @@ class OptionsValidationTest : GradleTest() { advanced("jsUseBridge", "x") } - configuration("invalidjvmProfiler") { + configuration("invalidJvmProfiler") { iterations = 1 iterationTime = 100 iterationTimeUnit = "ms" @@ -253,7 +253,7 @@ class OptionsValidationTest : GradleTest() { runner.runAndFail("invalidJsUseBridgeBenchmark") { assertOutputContains("Invalid value for 'jsUseBridge': 'x'. Expected a Boolean value.") } - runner.runAndFail("invalidjvmProfiler") { + runner.runAndFail("invalidJvmProfiler") { assertOutputContains("Invalid value for 'jvmProfiler': 'x'. Accepted values: ${ValidOptions.jvmProfilers.joinToString(", ")}.") } } From 2db1cb33204aa077591c378f0550760d4710a084 Mon Sep 17 00:00:00 2001 From: Henok Woldesenbet <62161211+wldeh@users.noreply.github.com> Date: Sun, 27 Aug 2023 18:59:57 -0700 Subject: [PATCH 5/9] fix error message --- runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index 8359bc43..d4bd872b 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -62,7 +62,7 @@ fun main(args: Array) { "stack" -> jmhOptions.addProfiler("stack") "cl" -> jmhOptions.addProfiler("cl") "comp" -> jmhOptions.addProfiler("comp") - else -> throw IllegalArgumentException("Invalid value for 'jvmProfiler': $profilerName. Accepted values: gc, stack, cl") + else -> throw IllegalArgumentException("Invalid value for 'jvmProfiler': $profilerName. Accepted values: gc, stack, cl, comp") } val reportFormat = ResultFormatType.valueOf(config.reportFormat.uppercase()) From 452c353fa30d886d5e65bbca170e7a568143a6ff Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Mon, 11 Sep 2023 12:28:13 -0700 Subject: [PATCH 6/9] allow jvmProfiler not to be specified --- runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index d4bd872b..95e35dec 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -62,6 +62,7 @@ fun main(args: Array) { "stack" -> jmhOptions.addProfiler("stack") "cl" -> jmhOptions.addProfiler("cl") "comp" -> jmhOptions.addProfiler("comp") + null -> {} else -> throw IllegalArgumentException("Invalid value for 'jvmProfiler': $profilerName. Accepted values: gc, stack, cl, comp") } From f3e0b8a7c2bfd62a5d4ab93447d42e4f42ba7b5a Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Mon, 11 Sep 2023 12:31:54 -0700 Subject: [PATCH 7/9] add jvmProfiler to advanced error msg --- plugin/main/src/kotlinx/benchmark/gradle/Utils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt index a43b7a24..294fe9f6 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt @@ -233,7 +233,7 @@ private fun validateConfig(config: BenchmarkConfiguration) { "Invalid value for 'jvmProfiler': '$value'. Accepted values: ${ValidOptions.jvmProfilers.joinToString(", ")}." } } - else -> throw IllegalArgumentException("Invalid advanced option name: '$param'. Accepted options: \"nativeFork\", \"nativeGCAfterIteration\", \"jvmForks\", \"jsUseBridge\".") + else -> throw IllegalArgumentException("Invalid advanced option name: '$param'. Accepted options: \"nativeFork\", \"nativeGCAfterIteration\", \"jvmForks\", \"jsUseBridge\", \"jvmProfiler\".") } } } From dc09ae1ecbfd9a5b500ed6a250ab7f15f06fe4c8 Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:00:41 -0700 Subject: [PATCH 8/9] add profiler tests --- .../benchmark/integration/JvmProfilerTest.kt | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 integration/src/test/kotlin/kotlinx/benchmark/integration/JvmProfilerTest.kt diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/JvmProfilerTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/JvmProfilerTest.kt new file mode 100644 index 00000000..2b3b52f7 --- /dev/null +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/JvmProfilerTest.kt @@ -0,0 +1,74 @@ +package kotlinx.benchmark.integration + +import kotlin.test.* + +class JvmProfilerTest : GradleTest() { + + @Test + fun testGcProfiler() { + val runner = project("kotlin-multiplatform") { + configuration("gcProfiler") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jvmProfiler", "gc") + } + } + + runner.run("jvmGcProfilerBenchmark") { + assertOutputContains("gc.alloc.rate") + assertOutputContains("BUILD SUCCESSFUL") + } + } + + @Test + fun testStackProfilerEffect() { + val runner = project("kotlin-multiplatform") { + configuration("stackProfiler") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jvmProfiler", "stack") + } + } + + runner.run("jvmStackProfilerBenchmark") { + assertOutputContains("stack") + assertOutputContains("BUILD SUCCESSFUL") + } + } + + @Test + fun testClProfiler() { + val runner = project("kotlin-multiplatform") { + configuration("clProfiler") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jvmProfiler", "cl") + } + } + + runner.run("jvmClProfilerBenchmark") { + assertOutputContains("class.unload.norm") + assertOutputContains("BUILD SUCCESSFUL") + } + } + + @Test + fun testCompProfilerEffect() { + val runner = project("kotlin-multiplatform") { + configuration("compProfiler") { + iterations = 1 + iterationTime = 100 + iterationTimeUnit = "ms" + advanced("jvmProfiler", "comp") + } + } + + runner.run("jvmCompProfilerBenchmark") { + assertOutputContains("compiler.time.profiled") + assertOutputContains("BUILD SUCCESSFUL") + } + } +} From 0964a230f970def1011e4f2bb7272477ffc60eba Mon Sep 17 00:00:00 2001 From: wldeh <62161211+wldeh@users.noreply.github.com> Date: Wed, 13 Sep 2023 20:28:09 -0700 Subject: [PATCH 9/9] add support for os specific profilers --- .../kotlinx/benchmark/integration/OptionsValidationTest.kt | 2 +- plugin/main/src/kotlinx/benchmark/gradle/Utils.kt | 2 +- .../jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt index 8350a316..a96f4a84 100644 --- a/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt +++ b/integration/src/test/kotlin/kotlinx/benchmark/integration/OptionsValidationTest.kt @@ -270,5 +270,5 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") - val jvmProfilers = setOf("stack", "gc", "cl", "comp") + val jvmProfilers = setOf("stack", "gc", "cl", "comp", "perf", "perfnorm", "perfasm", "xperfasm", " dtraceasm") } \ No newline at end of file diff --git a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt index 294fe9f6..509b0627 100644 --- a/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt +++ b/plugin/main/src/kotlinx/benchmark/gradle/Utils.kt @@ -249,7 +249,7 @@ private object ValidOptions { ) val modes = setOf("thrpt", "avgt", "Throughput", "AverageTime") val nativeForks = setOf("perBenchmark", "perIteration") - val jvmProfilers = setOf("stack", "gc", "cl", "comp") + val jvmProfilers = setOf("stack", "gc", "cl", "comp", "perf", "perfnorm", "perfasm", "xperfasm", " dtraceasm") } internal val Gradle.isConfigurationCacheAvailable diff --git a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt index 95e35dec..2d7cc4e6 100644 --- a/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt +++ b/runtime/jvmMain/src/kotlinx/benchmark/jvm/JvmBenchmarkRunner.kt @@ -62,6 +62,11 @@ fun main(args: Array) { "stack" -> jmhOptions.addProfiler("stack") "cl" -> jmhOptions.addProfiler("cl") "comp" -> jmhOptions.addProfiler("comp") + "perf" -> jmhOptions.addProfiler("perf") + "perfnorm" -> jmhOptions.addProfiler("perfnorm") + "perfasm" -> jmhOptions.addProfiler("perfasm") + "xperfasm" -> jmhOptions.addProfiler("xperfasm") + "dtraceasm" -> jmhOptions.addProfiler("dtraceasm") null -> {} else -> throw IllegalArgumentException("Invalid value for 'jvmProfiler': $profilerName. Accepted values: gc, stack, cl, comp") }