diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt index 6402383e7ce39..10f5bad2ba063 100644 --- a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/metrics/BuildAttributes.kt @@ -8,11 +8,9 @@ package org.jetbrains.kotlin.build.report.metrics import java.io.Serializable import java.util.* -class BuildAttributes : Serializable { - private val myAttributes = - EnumMap( - BuildAttribute::class.java - ) +data class BuildAttributes( + private val myAttributes: MutableMap = EnumMap(BuildAttribute::class.java) +) : Serializable { fun add(attr: BuildAttribute, count: Int = 1) { myAttributes[attr] = myAttributes.getOrDefault(attr, 0) + count diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/BuildReportService.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/BuildReportService.kt new file mode 100644 index 0000000000000..8ea284595cefc --- /dev/null +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/BuildReportService.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistics + +import org.jetbrains.kotlin.buildtools.api.KotlinLogger +import java.io.Serializable + +interface BuildReportService : Serializable { + fun process(data: T, log: KotlinLogger) +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/CompileStatisticsData.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/CompileStatisticsData.kt index a2087887eb5f7..b2523aec53cbf 100644 --- a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/CompileStatisticsData.kt +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/CompileStatisticsData.kt @@ -10,7 +10,7 @@ import java.text.SimpleDateFormat import java.util.* //Sensitive data. This object is used directly for statistic via http -private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC") } +internal val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC") } interface CompileStatisticsData { fun getVersion(): Int = 4 diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/FileReportService.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/FileReportService.kt new file mode 100644 index 0000000000000..e24bd1d8fffaf --- /dev/null +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/FileReportService.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistics + +import org.jetbrains.kotlin.buildtools.api.KotlinLogger +import java.io.File +import java.text.SimpleDateFormat +import java.util.* + +abstract class FileReportService( + buildReportDir: File, + projectName: String, + fileSuffix: String, +) : BuildReportService { + companion object { + internal val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC") } + } + + private val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) + private val outputFile = buildReportDir.resolve("$projectName-build-$ts.$fileSuffix") + + abstract fun printBuildReport(data: T, outputFile: File) + + override fun process(data: T, log: KotlinLogger) { + val buildReportPath = outputFile.toPath().toUri().toString() + try { + outputFile.parentFile.mkdirs() + if (!(outputFile.parentFile.exists() && outputFile.parentFile.isDirectory)) { + log.error("Kotlin build report cannot be created: '${outputFile.parentFile}' is a file or do not have permissions to create") + return + } + printBuildReport(data, outputFile) + + log.lifecycle("Kotlin build report is written to $buildReportPath") + } catch (e: Exception) { + log.error("Could not write Kotlin build report to $buildReportPath", e) + } + } +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/JsonReportService.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/JsonReportService.kt new file mode 100644 index 0000000000000..ce95a347874eb --- /dev/null +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/JsonReportService.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistics + +import com.google.gson.Gson +import java.io.File + +class JsonReportService( + buildReportDir: File, + projectName: String, +) : FileReportService(buildReportDir, projectName, "json") { + + /** + * Prints general build information and task/transform build metrics + */ + override fun printBuildReport(data: Any, outputFile: File) { + outputFile.bufferedWriter().use { + it.write(Gson().toJson(data)) + } + } +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/FileReportService.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/FileReportService.kt deleted file mode 100644 index fade0c3709e1e..0000000000000 --- a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/FileReportService.kt +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.build.report.statistics.file - -import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.build.report.statistics.* -import org.jetbrains.kotlin.build.report.statistics.asString -import org.jetbrains.kotlin.build.report.statistics.formatTime -import org.jetbrains.kotlin.buildtools.api.KotlinLogger -import java.io.File -import java.io.Serializable -import java.text.SimpleDateFormat -import java.util.* - -class FileReportService( - private val outputFile: File, - private val printMetrics: Boolean, - private val logger: KotlinLogger, -) : Serializable { - companion object { - private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC") } - fun reportBuildStatInFile( - buildReportDir: File, - projectName: String, - includeMetricsInReport: Boolean, - buildData: List>, - startParameters: BuildStartParameters, - failureMessages: List, - logger: KotlinLogger, - ) { - val ts = SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().time) - val reportFile = buildReportDir.resolve("$projectName-build-$ts.txt") - - FileReportService( - outputFile = reportFile, - printMetrics = includeMetricsInReport, - logger = logger - ).process(buildData, startParameters, failureMessages) - } - } - - private lateinit var p: Printer - - fun process( - statisticsData: List>, - startParameters: BuildStartParameters, - failureMessages: List = emptyList(), - ) { - val buildReportPath = outputFile.toPath().toUri().toString() - try { - outputFile.parentFile.mkdirs() - if (!(outputFile.parentFile.exists() && outputFile.parentFile.isDirectory)) { - logger.error("Kotlin build report cannot be created: '$outputFile.parentFile' is a file or do not have permissions to create") - return - } - - outputFile.bufferedWriter().use { writer -> - p = Printer(writer) - printBuildReport(statisticsData, startParameters, failureMessages) - } - - logger.lifecycle("Kotlin build report is written to $buildReportPath") - } catch (e: Exception) { - logger.error("Could not write Kotlin build report to $buildReportPath", e) - } - } - - private fun printBuildReport( - statisticsData: List>, - startParameters: BuildStartParameters, - failureMessages: List, - ) { - // NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. - // Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound - // a bit unfamiliar. - // TODO: If it is confusing, consider renaming "tasks" to "build operations" in this class. - printBuildInfo(startParameters, failureMessages) - if (printMetrics && statisticsData.isNotEmpty()) { - printMetrics( - statisticsData.map { it.getBuildTimesMetrics() }.reduce { agg, value -> - (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } - }, - statisticsData.map { it.getPerformanceMetrics() }.reduce { agg, value -> - (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } - }, - statisticsData.map { it.getNonIncrementalAttributes().asSequence() }.reduce { agg, value -> agg + value }.toList(), - aggregatedMetric = true - ) - p.println() - } - printTaskOverview(statisticsData) - printTasksLog(statisticsData) - } - - private fun printBuildInfo(startParameters: BuildStartParameters, failureMessages: List) { - p.withIndent("Gradle start parameters:") { - startParameters.let { - p.println("tasks = ${it.tasks}") - p.println("excluded tasks = ${it.excludedTasks}") - p.println("current dir = ${it.currentDir}") - p.println("project properties args = ${it.projectProperties}") - p.println("system properties args = ${it.systemProperties}") - } - } - p.println() - - if (failureMessages.isNotEmpty()) { - p.println("Build failed: ${failureMessages}") - p.println() - } - } - - private fun printMetrics( - buildTimesMetrics: Map, - performanceMetrics: Map, - nonIncrementalAttributes: Collection, - gcTimeMetrics: Map? = emptyMap(), - gcCountMetrics: Map? = emptyMap(), - aggregatedMetric: Boolean = false, - ) { - printBuildTimes(buildTimesMetrics) - if (aggregatedMetric) p.println() - - printBuildPerformanceMetrics(performanceMetrics) - if (aggregatedMetric) p.println() - - printBuildAttributes(nonIncrementalAttributes) - - //TODO: KT-57310 Implement build GC metric in - if (!aggregatedMetric) { - printGcMetrics(gcTimeMetrics, gcCountMetrics) - } - } - - private fun printGcMetrics( - gcTimeMetrics: Map?, - gcCountMetrics: Map?, - ) { - val keys = HashSet() - gcCountMetrics?.keys?.also { keys.addAll(it) } - gcTimeMetrics?.keys?.also { keys.addAll(it) } - if (keys.isEmpty()) return - - p.withIndent("GC metrics:") { - for (key in keys) { - p.println("$key:") - p.withIndent { - gcCountMetrics?.get(key)?.also { p.println("GC count: ${it}") } - gcTimeMetrics?.get(key)?.also { p.println("GC time: ${formatTime(it)}") } - } - } - } - } - - private fun printBuildTimes(buildTimes: Map) { - if (buildTimes.isEmpty()) return - - p.println("Time metrics:") - p.withIndent { - val visitedBuildTimes = HashSet() - fun printBuildTime(buildTime: BuildTime) { - if (!visitedBuildTimes.add(buildTime)) return - - val timeMs = buildTimes[buildTime] - if (timeMs != null) { - p.println("${buildTime.getReadableString()}: ${formatTime(timeMs)}") - p.withIndent { - buildTime.children()?.forEach { printBuildTime(it) } - } - } else { - //Skip formatting if parent metric does not set - buildTime.children()?.forEach { printBuildTime(it) } - } - } - - for (buildTime in buildTimes.keys.first().getAllMetrics()) { - if (buildTime.getParent() != null) continue - - printBuildTime(buildTime) - } - } - } - - private fun printBuildPerformanceMetrics(buildMetrics: Map) { - if (buildMetrics.isEmpty()) return - - p.withIndent("Size metrics:") { - for (metric in buildMetrics.keys.first().getAllMetrics()) { - buildMetrics[metric]?.let { printSizeMetric(metric, it) } - } - } - } - - private fun printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) { - fun BuildPerformanceMetric.numberOfAncestors(): Int { - var count = 0 - var parent: BuildPerformanceMetric? = getParent() - while (parent != null) { - count++ - parent = parent.getParent() - } - return count - } - - val indentLevel = sizeMetric.numberOfAncestors() - - repeat(indentLevel) { p.pushIndent() } - when (sizeMetric.getType()) { - ValueType.BYTES -> p.println("${sizeMetric.getReadableString()}: ${formatSize(value)}") - ValueType.NUMBER -> p.println("${sizeMetric.getReadableString()}: $value") - ValueType.NANOSECONDS -> p.println("${sizeMetric.getReadableString()}: $value") - ValueType.MILLISECONDS -> p.println("${sizeMetric.getReadableString()}: ${formatTime(value)}") - ValueType.TIME -> p.println("${sizeMetric.getReadableString()}: ${formatter.format(value)}") - } - repeat(indentLevel) { p.popIndent() } - } - - private fun printBuildAttributes(buildAttributes: Collection) { - if (buildAttributes.isEmpty()) return - - val buildAttributesMap = buildAttributes.groupingBy { it }.eachCount() - p.withIndent("Build attributes:") { - val attributesByKind = buildAttributesMap.entries.groupBy { it.key.kind }.toSortedMap() - for ((kind, attributesCounts) in attributesByKind) { - printMap(p, kind.name, attributesCounts.associate { (k, v) -> k.readableString to v }) - } - } - } - - private fun printTaskOverview(statisticsData: Collection>) { - var allTasksTimeMs = 0L - var kotlinTotalTimeMs = 0L - val kotlinTasks = ArrayList>() - - for (task in statisticsData) { - val taskTimeMs = task.getDurationMs() - allTasksTimeMs += taskTimeMs - - if (task.getFromKotlinPlugin() == true) { - kotlinTotalTimeMs += taskTimeMs - kotlinTasks.add(task) - } - } - - if (kotlinTasks.isEmpty()) { - p.println("No Kotlin task was run") - return - } - - val ktTaskPercent = (kotlinTotalTimeMs.toDouble() / allTasksTimeMs * 100).asString(1) - p.println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)") - - val table = TextTable("Time", "% of Kotlin time", "Task") - for (task in kotlinTasks.sortedWith(compareBy({ -it.getDurationMs() }, { it.getStartTimeMs() }))) { - val timeMs = task.getDurationMs() - val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1) - table.addRow(formatTime(timeMs), "$percent %", task.getTaskName()) - } - table.printTo(p) - p.println() - } - - private fun printTasksLog(statisticsData: List>) { - for (task in statisticsData.sortedWith(compareBy({ -it.getDurationMs() }, { it.getStartTimeMs() }))) { - printTaskLog(task) - p.println() - } - } - - private fun printTaskLog(statisticsData: CompileStatisticsData) { - val skipMessage = statisticsData.getSkipMessage() - if (skipMessage != null) { - p.println("Task '${statisticsData.getTaskName()}' was skipped: $skipMessage") - } else { - p.println("Task '${statisticsData.getTaskName()}' finished in ${formatTime(statisticsData.getDurationMs())}") - } - - statisticsData.getKotlinLanguageVersion()?.also { - p.withIndent("Task info:") { - p.println("Kotlin language version: $it") - } - } - - if (statisticsData.getIcLogLines().isNotEmpty()) { - p.withIndent("Compilation log for task '${statisticsData.getTaskName()}':") { - statisticsData.getIcLogLines().forEach { p.println(it) } - } - } - - if (printMetrics) { - printMetrics( - statisticsData.getBuildTimesMetrics(), statisticsData.getPerformanceMetrics(), statisticsData.getNonIncrementalAttributes(), - statisticsData.getGcTimeMetrics(), statisticsData.getGcCountMetrics() - ) - } - } -} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/Printer.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/Printer.kt index a103f18bfaefe..27ee35e42abbb 100644 --- a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/Printer.kt +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/Printer.kt @@ -9,7 +9,7 @@ import java.io.IOException private val LINE_SEPARATOR = System.getProperty("line.separator") -internal class Printer( +class Printer( private val out: Appendable, private val indentUnit: String = " ", private var indent: String = "" diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/ReadableFileReportService.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/ReadableFileReportService.kt new file mode 100644 index 0000000000000..07184bfbf21d1 --- /dev/null +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/ReadableFileReportService.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.build.report.statistics.file + +import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.statistics.* +import java.io.File + +data class ReadableFileReportData( + val statisticsData: List>, + val startParameters: BuildStartParameters, + val failureMessages: List = emptyList(), + val version: Int = 1 +) + +open class ReadableFileReportService( + buildReportDir: File, + projectName: String, + private val printMetrics: Boolean, +) : FileReportService>(buildReportDir, projectName, "txt") { + + open fun printCustomTaskMetrics(statisticsData: CompileStatisticsData, printer: Printer) {} + + /** + * Prints general build information, sum up compile metrics and detailed task and transform information. + * + * BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. + * We still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound a bit unfamiliar. + */ + override fun printBuildReport(data: ReadableFileReportData, outputFile: File) { + outputFile.bufferedWriter().use { writer -> + Printer(writer).printBuildReport(data, printMetrics) { compileStatisticsData -> + printCustomTaskMetrics( + compileStatisticsData, + this + ) + } + } + } +} \ No newline at end of file diff --git a/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/printerUtils.kt b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/printerUtils.kt new file mode 100644 index 0000000000000..1a1295fccfd57 --- /dev/null +++ b/compiler/build-tools/kotlin-build-statistics/src/org/jetbrains/kotlin/build/report/statistics/file/printerUtils.kt @@ -0,0 +1,253 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ +package org.jetbrains.kotlin.build.report.statistics.file + +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.ValueType +import org.jetbrains.kotlin.build.report.statistics.* +import org.jetbrains.kotlin.build.report.statistics.asString +import org.jetbrains.kotlin.build.report.statistics.formatTime +import org.jetbrains.kotlin.build.report.statistics.formatter +import java.util.ArrayList +import java.util.HashSet + +internal fun Printer.printBuildReport( + data: ReadableFileReportData, + printMetrics: Boolean, + printCustomTaskMetrics: Printer.(CompileStatisticsData) -> Unit, +) { + // NOTE: BuildExecutionData / BuildOperationRecord contains data for both tasks and transforms. + // Where possible, we still use the term "tasks" because saying "tasks/transforms" is a bit verbose and "build operations" may sound + // a bit unfamiliar. + + printBuildInfo(data.startParameters, data.failureMessages) + if (printMetrics && data.statisticsData.isNotEmpty()) { + printMetrics( + data.statisticsData.map { it.getBuildTimesMetrics() }.reduce { agg, value -> + (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } + }, + data.statisticsData.map { it.getPerformanceMetrics() }.reduce { agg, value -> + (agg.keys + value.keys).associateWith { (agg[it] ?: 0) + (value[it] ?: 0) } + }, + data.statisticsData.map { it.getNonIncrementalAttributes().asSequence() }.reduce { agg, value -> agg + value }.toList(), + aggregatedMetric = true, + ) + println() + } + printTaskOverview(data.statisticsData) + printTasksLog(data.statisticsData, printMetrics, printCustomTaskMetrics) +} + +private fun Printer.printBuildInfo(startParameters: BuildStartParameters, failureMessages: List) { + withIndent("Gradle start parameters:") { + startParameters.let { + println("tasks = ${it.tasks}") + println("excluded tasks = ${it.excludedTasks}") + println("current dir = ${it.currentDir}") + println("project properties args = ${it.projectProperties}") + println("system properties args = ${it.systemProperties}") + } + } + println() + + if (failureMessages.isNotEmpty()) { + println("Build failed: ${failureMessages}") + println() + } +} + +private fun Printer.printMetrics( + buildTimesMetrics: Map, + performanceMetrics: Map, + nonIncrementalAttributes: Collection, + gcTimeMetrics: Map? = emptyMap(), + gcCountMetrics: Map? = emptyMap(), + aggregatedMetric: Boolean = false, +) { + printBuildTimes(buildTimesMetrics) + if (aggregatedMetric) println() + + printBuildPerformanceMetrics(performanceMetrics) + if (aggregatedMetric) println() + + printBuildAttributes(nonIncrementalAttributes) + + //TODO: KT-57310 Implement build GC metric in + if (!aggregatedMetric) { + printGcMetrics(gcTimeMetrics, gcCountMetrics) + } +} + +private fun Printer.printGcMetrics( + gcTimeMetrics: Map?, + gcCountMetrics: Map?, +) { + val keys = HashSet() + gcCountMetrics?.keys?.also { keys.addAll(it) } + gcTimeMetrics?.keys?.also { keys.addAll(it) } + if (keys.isEmpty()) return + + withIndent("GC metrics:") { + for (key in keys) { + println("$key:") + withIndent { + gcCountMetrics?.get(key)?.also { println("GC count: ${it}") } + gcTimeMetrics?.get(key)?.also { println("GC time: ${formatTime(it)}") } + } + } + } +} + +private fun Printer.printBuildTimes(buildTimes: Map) { + if (buildTimes.isEmpty()) return + + println("Time metrics:") + withIndent { + val visitedBuildTimes = HashSet() + fun printBuildTime(buildTime: BuildTime) { + if (!visitedBuildTimes.add(buildTime)) return + + val timeMs = buildTimes[buildTime] + if (timeMs != null) { + println("${buildTime.getReadableString()}: ${formatTime(timeMs)}") + withIndent { + buildTime.children()?.forEach { printBuildTime(it) } + } + } else { + //Skip formatting if parent metric does not set + buildTime.children()?.forEach { printBuildTime(it) } + } + } + + for (buildTime in buildTimes.keys.first().getAllMetrics()) { + if (buildTime.getParent() != null) continue + + printBuildTime(buildTime) + } + } +} + +private fun Printer.printBuildPerformanceMetrics(buildMetrics: Map) { + if (buildMetrics.isEmpty()) return + + withIndent("Size metrics:") { + for (metric in buildMetrics.keys.first().getAllMetrics()) { + buildMetrics[metric]?.let { printSizeMetric(metric, it) } + } + } +} + +private fun Printer.printSizeMetric(sizeMetric: BuildPerformanceMetric, value: Long) { + fun BuildPerformanceMetric.numberOfAncestors(): Int { + var count = 0 + var parent: BuildPerformanceMetric? = getParent() + while (parent != null) { + count++ + parent = parent.getParent() + } + return count + } + + val indentLevel = sizeMetric.numberOfAncestors() + + repeat(indentLevel) { pushIndent() } + when (sizeMetric.getType()) { + ValueType.BYTES -> println("${sizeMetric.getReadableString()}: ${formatSize(value)}") + ValueType.NUMBER -> println("${sizeMetric.getReadableString()}: $value") + ValueType.NANOSECONDS -> println("${sizeMetric.getReadableString()}: $value") + ValueType.MILLISECONDS -> println("${sizeMetric.getReadableString()}: ${formatTime(value)}") + ValueType.TIME -> println("${sizeMetric.getReadableString()}: ${formatter.format(value)}") + } + repeat(indentLevel) { popIndent() } +} + +private fun Printer.printBuildAttributes(buildAttributes: Collection) { + if (buildAttributes.isEmpty()) return + + val buildAttributesMap = buildAttributes.groupingBy { it }.eachCount() + withIndent("Build attributes:") { + val attributesByKind = buildAttributesMap.entries.groupBy { it.key.kind }.toSortedMap() + for ((kind, attributesCounts) in attributesByKind) { + printMap(this, kind.name, attributesCounts.associate { (k, v) -> k.readableString to v }) + } + } +} + +private fun Printer.printTaskOverview(statisticsData: Collection>) { + var allTasksTimeMs = 0L + var kotlinTotalTimeMs = 0L + val kotlinTasks = ArrayList>() + + for (task in statisticsData) { + val taskTimeMs = task.getDurationMs() + allTasksTimeMs += taskTimeMs + + if (task.getFromKotlinPlugin() == true) { + kotlinTotalTimeMs += taskTimeMs + kotlinTasks.add(task) + } + } + + if (kotlinTasks.isEmpty()) { + println("No Kotlin task was run") + return + } + + val ktTaskPercent = (kotlinTotalTimeMs.toDouble() / allTasksTimeMs * 100).asString(1) + println("Total time for Kotlin tasks: ${formatTime(kotlinTotalTimeMs)} ($ktTaskPercent % of all tasks time)") + + val table = TextTable("Time", "% of Kotlin time", "Task") + for (task in kotlinTasks.sortedWith(compareBy({ -it.getDurationMs() }, { it.getStartTimeMs() }))) { + val timeMs = task.getDurationMs() + val percent = (timeMs.toDouble() / kotlinTotalTimeMs * 100).asString(1) + table.addRow(formatTime(timeMs), "$percent %", task.getTaskName()) + } + table.printTo(this) + println() +} + +private fun Printer.printTasksLog( + statisticsData: List>, + printMetrics: Boolean, + printCustomTaskMetrics: Printer.(CompileStatisticsData) -> Unit, +) { + for (taskData in statisticsData.sortedWith(compareBy({ -it.getDurationMs() }, { it.getStartTimeMs() }))) { + printTaskLog(taskData) + if (printMetrics) { + printMetrics( + taskData.getBuildTimesMetrics(), taskData.getPerformanceMetrics(), taskData.getNonIncrementalAttributes(), + taskData.getGcTimeMetrics(), taskData.getGcCountMetrics() + ) + printCustomTaskMetrics(taskData) + } + println() + } +} + + +private fun Printer.printTaskLog( + statisticsData: CompileStatisticsData, +) { + val skipMessage = statisticsData.getSkipMessage() + if (skipMessage != null) { + println("Task '${statisticsData.getTaskName()}' was skipped: $skipMessage") + } else { + println("Task '${statisticsData.getTaskName()}' finished in ${formatTime(statisticsData.getDurationMs())}") + } + + statisticsData.getKotlinLanguageVersion()?.also { + withIndent("Task info:") { + println("Kotlin language version: $it") + } + } + + if (statisticsData.getIcLogLines().isNotEmpty()) { + withIndent("Compilation log for task '${statisticsData.getTaskName()}':") { + statisticsData.getIcLogLines().forEach { println(it) } + } + } +} diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsFileReportService.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsFileReportService.kt new file mode 100644 index 0000000000000..6ceb75b722aee --- /dev/null +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsFileReportService.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.jps.statistic + +import org.jetbrains.kotlin.build.report.metrics.JpsBuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.JpsBuildTime +import org.jetbrains.kotlin.build.report.statistics.CompileStatisticsData +import org.jetbrains.kotlin.build.report.statistics.file.Printer +import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportService +import java.io.File +import kotlin.math.min + +internal class JpsFileReportService( + buildReportDir: File, + projectName: String, + printMetrics: Boolean, +) : ReadableFileReportService(buildReportDir, projectName, printMetrics) { + override fun printCustomTaskMetrics(statisticsData: CompileStatisticsData, printer: Printer) { + printer.println("Changed files: ${statisticsData.getChanges()}") + printer.println("Execution result: ${statisticsData.getTaskResult()}") + } +} \ No newline at end of file diff --git a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt index d1fcaee35ac76..643789a7c85a7 100644 --- a/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt +++ b/jps/jps-plugin/src/org/jetbrains/kotlin/jps/statistic/JpsStatisticsReportService.kt @@ -11,11 +11,8 @@ import org.jetbrains.jps.incremental.CompileContext import org.jetbrains.kotlin.build.report.FileReportSettings import org.jetbrains.kotlin.build.report.HttpReportSettings import org.jetbrains.kotlin.build.report.metrics.* -import org.jetbrains.kotlin.build.report.statistics.BuildDataType -import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters -import org.jetbrains.kotlin.build.report.statistics.HttpReportService -import org.jetbrains.kotlin.build.report.statistics.StatTag -import org.jetbrains.kotlin.build.report.statistics.file.FileReportService +import org.jetbrains.kotlin.build.report.statistics.* +import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportData import org.jetbrains.kotlin.compilerRunner.JpsKotlinLogger import java.io.File import java.net.InetAddress @@ -192,9 +189,14 @@ class JpsStatisticsReportServiceImpl( val compileStatisticsData = finishedModuleBuildMetrics.map { it.flush(context) } httpService?.sendData(compileStatisticsData, loggerAdapter) fileReportSettings?.also { - FileReportService.reportBuildStatInFile( - it.buildReportDir, context.projectDescriptor.project.name, true, compileStatisticsData, - BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList(), loggerAdapter + JpsFileReportService( + it.buildReportDir, context.projectDescriptor.project.name, true + ).process( + ReadableFileReportData( + compileStatisticsData, + BuildStartParameters(tasks = listOf(jpsBuildTaskName)), emptyList() + ), + loggerAdapter ) } } diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt index b1ed7ca7901fe..a75c5a43ebf51 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildReportsIT.kt @@ -5,18 +5,29 @@ package org.jetbrains.kotlin.gradle +import com.google.gson.* +import com.google.gson.stream.JsonReader import org.gradle.api.logging.LogLevel import org.gradle.util.GradleVersion +import org.jetbrains.kotlin.build.report.metrics.BuildMetrics +import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime +import org.jetbrains.kotlin.build.report.statistics.StatTag +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.internal.build.metrics.GradleBuildMetricsData import org.jetbrains.kotlin.gradle.report.BuildReportType +import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData +import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.testbase.* +import org.jetbrains.kotlin.gradle.testbase.TestVersions.ThirdPartyDependencies.GRADLE_ENTERPRISE_PLUGIN_VERSION +import org.jetbrains.kotlin.incremental.ChangedFiles import org.junit.jupiter.api.DisplayName import java.io.ObjectInputStream +import java.lang.reflect.Type import java.nio.file.Path import kotlin.io.path.* +import kotlin.test.assertEquals import kotlin.test.assertTrue -import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -import org.jetbrains.kotlin.gradle.testbase.TestVersions.ThirdPartyDependencies.GRADLE_ENTERPRISE_PLUGIN_VERSION @DisplayName("Build reports") @JvmGradlePluginTests @@ -92,17 +103,25 @@ class BuildReportsIT : KGPBaseTest() { ) @GradleTest fun testBuildMetricsForJsProject(gradleVersion: GradleVersion) { - testBuildReportInFile("kotlin-js-plugin-project", "compileKotlinJs", gradleVersion, - languageVersion = KotlinVersion.KOTLIN_1_7.version) + testBuildReportInFile( + "kotlin-js-plugin-project", + "compileKotlinJs", + gradleVersion, + languageVersion = KotlinVersion.KOTLIN_1_7.version + ) } - private fun testBuildReportInFile(project: String, task: String, gradleVersion: GradleVersion, - languageVersion: String = KotlinVersion.KOTLIN_2_0.version) { + private fun testBuildReportInFile( + project: String, + task: String, + gradleVersion: GradleVersion, + languageVersion: String = KotlinVersion.KOTLIN_2_0.version, + ) { project(project, gradleVersion) { build(task) { assertBuildReportPathIsPrinted() } - //Should contains build metrics for all compile kotlin tasks + //Should contain build metrics for all compile kotlin tasks validateBuildReportFile(KotlinVersion.DEFAULT.version) } @@ -110,7 +129,7 @@ class BuildReportsIT : KGPBaseTest() { build(task, buildOptions = buildOptions.copy(languageVersion = languageVersion)) { assertBuildReportPathIsPrinted() } - //Should contains build metrics for all compile kotlin tasks + //Should contain build metrics for all compile kotlin tasks validateBuildReportFile(languageVersion) } } @@ -389,4 +408,80 @@ class BuildReportsIT : KGPBaseTest() { } } + @DisplayName("json validation") + @GradleTestVersions( + additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0], + ) + @GradleTest + fun testJsonBuildMetricsFileValidation(gradleVersion: GradleVersion) { + project("simpleProject", gradleVersion) { + buildAndFail( + "compileKotlin", "-Pkotlin.build.report.output=JSON", + ) { + assertOutputContains("Can't configure json report: 'kotlin.build.report.json.directory' property is mandatory") + } + } + } + + @DisplayName("json report") + @GradleTestVersions( + additionalVersions = [TestVersions.Gradle.G_7_6, TestVersions.Gradle.G_8_0], + ) + @GradleTest + fun testJsonBuildReport(gradleVersion: GradleVersion) { + project("simpleProject", gradleVersion) { + build( + "compileKotlin", + "-Pkotlin.build.report.output=JSON", + "-Pkotlin.build.report.json.directory=${projectPath.resolve("report").pathString}" + ) { + //TODO: KT-66071 update deserialization + val gsonBuilder = GsonBuilder() + .registerTypeAdapter(BuildOperationRecord::class.java, object : JsonDeserializer { + override fun deserialize( + json: JsonElement?, + typeOfT: Type?, + context: JsonDeserializationContext?, + ): BuildOperationRecord? { + //workaround to read both TaskRecord and TransformRecord + return context?.deserialize(json, BuildOperationRecordImpl::class.java) + } + }).registerTypeAdapter(ChangedFiles::class.java, object : JsonDeserializer { + override fun deserialize( + json: JsonElement?, + typeOfT: Type?, + context: JsonDeserializationContext, + ): ChangedFiles? { + return null //ignore source changes right now + } + }) + + val jsonReport = projectPath.getSingleFileInDir("report") + val buildExecutionData = jsonReport.bufferedReader().use { + gsonBuilder.create().fromJson(JsonReader(it), BuildExecutionData::class.java) as BuildExecutionData + } + val buildOperationRecords = + buildExecutionData.buildOperationRecord.first { it.path == ":compileKotlin" } as BuildOperationRecordImpl + assertEquals(KotlinVersion.DEFAULT, buildOperationRecords.kotlinLanguageVersion) + } + } + } + } + +data class BuildOperationRecordImpl( + override val path: String, + override val classFqName: String, + override val isFromKotlinPlugin: Boolean, + override val startTimeMs: Long, // Measured by System.currentTimeMillis(), + override val totalTimeMs: Long, + override val buildMetrics: BuildMetrics, + override val didWork: Boolean, + override val skipMessage: String?, + override val icLogLines: List, + //taskRecords + val kotlinLanguageVersion: KotlinVersion?, + val changedFiles: ChangedFiles? = null, + val compilerArguments: Array = emptyArray(), + val statTags: Set = emptySet(), +) : BuildOperationRecord \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt index c5da776d6673f..627f6107d0a94 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/PropertiesProvider.kt @@ -110,6 +110,9 @@ internal class PropertiesProvider private constructor(private val project: Proje val buildReportMetrics: Boolean get() = booleanProperty("kotlin.build.report.metrics") ?: false + val buildReportJsonDir: File? + get() = property(PropertyNames.KOTLIN_BUILD_REPORT_JSON_DIR).orNull?.let { File(it) } + val buildReportVerbose: Boolean get() = booleanProperty("kotlin.build.report.verbose") ?: false @@ -623,6 +626,7 @@ internal class PropertiesProvider private constructor(private val project: Proje val KOTLIN_JS_KARMA_BROWSERS = property("kotlin.js.browser.karma.browsers") val KOTLIN_BUILD_REPORT_SINGLE_FILE = property("kotlin.build.report.single_file") val KOTLIN_BUILD_REPORT_HTTP_URL = property("kotlin.build.report.http.url") + val KOTLIN_BUILD_REPORT_JSON_DIR = property("kotlin.build.report.json.directory") val KOTLIN_OPTIONS_SUPPRESS_FREEARGS_MODIFICATION_WARNING = property("kotlin.options.suppressFreeCompilerArgsModificationWarning") val KOTLIN_NATIVE_USE_XCODE_MESSAGE_STYLE = property("kotlin.native.useXcodeMessageStyle") val KOTLIN_COMPILER_USE_PRECISE_COMPILATION_RESULTS_BACKUP = property("kotlin.compiler.preciseCompilationResultsBackup") diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/GradleFileReportService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/GradleFileReportService.kt new file mode 100644 index 0000000000000..de03ee2492c69 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/GradleFileReportService.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime +import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportService +import java.io.File + +class GradleFileReportService( + buildReportDir: File, + projectName: String, + printMetrics: Boolean, +) : ReadableFileReportService(buildReportDir, projectName, printMetrics) \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt new file mode 100644 index 0000000000000..76e033a1c26e4 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/plugin/statistics/kotlinBuildStatisticsUtils.kt @@ -0,0 +1,203 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.plugin.statistics + +import org.gradle.api.Project +import org.gradle.api.artifacts.DependencySet +import org.gradle.api.artifacts.ProjectDependency +import org.gradle.api.invocation.Gradle +import org.jetbrains.kotlin.gradle.plugin.statistics.plugins.ObservablePlugins +import org.jetbrains.kotlin.gradle.report.BuildReportType +import org.jetbrains.kotlin.gradle.utils.* +import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics +import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics +import org.jetbrains.kotlin.statistics.metrics.StringMetrics +import kotlin.system.measureTimeMillis + +/** + * Collect general configuration metrics + **/ +internal fun collectGeneralConfigurationTimeMetrics( + gradle: Gradle, + buildReportOutputs: List, + useClasspathSnapshot: Boolean, + pluginVersion: String, + isProjectIsolationEnabled: Boolean, +): MetricContainer { + val configurationTimeMetrics = MetricContainer() + + val statisticOverhead = measureTimeMillis { + configurationTimeMetrics.put(StringMetrics.KOTLIN_COMPILER_VERSION, pluginVersion) + configurationTimeMetrics.put(StringMetrics.USE_CLASSPATH_SNAPSHOT, useClasspathSnapshot.toString()) + configurationTimeMetrics.put(StringMetrics.PROJECT_PATH, gradle.rootProject.projectDir.absolutePath) + configurationTimeMetrics.put(StringMetrics.GRADLE_VERSION, gradle.gradleVersion) + + //will be updated with KT-58266 + if (!isProjectIsolationEnabled) { + gradle.taskGraph.whenReady { taskExecutionGraph -> + val executedTaskNames = taskExecutionGraph.allTasks.map { it.name }.distinct() + configurationTimeMetrics.put(BooleanMetrics.MAVEN_PUBLISH_EXECUTED, executedTaskNames.contains("install")) + } + } + } + configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) + + return configurationTimeMetrics + +} + +/** + * Collect project's configuration metrics including applied plugins. It should be called inside afterEvaluate block. + */ +internal fun collectProjectConfigurationTimeMetrics( + project: Project, +): MetricContainer { + val configurationTimeMetrics = MetricContainer() + + val statisticOverhead = measureTimeMillis { + collectAppliedPluginsStatistics(project, configurationTimeMetrics) + + val configurations = project.configurations.asMap.values + for (configuration in configurations) { + try { + val configurationName = configuration.name + val dependencies = configuration.dependencies + + when (configurationName) { + "KoverEngineConfig" -> { + configurationTimeMetrics.put(BooleanMetrics.ENABLED_KOVER, true) + } + "kapt" -> { + configurationTimeMetrics.put(BooleanMetrics.ENABLED_KAPT, true) + for (dependency in dependencies) { + when (dependency.group) { + "com.google.dagger" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DAGGER, true) + "com.android.databinding" -> configurationTimeMetrics.put(BooleanMetrics.ENABLED_DATABINDING, true) + } + } + } + API -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_API_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + IMPLEMENTATION -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_IMPLEMENTATION_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + COMPILE -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_COMPILE_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + RUNTIME -> { + configurationTimeMetrics.put(NumericalMetrics.CONFIGURATION_RUNTIME_COUNT, 1) + reportLibrariesVersions(configurationTimeMetrics, dependencies) + } + } + } catch (e: Throwable) { + // Ignore exceptions for FUS metrics + } + } + + configurationTimeMetrics.put(NumericalMetrics.NUMBER_OF_SUBPROJECTS, 1) + + + configurationTimeMetrics.put( + BooleanMetrics.KOTLIN_KTS_USED, + project.buildscript.sourceFile?.name?.endsWith(".kts") ?: false + ) + + addTaskMetrics(project, configurationTimeMetrics) + + if (project.name == "buildSrc") { + configurationTimeMetrics.put(NumericalMetrics.BUILD_SRC_COUNT, 1) + configurationTimeMetrics.put(BooleanMetrics.BUILD_SRC_EXISTS, true) + } + } + configurationTimeMetrics.put(NumericalMetrics.STATISTICS_VISIT_ALL_PROJECTS_OVERHEAD, statisticOverhead) + + return configurationTimeMetrics +} + +private fun addTaskMetrics( + project: Project, + configurationTimeMetrics: MetricContainer, +) { + try { + val taskNames = project.tasks.names.toList() + configurationTimeMetrics.put(NumericalMetrics.GRADLE_NUMBER_OF_TASKS, taskNames.size.toLong()) + configurationTimeMetrics.put( + NumericalMetrics.GRADLE_NUMBER_OF_UNCONFIGURED_TASKS, + taskNames.count { name -> + try { + project.tasks.named(name).javaClass.name.contains("TaskCreatingProvider") + } catch (_: Exception) { + true + } + }.toLong() + ) + } catch (e: Exception) { + //ignore exceptions for KT-62131. + } +} + +private fun collectAppliedPluginsStatistics( + project: Project, + configurationTimeMetrics: MetricContainer, +) { + for (plugin in ObservablePlugins.values()) { + project.plugins.withId(plugin.title) { + configurationTimeMetrics.put(plugin.metric, true) + } + } +} + +private fun reportLibrariesVersions( + configurationTimeMetrics: MetricContainer, + dependencies: DependencySet?, +) { + dependencies?.filter { it !is ProjectDependency }?.forEach { dependency -> + when { + dependency.group?.startsWith("org.springframework") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_SPRING_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("com.vaadin") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_VAADIN_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("com.google.gwt") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_GWT_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group?.startsWith("org.hibernate") ?: false -> configurationTimeMetrics.put( + StringMetrics.LIBRARY_HIBERNATE_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlin" && dependency.name.startsWith("kotlin-stdlib") -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_STDLIB_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlinx" && dependency.name == "kotlinx-coroutines" -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_COROUTINES_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlin" && dependency.name == "kotlin-reflect" -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_REFLECT_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "org.jetbrains.kotlinx" && dependency.name + .startsWith("kotlinx-serialization-runtime") -> configurationTimeMetrics.put( + StringMetrics.KOTLIN_SERIALIZATION_VERSION, + dependency.version ?: "0.0.0" + ) + dependency.group == "com.android.tools.build" && dependency.name.startsWith("gradle") -> configurationTimeMetrics.put( + StringMetrics.ANDROID_GRADLE_PLUGIN_VERSION, + dependency.version ?: "0.0.0" + ) + } + } +} + diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportType.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportType.kt index 8f15e133ea0b1..b0b3439a30d45 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportType.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportType.kt @@ -13,9 +13,10 @@ enum class BuildReportType : Serializable { BUILD_SCAN, SINGLE_FILE, TRY_K2_CONSOLE, + JSON, ; companion object { - const val serialVersionUID: Long = 1L + const val serialVersionUID: Long = 2L } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt index 9a8c75203f768..7df33167ffa40 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt @@ -10,12 +10,14 @@ import org.gradle.api.logging.Logging import org.gradle.tooling.events.task.TaskFinishEvent import org.jetbrains.kotlin.build.report.metrics.ValueType import org.jetbrains.kotlin.build.report.statistics.HttpReportService -import org.jetbrains.kotlin.build.report.statistics.file.FileReportService import org.jetbrains.kotlin.build.report.statistics.formatSize import org.jetbrains.kotlin.build.report.statistics.BuildFinishStatisticsData import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters import org.jetbrains.kotlin.build.report.statistics.StatTag +import org.jetbrains.kotlin.build.report.statistics.* +import org.jetbrains.kotlin.build.report.statistics.file.ReadableFileReportData import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +import org.jetbrains.kotlin.gradle.plugin.statistics.GradleFileReportService import org.jetbrains.kotlin.gradle.report.data.BuildExecutionData import org.jetbrains.kotlin.gradle.report.data.BuildOperationRecord import org.jetbrains.kotlin.gradle.report.data.GradleCompileStatisticsData @@ -64,13 +66,16 @@ class BuildReportsService { executorService.submit { reportBuildFinish(parameters) } } reportingSettings.fileReportSettings?.also { - FileReportService.reportBuildStatInFile( + GradleFileReportService( it.buildReportDir, parameters.projectName, it.includeMetricsInReport, - transformOperationRecordsToCompileStatisticsData(buildOperationRecords, parameters, onlyKotlinTask = false), - parameters.startParameters, - failureMessages.filter { it.isNotEmpty() }, + ).process( + ReadableFileReportData( + transformOperationRecordsToCompileStatisticsData(buildOperationRecords, parameters, onlyKotlinTask = false), + parameters.startParameters, + failureMessages.filter { it.isNotEmpty() }, + ), loggerAdapter ) } @@ -83,6 +88,10 @@ class BuildReportsService { reportTryK2ToConsole(buildData) } + reportingSettings.jsonOutputDir?.also { + JsonReportService(it, parameters.projectName).process(buildData, loggerAdapter) + } + //It's expected that bad internet connection can cause a significant delay for big project executorService.shutdown() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt index 6fbef7b8ef855..5434ac2248ef5 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/ReportingSettings.kt @@ -19,10 +19,11 @@ data class ReportingSettings( val buildScanReportSettings: BuildScanSettings? = null, val singleOutputFile: File? = null, val experimentalTryK2ConsoleOutput: Boolean = false, + val jsonOutputDir: File? = null, val includeCompilerArguments: Boolean = false, ) : Serializable { companion object { - const val serialVersionUID: Long = 1 + const val serialVersionUID: Long = 2L } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt index bc52e8d8795ca..97bcaea42840c 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/configureReporing.kt @@ -15,6 +15,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_SINGLE_FILE import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_HTTP_URL +import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.PropertyNames.KOTLIN_BUILD_REPORT_JSON_DIR import org.jetbrains.kotlin.gradle.plugin.internal.isProjectIsolationEnabled import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly @@ -77,6 +78,11 @@ internal fun reportingSettings(project: Project): ReportingSettings { ?: throw IllegalStateException("Can't configure single file report: '$KOTLIN_BUILD_REPORT_SINGLE_FILE' property is mandatory") } else null + val jsonReportDir = if (buildReportOutputTypes.contains(BuildReportType.JSON)) { + properties.buildReportJsonDir + ?: throw IllegalStateException("Can't configure json report: '$KOTLIN_BUILD_REPORT_JSON_DIR' property is mandatory") + } else null + //temporary solution. support old property val oldSingleBuildMetric = properties.singleBuildMetricsFile?.also { buildReportOutputTypes.add(BuildReportType.SINGLE_FILE) } @@ -88,6 +94,7 @@ internal fun reportingSettings(project: Project): ReportingSettings { buildScanReportSettings = buildScanSettings, buildReportOutputs = buildReportOutputTypes, singleOutputFile = singleOutputFile ?: oldSingleBuildMetric, + jsonOutputDir = jsonReportDir, includeCompilerArguments = properties.buildReportIncludeCompilerArguments, experimentalTryK2ConsoleOutput = experimentalTryK2Enabled ) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt index 011e45cd82c5c..4de3413dbfd31 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/data/BuildExecutionData.kt @@ -10,14 +10,13 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime import org.jetbrains.kotlin.build.report.statistics.BuildStartParameters -class BuildExecutionData( +data class BuildExecutionData( val startParameters: BuildStartParameters, val failureMessages: List, - val buildOperationRecord: Collection + val buildOperationRecord: Collection, ) { - val aggregatedMetrics by lazy { - BuildMetrics().also { acc -> - buildOperationRecord.forEach { acc.addAll(it.buildMetrics) } - } + val aggregatedMetrics = BuildMetrics().also { acc -> + buildOperationRecord.forEach { acc.addAll(it.buildMetrics) } } + } \ No newline at end of file