From b635c1c628a2935f35859f3e41a3db47f489c7c4 Mon Sep 17 00:00:00 2001 From: Ting-Yuan Huang Date: Fri, 28 Jun 2024 11:05:19 -0700 Subject: [PATCH] Fork common-util for intellij dependences so that they can be built with different versions (cherry picked from commit bd04d69d8f79325453b5c702bbb769f9c0d4f418) --- common-util/build.gradle.kts | 19 - compiler-plugin/build.gradle.kts | 1 + .../ksp/common/IncrementalContextBase.kt | 0 .../ksp/common/IncrementalWrapperBase.kt | 0 .../devtools/ksp/common/PersistentMap.kt | 0 .../google/devtools/ksp/common/PsiUtils.kt | 0 .../ksp/common/impl/KSPCompilationError.kt | 0 kotlin-analysis-api/build.gradle.kts | 1 + .../ksp/common/IncrementalContextBase.kt | 574 ++++++++++++++++++ .../ksp/common/IncrementalWrapperBase.kt | 48 ++ .../devtools/ksp/common/PersistentMap.kt | 74 +++ .../google/devtools/ksp/common/PsiUtils.kt | 96 +++ .../ksp/common/impl/KSPCompilationError.kt | 6 + 13 files changed, 800 insertions(+), 19 deletions(-) rename {common-util => compiler-plugin}/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt (100%) rename {common-util => compiler-plugin}/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt (100%) rename {common-util => compiler-plugin}/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt (100%) rename {common-util => compiler-plugin}/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt (100%) rename {common-util => compiler-plugin}/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt (100%) create mode 100644 kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt create mode 100644 kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt create mode 100644 kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt create mode 100644 kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt create mode 100644 kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt diff --git a/common-util/build.gradle.kts b/common-util/build.gradle.kts index 447fc50d4e..7ce453a791 100644 --- a/common-util/build.gradle.kts +++ b/common-util/build.gradle.kts @@ -4,7 +4,6 @@ evaluationDependsOn(":api") description = "Kotlin Symbol Processing Util" -val intellijVersion: String by project val junitVersion: String by project val kotlinBaseVersion: String by project @@ -18,26 +17,8 @@ plugins { } dependencies { - listOf( - "com.jetbrains.intellij.platform:util-rt", - "com.jetbrains.intellij.platform:util-class-loader", - "com.jetbrains.intellij.platform:util-text-matching", - "com.jetbrains.intellij.platform:util", - "com.jetbrains.intellij.platform:util-base", - "com.jetbrains.intellij.platform:util-xml-dom", - "com.jetbrains.intellij.platform:core", - "com.jetbrains.intellij.platform:core-impl", - "com.jetbrains.intellij.platform:extensions", - "com.jetbrains.intellij.java:java-frontback-psi", - "com.jetbrains.intellij.java:java-psi", - "com.jetbrains.intellij.java:java-psi-impl", - ).forEach { - implementation("$it:$intellijVersion") { isTransitive = false } - } - implementation(project(":api")) implementation(kotlin("stdlib", kotlinBaseVersion)) - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") testImplementation("junit:junit:$junitVersion") } diff --git a/compiler-plugin/build.gradle.kts b/compiler-plugin/build.gradle.kts index 22f24ddbfc..c167edd98e 100644 --- a/compiler-plugin/build.gradle.kts +++ b/compiler-plugin/build.gradle.kts @@ -53,6 +53,7 @@ dependencies { implementation(kotlin("stdlib", kotlinBaseVersion)) compileOnly("org.jetbrains.kotlin:kotlin-compiler:$kotlinBaseVersion") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") implementation(project(":api")) implementation(project(":common-util")) diff --git a/common-util/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt similarity index 100% rename from common-util/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt rename to compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt diff --git a/common-util/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt similarity index 100% rename from common-util/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt rename to compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt diff --git a/common-util/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt similarity index 100% rename from common-util/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt rename to compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt diff --git a/common-util/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt similarity index 100% rename from common-util/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt rename to compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt diff --git a/common-util/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt b/compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt similarity index 100% rename from common-util/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt rename to compiler-plugin/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt diff --git a/kotlin-analysis-api/build.gradle.kts b/kotlin-analysis-api/build.gradle.kts index c0f84dd29f..82bb69a847 100644 --- a/kotlin-analysis-api/build.gradle.kts +++ b/kotlin-analysis-api/build.gradle.kts @@ -75,6 +75,7 @@ dependencies { } implementation("org.jetbrains.kotlinx:kotlinx-collections-immutable-jvm:0.3.4") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") compileOnly(kotlin("stdlib", aaKotlinBaseVersion)) implementation("com.google.guava:guava:$aaGuavaVersion") diff --git a/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt new file mode 100644 index 0000000000..37867f7a9f --- /dev/null +++ b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalContextBase.kt @@ -0,0 +1,574 @@ +/* + * Copyright 2020 Google LLC + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.ksp.common + +import com.google.devtools.ksp.isPrivate +import com.google.devtools.ksp.symbol.KSClassDeclaration +import com.google.devtools.ksp.symbol.KSDeclaration +import com.google.devtools.ksp.symbol.KSDeclarationContainer +import com.google.devtools.ksp.symbol.KSFile +import com.google.devtools.ksp.symbol.KSFunctionDeclaration +import com.google.devtools.ksp.symbol.KSNode +import com.google.devtools.ksp.visitor.KSDefaultVisitor +import com.intellij.psi.PsiJavaFile +import com.intellij.psi.PsiPackage +import com.intellij.util.containers.MultiMap +import java.io.File +import java.util.Date + +object symbolCollector : KSDefaultVisitor<(LookupSymbolWrapper) -> Unit, Unit>() { + override fun defaultHandler(node: KSNode, data: (LookupSymbolWrapper) -> Unit) = Unit + + override fun visitDeclaration(declaration: KSDeclaration, data: (LookupSymbolWrapper) -> Unit) { + if (declaration.isPrivate()) + return + + val name = declaration.simpleName.asString() + val scope = + declaration.qualifiedName?.asString()?.let { it.substring(0, Math.max(it.length - name.length - 1, 0)) } + ?: return + data(LookupSymbolWrapper(name, scope)) + } + + override fun visitDeclarationContainer( + declarationContainer: KSDeclarationContainer, + data: (LookupSymbolWrapper) -> Unit + ) { + // Local declarations aren't visible to other files / classes. + if (declarationContainer is KSFunctionDeclaration) + return + + declarationContainer.declarations.forEach { + it.accept(this, data) + } + } +} + +abstract class IncrementalContextBase( + protected val anyChangesWildcard: File, + protected val incrementalLog: Boolean, + protected val baseDir: File, + protected val cachesDir: File, + protected val kspOutputDir: File, + protected val knownModified: List, + protected val knownRemoved: List, + protected val changedClasses: List, +) { + // Symbols defined in changed files. This is used to update symbolsMap in the end. + private val updatedSymbols = MultiMap.createSet() + + // Sealed classes / interfaces on which `getSealedSubclasses` is invoked. + // This is used to update sealedMap in the end. + private val updatedSealed = MultiMap.createSet() + + // Sealed classes / interfaces on which `getSealedSubclasses` is invoked. + // This is saved across processing. + protected val sealedMap = FileToSymbolsMap(File(cachesDir, "sealed")) + + // Symbols defined in each file. This is saved across processing. + protected val symbolsMap = FileToSymbolsMap(File(cachesDir, "symbols")) + + private val cachesUpToDateFile = File(cachesDir, "caches.uptodate") + private val rebuild = !cachesUpToDateFile.exists() + + private val logsDir = File(cachesDir, "logs").apply { mkdirs() } + private val buildTime = Date().time + + private val modified = knownModified.map { it.relativeTo(baseDir) }.toSet() + private val removed = knownRemoved.map { it.relativeTo(baseDir) }.toSet() + + protected abstract val isIncremental: Boolean + + protected abstract val symbolLookupTracker: LookupTrackerWrapper + protected abstract val symbolLookupCache: LookupStorageWrapper + + protected abstract val classLookupTracker: LookupTrackerWrapper + protected abstract val classLookupCache: LookupStorageWrapper + + private val sourceToOutputsMap = FileToFilesMap(File(cachesDir, "sourceToOutputs")) + + private fun String.toRelativeFile() = File(this).relativeTo(baseDir) + private val KSFile.relativeFile + get() = filePath.toRelativeFile() + + private fun collectDefinedSymbols(ksFiles: Collection) { + ksFiles.forEach { file -> + file.accept(symbolCollector) { + updatedSymbols.putValue(file.relativeFile, it) + } + } + } + + private val removedOutputsKey = File("") + + private fun updateFromRemovedOutputs() { + val removedOutputs = sourceToOutputsMap.get(removedOutputsKey) ?: return + + symbolLookupCache.removeLookupsFrom(removedOutputs.asSequence()) + classLookupCache.removeLookupsFrom(removedOutputs.asSequence()) + removedOutputs.forEach { + symbolsMap.remove(it) + sealedMap.remove(it) + } + + sourceToOutputsMap.removeRecursively(removedOutputsKey) + } + + private fun updateLookupCache(dirtyFiles: Collection) { + symbolLookupCache.update(symbolLookupTracker, dirtyFiles, knownRemoved) + symbolLookupCache.flush() + symbolLookupCache.close() + + classLookupCache.update(classLookupTracker, dirtyFiles, knownRemoved) + classLookupCache.flush() + classLookupCache.close() + } + + private fun logSourceToOutputs(outputs: Set, sourceToOutputs: Map>) { + if (!incrementalLog) + return + + val logFile = File(logsDir, "kspSourceToOutputs.log") + logFile.appendText("=== Build $buildTime ===\n") + logFile.appendText("Accumulated source to outputs map\n") + sourceToOutputsMap.keys.forEach { source -> + logFile.appendText(" $source:\n") + sourceToOutputsMap[source]!!.forEach { output -> + logFile.appendText(" $output\n") + } + } + logFile.appendText("\n") + + logFile.appendText("Reprocessed sources and their outputs\n") + sourceToOutputs.forEach { (source, outputs) -> + logFile.appendText(" $source:\n") + outputs.forEach { + logFile.appendText(" $it\n") + } + } + logFile.appendText("\n") + + // Can be larger than the union of the above, because some outputs may have no source. + logFile.appendText("All reprocessed outputs\n") + outputs.forEach { + logFile.appendText(" $it\n") + } + logFile.appendText("\n") + } + + private fun logDirtyFiles( + files: Collection, + allFiles: Collection, + removedOutputs: Collection = emptyList(), + dirtyFilesByCP: Collection = emptyList(), + dirtyFilesByNewSyms: Collection = emptyList(), + dirtyFilesBySealed: Collection = emptyList(), + ) { + if (!incrementalLog) + return + + val logFile = File(logsDir, "kspDirtySet.log") + logFile.appendText("=== Build $buildTime ===\n") + logFile.appendText("All Files\n") + allFiles.forEach { logFile.appendText(" ${it.relativeFile}\n") } + logFile.appendText("Modified\n") + modified.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Removed\n") + removed.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Disappeared Outputs\n") + removedOutputs.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Affected By CP\n") + dirtyFilesByCP.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Affected By new syms\n") + dirtyFilesByNewSyms.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Affected By sealed\n") + dirtyFilesBySealed.forEach { logFile.appendText(" $it\n") } + logFile.appendText("CP changes\n") + changedClasses.forEach { logFile.appendText(" $it\n") } + logFile.appendText("Dirty:\n") + files.forEach { + logFile.appendText(" ${it.relativeFile}\n") + } + val percentage = "%.2f".format(files.size.toDouble() / allFiles.size.toDouble() * 100) + logFile.appendText("\nDirty / All: $percentage%\n\n") + } + + // Beware: no side-effects here; Caches should only be touched in updateCaches. + fun calcDirtyFiles(ksFiles: List): Collection = closeFilesOnException { + if (!isIncremental) { + return ksFiles + } + + if (rebuild) { + collectDefinedSymbols(ksFiles) + logDirtyFiles(ksFiles, ksFiles) + return ksFiles + } + + val newSyms = mutableSetOf() + + // Parse and add newly defined symbols in modified files. + ksFiles.filter { it.relativeFile in modified }.forEach { file -> + file.accept(symbolCollector) { + updatedSymbols.putValue(file.relativeFile, it) + newSyms.add(it) + } + } + + val dirtyFilesByNewSyms = newSyms.flatMap { + symbolLookupCache.get(it).map { File(it) } + } + + val dirtyFilesBySealed = sealedMap.keys + + // Calculate dirty files by dirty classes in CP. + val dirtyFilesByCP = changedClasses.flatMap { fqn -> + val name = fqn.substringAfterLast('.') + val scope = fqn.substringBeforeLast('.', "") + classLookupCache.get(LookupSymbolWrapper(name, scope)).map { File(it) } + + symbolLookupCache.get(LookupSymbolWrapper(name, scope)).map { File(it) } + }.toSet() + + // output files that exist in CURR~2 but not in CURR~1 + val removedOutputs = sourceToOutputsMap.get(removedOutputsKey) ?: emptyList() + + val noSourceFiles = changedClasses.map { fqn -> + NoSourceFile(baseDir, fqn).filePath.toRelativeFile() + }.toSet() + + val initialSet = mutableSetOf() + initialSet.addAll(modified) + initialSet.addAll(removed) + initialSet.addAll(removedOutputs) + initialSet.addAll(dirtyFilesByCP) + initialSet.addAll(dirtyFilesByNewSyms) + initialSet.addAll(dirtyFilesBySealed) + initialSet.addAll(noSourceFiles) + + // modified can be seen as removed + new. Therefore the following check doesn't work: + // if (modified.any { it !in sourceToOutputsMap.keys }) ... + if (modified.isNotEmpty() || changedClasses.isNotEmpty()) { + initialSet.add(anyChangesWildcard) + } + + val dirtyFiles = DirtinessPropagator( + symbolLookupCache, + symbolsMap, + sourceToOutputsMap, + anyChangesWildcard, + removedOutputsKey + ).propagate(initialSet) + + updateFromRemovedOutputs() + + logDirtyFiles( + ksFiles.filter { it.relativeFile in dirtyFiles }, + ksFiles, + removedOutputs, + dirtyFilesByCP, + dirtyFilesByNewSyms, + dirtyFilesBySealed + ) + return ksFiles.filter { it.relativeFile in dirtyFiles } + } + + // Loop detection isn't needed because of overwritten checks in CodeGeneratorImpl + private fun FileToFilesMap.removeRecursively(src: File) { + get(src)?.forEach { out -> + removeRecursively(out) + } + remove(src) + } + + private fun updateSourceToOutputs( + dirtyFiles: Collection, + outputs: Set, + sourceToOutputs: Map>, + removedOutputs: List, + ) { + // Prune deleted sources in source-to-outputs map. + removed.forEach { + sourceToOutputsMap.removeRecursively(it) + } + + dirtyFiles.filterNot { sourceToOutputs.containsKey(it) }.forEach { + sourceToOutputsMap.removeRecursively(it) + } + + removedOutputs.forEach { + sourceToOutputsMap.removeRecursively(it) + } + sourceToOutputsMap.put(removedOutputsKey, removedOutputs) + + // Update source-to-outputs map from those reprocessed. + sourceToOutputs.forEach { src, outs -> + sourceToOutputsMap.put(src, outs.toList()) + } + + logSourceToOutputs(outputs, sourceToOutputs) + + sourceToOutputsMap.flush() + } + + private fun updateOutputs(outputs: Set, cleanOutputs: Collection) { + val outRoot = kspOutputDir + val bakRoot = File(cachesDir, "backups") + + fun File.abs() = File(baseDir, path) + fun File.bak() = File(bakRoot, abs().toRelativeString(outRoot)) + + // Backing up outputs is necessary for two reasons: + // + // 1. Currently, outputs are always cleaned up in gradle plugin before compiler is called. + // Untouched outputs need to be restore. + // + // TODO: need a change in upstream to not clean files in gradle plugin. + // Not cleaning files in gradle plugin has potentially fewer copies when processing succeeds. + // + // 2. Even if outputs are left from last compilation / processing, processors can still + // fail and the outputs will need to be restored. + + // Backup + outputs.forEach { generated -> + copyWithTimestamp(generated.abs(), generated.bak(), true) + } + + // Restore non-dirty outputs + cleanOutputs.forEach { dst -> + if (dst !in outputs) { + copyWithTimestamp(dst.bak(), dst.abs(), true) + } + } + } + + private fun updateCaches(dirtyFiles: Collection, outputs: Set, sourceToOutputs: Map>) { + // dirtyFiles may contain new files, which are unknown to sourceToOutputsMap. + val oldOutputs = dirtyFiles.flatMap { sourceToOutputsMap[it] ?: emptyList() }.distinct() + val removedOutputs = oldOutputs.filterNot { it in outputs } + updateSourceToOutputs(dirtyFiles, outputs, sourceToOutputs, removedOutputs) + updateLookupCache(dirtyFiles) + + // Update symbolsMap + fun , V> update(m: PersistentMap>, u: MultiMap) { + // Update symbol caches from modified files. + u.keySet().forEach { + m.put(it, u[it].toList()) + } + } + + fun , V> remove(m: PersistentMap>, removedKeys: Collection) { + // Remove symbol caches from removed files. + removedKeys.forEach { + m.remove(it) + } + } + + if (!rebuild) { + update(sealedMap, updatedSealed) + remove(sealedMap, removed) + + update(symbolsMap, updatedSymbols) + remove(symbolsMap, removed) + } else { + symbolsMap.clear() + update(symbolsMap, updatedSymbols) + + sealedMap.clear() + update(sealedMap, updatedSealed) + } + symbolsMap.flush() + sealedMap.flush() + } + + fun registerGeneratedFiles(newFiles: Collection) = closeFilesOnException { + if (!isIncremental) + return@closeFilesOnException + + collectDefinedSymbols(newFiles) + } + + private inline fun closeFilesOnException(f: () -> T): T { + try { + return f() + } catch (e: Exception) { + symbolsMap.flush() + sealedMap.flush() + symbolLookupCache.close() + classLookupCache.close() + sourceToOutputsMap.flush() + throw e + } + } + + // TODO: add a wildcard for outputs with no source and get rid of the outputs parameter. + fun updateCachesAndOutputs( + dirtyFiles: Collection, + outputs: Set, + sourceToOutputs: Map>, + ) = closeFilesOnException { + if (!isIncremental) + return + + cachesUpToDateFile.delete() + assert(!cachesUpToDateFile.exists()) + + val dirtySources = dirtyFiles.map { it.relativeFile } + + // Throw away results from clean inputs. + // + // One common misuse of incremental APIs is associating a non-root source, instead of the ones obtained from + // root functions (e.g., getSymbolsWithAnnotation), to an output. This non-root source can be reached and + // reprocessed even when it is clean. Because it is clean, it is not available via root functions. As a result, + // other outputs that are solely based on it won't be re-generated and is deemed as removed. + // + // Assuming that the processors are deterministic, we are throwing away outputs from clean inputs, and + // recovering them from the backup as a workaround for processors. + + val unassociated = outputs - sourceToOutputs.values.flatten() + val dirties = HashSet(unassociated) + fun markDirty(file: File) { + dirties.add(file) + sourceToOutputs.get(file)?.forEach { + markDirty(it) + } + } + fun isDirty(file: File) = file in dirties + + val roots = mutableSetOf(anyChangesWildcard, removedOutputsKey) + roots.addAll(dirtySources) + // TODO: find a better way to identify NoSourceFile + roots.addAll( + sourceToOutputs.keys.filter { + it.path.startsWith("") + } + ) + roots.forEach { + markDirty(it) + } + + val dirtySourceToOutputs = sourceToOutputs.filter { (src, outs) -> + isDirty(src) + } + val dirtyOutputs = outputs.filter(::isDirty).toSet() + + updateCaches(dirtySources, dirtyOutputs, dirtySourceToOutputs) + + val cleanOutputs = mutableSetOf() + sourceToOutputsMap.keys.forEach { source -> + if (!isDirty(source)) + cleanOutputs.addAll(sourceToOutputsMap[source]!!) + } + sourceToOutputsMap.flush() + updateOutputs(dirtyOutputs, cleanOutputs) + + cachesUpToDateFile.createNewFile() + assert(cachesUpToDateFile.exists()) + } + + // Insert Java file -> names lookup records. + fun recordLookup(psiFile: PsiJavaFile, fqn: String) { + val path = psiFile.virtualFile.path + val name = fqn.substringAfterLast('.') + val scope = fqn.substringBeforeLast('.', "") + + // Java types are classes. Therefore lookups only happen in packages. + fun record(scope: String, name: String) = + symbolLookupTracker.record(path, scope, name) + + record(scope, name) + + // If a resolved name is from some * import, it is overridable by some out-of-file changes. + // Therefore, the potential providers all need to be inserted. They are + // 1. definition of the name in the same package + // 2. other * imports + val onDemandImports = + psiFile.getOnDemandImports(false, false).mapNotNull { (it as? PsiPackage)?.qualifiedName } + if (scope in onDemandImports) { + record(psiFile.packageName, name) + onDemandImports.forEach { + record(it, name) + } + } + } + + fun recordGetSealedSubclasses(classDeclaration: KSClassDeclaration) { + val name = classDeclaration.simpleName.asString() + val scope = classDeclaration.qualifiedName?.asString() + ?.let { it.substring(0, Math.max(it.length - name.length - 1, 0)) } ?: return + updatedSealed.putValue(classDeclaration.containingFile!!.relativeFile, LookupSymbolWrapper(name, scope)) + } +} + +internal class DirtinessPropagator( + private val lookupCache: LookupStorageWrapper, + private val symbolsMap: FileToSymbolsMap, + private val sourceToOutputs: FileToFilesMap, + private val anyChangesWildcard: File, + private val removedOutputsKey: File +) { + private val visitedFiles = mutableSetOf() + private val visitedSyms = mutableSetOf() + + private val outputToSources = mutableMapOf>().apply { + sourceToOutputs.keys.forEach { source -> + if (source != anyChangesWildcard && source != removedOutputsKey) { + sourceToOutputs[source]!!.forEach { output -> + getOrPut(output) { mutableSetOf() }.add(source) + } + } + } + } + + private fun visit(sym: LookupSymbolWrapper) { + if (sym in visitedSyms) + return + visitedSyms.add(sym) + + lookupCache.get(sym).forEach { + visit(File(it)) + } + } + + private fun visit(file: File) { + if (file in visitedFiles) + return + visitedFiles.add(file) + + // Propagate by dependencies + symbolsMap[file]?.forEach { + visit(it) + } + + // Propagate by input-output relations + // Given (..., I, ...) -> O: + // 1) if I is dirty, then O is dirty. + // 2) if O is dirty, then O must be regenerated, which requires all of its inputs to be reprocessed. + sourceToOutputs[file]?.forEach { + visit(it) + } + outputToSources[file]?.forEach { + visit(it) + } + } + + fun propagate(initialSet: Collection): Set { + initialSet.forEach { visit(it) } + return visitedFiles + } +} diff --git a/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt new file mode 100644 index 0000000000..b7efb8331f --- /dev/null +++ b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/IncrementalWrapperBase.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2023 Google LLC + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.ksp.common + +import com.intellij.util.containers.MultiMap +import java.io.File + +interface LookupTrackerWrapper { + fun record( + filePath: String, + scopeFqName: String, + name: String + ) + + val lookups: MultiMap +} + +data class LookupSymbolWrapper(val name: String, val scope: String) : Comparable { + override fun compareTo(other: LookupSymbolWrapper): Int { + val scopeCompare = scope.compareTo(other.scope) + if (scopeCompare != 0) return scopeCompare + + return name.compareTo(other.name) + } +} + +interface LookupStorageWrapper { + fun removeLookupsFrom(files: Sequence) + fun update(lookupTracker: LookupTrackerWrapper, filesToCompile: Iterable, removedFiles: Iterable) + fun flush() + fun close() + operator fun get(lookupSymbol: LookupSymbolWrapper): Collection +} diff --git a/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt new file mode 100644 index 0000000000..9b7811ebfa --- /dev/null +++ b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PersistentMap.kt @@ -0,0 +1,74 @@ +package com.google.devtools.ksp.common + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.builtins.MapSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import kotlinx.serialization.json.encodeToStream +import java.io.File + +private object FileSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("File", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: File) = encoder.encodeString(value.path) + override fun deserialize(decoder: Decoder): File = File(decoder.decodeString()) +} + +private object SymbolSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("LookupSymbolWrapper", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: LookupSymbolWrapper) = + encoder.encodeString("${value.name}:${value.scope}") + override fun deserialize(decoder: Decoder): LookupSymbolWrapper { + val (name, scope) = decoder.decodeString().split(':') + return LookupSymbolWrapper(name, scope) + } +} + +private val fileToFilesMapSerializer = MapSerializer(FileSerializer, ListSerializer(FileSerializer)) +private val fileToSymbolsMapSerializer = MapSerializer(FileSerializer, ListSerializer(SymbolSerializer)) + +abstract class PersistentMap( + private val serializer: KSerializer>, + private val storage: File, + private val m: MutableMap, +) : MutableMap by m { + + @OptIn(ExperimentalSerializationApi::class) + fun flush() { + storage.outputStream().use { + Json.encodeToStream(serializer, m.toMap(), it) + } + } + override fun toString() = m.toString() + + companion object { + @JvmStatic + @OptIn(ExperimentalSerializationApi::class) + protected fun deserialize(serializer: KSerializer>, storage: File): MutableMap { + return if (storage.exists()) { + Json.decodeFromStream(serializer, storage.inputStream()).toMutableMap() + } else { + mutableMapOf() + } + } + } +} + +class FileToFilesMap( + storage: File +) : PersistentMap>(fileToFilesMapSerializer, storage, deserialize(fileToFilesMapSerializer, storage)) + +class FileToSymbolsMap( + storage: File +) : PersistentMap>( + fileToSymbolsMapSerializer, + storage, + deserialize(fileToSymbolsMapSerializer, storage) +) diff --git a/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt new file mode 100644 index 0000000000..a760f3340d --- /dev/null +++ b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/PsiUtils.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2022 Google LLC + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.devtools.ksp.common + +import com.google.devtools.ksp.symbol.* +import com.intellij.lang.jvm.JvmModifier +import com.intellij.openapi.project.Project +import com.intellij.psi.PsiDocumentManager +import com.intellij.psi.PsiElement +import com.intellij.psi.PsiFile +import com.intellij.psi.PsiModifierListOwner + +val jvmModifierMap = mapOf( + JvmModifier.PUBLIC to Modifier.PUBLIC, + JvmModifier.PRIVATE to Modifier.PRIVATE, + JvmModifier.ABSTRACT to Modifier.ABSTRACT, + JvmModifier.FINAL to Modifier.FINAL, + JvmModifier.PROTECTED to Modifier.PROTECTED, + JvmModifier.STATIC to Modifier.JAVA_STATIC, + JvmModifier.STRICTFP to Modifier.JAVA_STRICT, + JvmModifier.NATIVE to Modifier.JAVA_NATIVE, + JvmModifier.SYNCHRONIZED to Modifier.JAVA_SYNCHRONIZED, + JvmModifier.TRANSIENT to Modifier.JAVA_TRANSIENT, + JvmModifier.VOLATILE to Modifier.JAVA_VOLATILE +) + +val javaModifiers = setOf( + Modifier.ABSTRACT, + Modifier.FINAL, + Modifier.JAVA_DEFAULT, + Modifier.JAVA_NATIVE, + Modifier.JAVA_STATIC, + Modifier.JAVA_STRICT, + Modifier.JAVA_SYNCHRONIZED, + Modifier.JAVA_TRANSIENT, + Modifier.JAVA_VOLATILE, + Modifier.PRIVATE, + Modifier.PROTECTED, + Modifier.PUBLIC, +) + +fun PsiModifierListOwner.toKSModifiers(): Set { + val modifiers = mutableSetOf() + modifiers.addAll( + jvmModifierMap.entries.filter { this.hasModifier(it.key) } + .map { it.value } + .toSet() + ) + if (this.modifierList?.hasExplicitModifier("default") == true) { + modifiers.add(Modifier.JAVA_DEFAULT) + } + return modifiers +} + +fun Project.findLocationString(file: PsiFile, offset: Int): String { + val psiDocumentManager = PsiDocumentManager.getInstance(this) + val document = psiDocumentManager.getDocument(file) ?: return "" + val lineNumber = document.getLineNumber(offset) + val offsetInLine = offset - document.getLineStartOffset(lineNumber) + return "${file.virtualFile.path}: (${lineNumber + 1}, ${offsetInLine + 1})" +} + +private fun parseDocString(raw: String): String? { + val t1 = raw.trim() + if (!t1.startsWith("/**") || !t1.endsWith("*/")) + return null + val lineSep = t1.findAnyOf(listOf("\r\n", "\n", "\r"))?.second ?: "" + return t1.trim('/').trim('*').lines().joinToString(lineSep) { + it.trimStart().trimStart('*') + } +} + +inline fun PsiElement.findParentOfType(): T? { + var parent = this.parent + while (parent != null && parent !is T) { + parent = parent.parent + } + return parent as? T +} + +fun Sequence.memoized() = MemoizedSequence(this) diff --git a/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt new file mode 100644 index 0000000000..4f27e711c8 --- /dev/null +++ b/kotlin-analysis-api/src/main/kotlin/com/google/devtools/ksp/common/impl/KSPCompilationError.kt @@ -0,0 +1,6 @@ +package com.google.devtools.ksp.common.impl + +import com.intellij.psi.PsiFile + +// PsiElement.toLocation() isn't available before ResolveImpl is initialized. +class KSPCompilationError(val file: PsiFile, val offset: Int, override val message: String) : Exception()