Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add manifest files generated from ksp into final jar #1094

Merged
merged 11 commits into from
Jan 26, 2024
5 changes: 5 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ maven.install(
"org.pantsbuild:jarjar:1.7.2",
"org.jetbrains.kotlinx:atomicfu-js:0.15.2",
"org.jetbrains.kotlinx:kotlinx-serialization-runtime:1.0-M1-1.4.0-rc",
"dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
"com.squareup.moshi:moshi:1.15.0",
"com.squareup.moshi:moshi-kotlin:1.15.0",
"com.squareup.moshi:moshi-kotlin-codegen:1.15.0",
"com.google.auto.service:auto-service-annotations:jar:1.1.1",
],
fetch_sources = True,
repositories = [
Expand Down
11 changes: 11 additions & 0 deletions examples/ksp/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,15 @@ kt_ksp_plugin(
],
)

kt_ksp_plugin(
name = "autoservice",
processor_class = "dev.zacsweers.autoservice.ksp.AutoServiceSymbolProcessor$Provider",
deps = [
"@maven//:com_google_auto_service_auto_service_annotations",
"@maven//:dev_zacsweers_autoservice_auto_service_ksp",
],
)

kt_jvm_library(
name = "coffee_lib",
srcs = glob([
Expand All @@ -47,8 +56,10 @@ kt_jvm_library(
plugins = [
"//:moshi-kotlin-codegen",
"//:autovalue",
"//:autoservice",
],
deps = [
"@maven//:com_google_auto_service_auto_service_annotations",
"@maven//:com_google_auto_value_auto_value_annotations",
"@maven//:com_squareup_moshi_moshi",
"@maven//:com_squareup_moshi_moshi_kotlin",
Expand Down
2 changes: 1 addition & 1 deletion examples/ksp/CoffeeAppJavaModel.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ abstract static class Builder {

abstract CoffeeAppJavaModel build();
}
}
}
7 changes: 7 additions & 0 deletions examples/ksp/CoffeeAppService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package coffee;

import com.google.auto.service.AutoService;

@AutoService(Object.class)
public class CoffeeAppService {
}
2 changes: 2 additions & 0 deletions examples/ksp/WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,10 @@ maven_install(
"com.squareup.moshi:moshi:1.14.0",
"com.squareup.moshi:moshi-kotlin:1.14.0",
"com.squareup.moshi:moshi-kotlin-codegen:1.14.0",
"com.google.auto.service:auto-service-annotations:jar:1.1.1",
"com.google.auto.value:auto-value:1.10.1",
"com.google.auto.value:auto-value-annotations:1.10.1",
"dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
],
repositories = [
"https://maven.google.com",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import io.bazel.kotlin.builder.toolchain.KotlinToolchain
import io.bazel.kotlin.builder.utils.IS_JVM_SOURCE_FILE
import io.bazel.kotlin.builder.utils.bazelRuleKind
import io.bazel.kotlin.builder.utils.jars.JarCreator
import io.bazel.kotlin.builder.utils.jars.JarHelper.Companion.MANIFEST_DIR
import io.bazel.kotlin.builder.utils.jars.SourceJarExtractor
import io.bazel.kotlin.builder.utils.partitionJvmSources
import io.bazel.kotlin.model.JvmCompilationTask
Expand All @@ -39,9 +40,13 @@ import java.nio.file.Files.isDirectory
import java.nio.file.Files.walk
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import java.util.Base64
import java.util.stream.Collectors.toList
import java.util.stream.Stream
import kotlin.io.path.exists

private const val SOURCE_JARS_DIR = "_srcjars"

fun JvmCompilationTask.codeGenArgs(): CompilationArgs = CompilationArgs()
.absolutePaths(info.friendPathsList) {
Expand Down Expand Up @@ -438,16 +443,16 @@ fun JvmCompilationTask.compileKotlin(

/**
* If any srcjars were provided expand the jars sources and create a new [JvmCompilationTask] with the
* Java and Kotlin sources merged in.
* Java, Kotlin sources and META folder merged in.
*/
internal fun JvmCompilationTask.expandWithSourceJarSources(): JvmCompilationTask =
if (inputs.sourceJarsList.isEmpty()) {
this
} else {
expandWithSources(
SourceJarExtractor(
destDir = Paths.get(directories.temp).resolve("_srcjars"),
fileMatcher = IS_JVM_SOURCE_FILE,
destDir = Paths.get(directories.temp).resolve(SOURCE_JARS_DIR),
fileMatcher = { str: String -> IS_JVM_SOURCE_FILE.test(str) || "/$MANIFEST_DIR" in str },
Comment on lines 452 to +455
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not fully aware of the whole compilation process but if we add this logic here, it will also run for jars generated by kapt (as it's based on whole input.sourceJarsList). Since the issue for missing manifest files seems to be related to ksp only, could we possibly put this expand logic in a place where it wouldn't affect kapt generated jars? 💭

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zalewskise I think we actually want this for Kapt as wellsince the meta directory could contain files needed in the final jar. One example that comes to mind is google-auto-service where it spits out information that's needed in the final jar.

).also {
it.jarFiles.addAll(inputs.sourceJarsList.map { p -> Paths.get(p) })
it.execute()
Expand Down Expand Up @@ -485,10 +490,12 @@ fun JvmCompilationTask.expandWithGeneratedSources(): JvmCompilationTask =

private fun JvmCompilationTask.expandWithSources(sources: Iterator<String>): JvmCompilationTask =
updateBuilder { builder ->
sources.filterOutNonCompilableSources().partitionJvmSources(
{ builder.inputsBuilder.addKotlinSources(it) },
{ builder.inputsBuilder.addJavaSources(it) },
)
sources.copyManifestFilesToGeneratedClasses(directories)
.filterOutNonCompilableSources()
.partitionJvmSources(
{ builder.inputsBuilder.addKotlinSources(it) },
{ builder.inputsBuilder.addJavaSources(it) },
)
}

private fun JvmCompilationTask.updateBuilder(
Expand All @@ -499,6 +506,29 @@ private fun JvmCompilationTask.updateBuilder(
it.build()
}

/**
* Copy generated manifest files from KSP task into generated folder
*/
internal fun Iterator<String>.copyManifestFilesToGeneratedClasses(
directories: Directories,
): Iterator<String> {
val result = mutableSetOf<String>()
this.forEach {
if ("/$MANIFEST_DIR" in it) {
val path = Paths.get(it)
val srcJarsPath = Paths.get(directories.temp, SOURCE_JARS_DIR)
if (srcJarsPath.exists()) {
val relativePath = srcJarsPath.relativize(path)
val destPath = Paths.get(directories.generatedClasses).resolve(relativePath)
destPath.parent.toFile().mkdirs()
Files.copy(path, destPath, StandardCopyOption.REPLACE_EXISTING)
}
}
result.add(it)
}
return result.iterator()
}

/**
* Only keep java and kotlin files for the iterator. Filter our all other non-compilable files.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package io.bazel.kotlin.builder.utils.jars

import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.jar.JarFile

open class JarExtractor protected constructor(
Expand All @@ -42,7 +43,7 @@ open class JarExtractor protected constructor(
Files.createDirectories(target)
else -> jar.getInputStream(entry).use {
Files.createDirectories(target.parent)
Files.copy(it, target)
Files.copy(it, target, StandardCopyOption.REPLACE_EXISTING)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happened here to need the REPLACE_EXISTING attribute?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since we are using the same generated classes directory for all the jar creation processes during KotlinCompile when its trying to copy there is a java.nio.file.FileAlreadyExistsException. This is to avoid that.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think one common file that would be a duplicate is META-INF/MANIFEST.MF.

}
}
}
Expand Down
5 changes: 5 additions & 0 deletions src/main/starlark/core/repositories/setup.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ def kt_configure():
"org.pantsbuild:jarjar:1.7.2",
"org.jetbrains.kotlinx:atomicfu-js:0.15.2",
"org.jetbrains.kotlinx:kotlinx-serialization-runtime:1.0-M1-1.4.0-rc",
"dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
"com.squareup.moshi:moshi:1.15.0",
"com.squareup.moshi:moshi-kotlin:1.15.0",
"com.squareup.moshi:moshi-kotlin-codegen:1.15.0",
"com.google.auto.service:auto-service-annotations:jar:1.1.1",
],
repositories = [
"https://maven-central.storage.googleapis.com/repos/central/data/",
Expand Down
64 changes: 64 additions & 0 deletions src/test/data/jvm/ksp/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
load("//kotlin:core.bzl", "kt_ksp_plugin")
load("//kotlin:jvm.bzl", "kt_jvm_library")

# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//visibility:private"])

kt_ksp_plugin(
name = "autoservice",
processor_class = "dev.zacsweers.autoservice.ksp.AutoServiceSymbolProcessor$Provider",
deps = [
"@kotlin_rules_maven//:dev_zacsweers_autoservice_auto_service_ksp",
],
)

kt_ksp_plugin(
name = "moshi",
processor_class = "com.squareup.moshi.kotlin.codegen.ksp.JsonClassSymbolProcessorProvider",
deps = [
"@kotlin_rules_maven//:com_squareup_moshi_moshi",
"@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin",
"@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin_codegen",
],
)

kt_jvm_library(
name = "moshi_lib",
srcs = ["CoffeeAppModel.kt"],
plugins = [":moshi"],
deps = [
"@kotlin_rules_maven//:com_squareup_moshi_moshi",
"@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin",
"@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin_codegen",
],
)

kt_jvm_library(
name = "coffee_lib",
srcs = ["CoffeeAppService.java"],
plugins = [":autoservice"],
deps = [
"@kotlin_rules_maven//:dev_zacsweers_autoservice_auto_service_ksp",
],
)

filegroup(
name = "ksp",
srcs = [
":coffee_lib.jar",
":moshi_lib.jar",
],
visibility = ["//visibility:public"],
)
29 changes: 29 additions & 0 deletions src/test/data/jvm/ksp/CoffeeApp.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright 2018 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package coffee

import com.squareup.moshi.Moshi

class CoffeeApp {

companion object {

private val adapter = CoffeeAppModelJsonAdapter(Moshi.Builder().build())
private val d = AutoValue_CoffeeAppJavaModel.Builder()
.setCoffeeAppModel(CoffeeAppModel("1"))
.build()
}
}
21 changes: 21 additions & 0 deletions src/test/data/jvm/ksp/CoffeeAppModel.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright 2018 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package src.test.data.jvm.ksp
Bencodes marked this conversation as resolved.
Show resolved Hide resolved

import com.squareup.moshi.JsonClass

@JsonClass(generateAdapter = true)
data class CoffeeAppModel(val id: Int)
22 changes: 22 additions & 0 deletions src/test/data/jvm/ksp/CoffeeAppService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Copyright 2018 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package src.test.data.jvm.ksp;
Bencodes marked this conversation as resolved.
Show resolved Hide resolved

import com.google.auto.service.AutoService;

@AutoService(Object.class)
public class CoffeeAppService {
}
7 changes: 7 additions & 0 deletions src/test/kotlin/io/bazel/kotlin/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,20 @@ kt_rules_e2e_test(
srcs = ["KotlinJvm13Test.kt"],
)

kt_rules_e2e_test(
name = "KotlinJvmKspAssertionTest",
srcs = ["KotlinJvmKspAssertionTest.kt"],
data = ["//src/test/data/jvm/ksp"],
)

test_suite(
name = "assertion_tests",
tests = [
"KotlinJvm13Test",
"KotlinJvmAssociatesBasicVisibilityTest",
"KotlinJvmBasicAssertionTest",
"KotlinJvmKaptAssertionTest",
"KotlinJvmKspAssertionTest",
],
)

Expand Down
9 changes: 9 additions & 0 deletions src/test/kotlin/io/bazel/kotlin/KotlinAssertionTestCase.kt
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ abstract class KotlinAssertionTestCase(root: String) : BasicAssertionTestCase()
}
}

protected fun JarFile.assertContainsExactEntries(vararg want: String) {
val got = this.entries().asSequence().map { it.name }.toSet()
val missing = want.toSet() - got
val extra = got - want.toSet()
check(missing.isEmpty() && extra.isEmpty()) {
"Entries Missing: $missing \nFrom: $got \nEntries Extra: $extra \nFrom: $got"
}
}

/**
* Validated the entry is compressed and has the DOS epoch for it's timestamp.
*/
Expand Down
Loading