add librarianGenerateBintrayGroups task

README.md

@@ -87,6 +87,8 @@ apply plugin: 'librarian-preset'
     - generate artifacts
   - `librarianGenerateGroups`
     - generate groups
+  - `librarianGenerateBintrayGroups`
+    - generate groups using by Bintray package information
   - `librarianGeneratePages`
     - generate page, must execute after `librarianGenerateArtifacts`
   - `librarianGeneratePipeline`
@@ -106,6 +108,7 @@ librarian {
     depth = "firstLevel" // String, firstLevel or allLevel, default value is firstLevel
     failOnGeneratePageWhenFoundPlaceholder = true // Boolean, default value is true
     failOnOverrideUnMatchedLicense = true // Boolean, default value is true
+    useBintray = true // Boolean, default value is true
     additionalModules = [] // Array of String
 
     pages {
@@ -146,6 +149,7 @@ librarian {
 |librarian.depth|search dependency depth, firstLevel find your directly dependency|
 |librarian.failOnGeneratePageWhenFoundPlaceholder|fail on `librarianGeneratePages` when found placeholder|
 |librarian.failOnOverrideUnMatchedLicense|fail on override un matched license by group|
+|librarian.useBintray|if true, actually use Bintray api at `librarianGenerateBintrayGroups` task`|
 |librarian.additionalModules|additional resolve modules, use when like android dynamic feature module|
 
 ### Generate Notice Page
@@ -154,6 +158,7 @@ librarian {
    - `librarianShowConfigurations` task helps when configure your project
 1. execute `librarianGenerateArtifacts` task
 1. execute `librarianGeneratePresetGroups` task if using `librarian-preset`
+1. execute `librarianGenerateBintrayGroups` task if using Bintray package information
 1. execute `librarianGeneratePages` task
 1. if output error or incomplete result, configure your project that put `groups` and execute `librarianGenerateGroups` task
    - then execute `librarianGeneratePages` task

build.gradle

@@ -30,6 +30,9 @@ buildscript {
             classpath Dependencies.TikXml.core
             classpath Dependencies.TikXml.coreAnnotation
             classpath Dependencies.SquareUp.okio
+            classpath Dependencies.SquareUp.okhttp
+            classpath Dependencies.SquareUp.retrofit
+            classpath Dependencies.JakeWharton.retrofitKotlinSerializationConverter
         } else {
             classpath Dependencies.Librarian.pluginCore
         }

buildSrc/src/main/kotlin/net/meilcli/librarian/gradle/Dependencies.kt

@@ -76,5 +76,16 @@ object Dependencies {
         val okio = Dependency("com.squareup.okio", "okio", "2.5.0")
         val moshi = Dependency("com.squareup.moshi", "moshi", "1.9.2")
         val moshiKapt = Dependency("com.squareup.moshi", "moshi-kotlin-codegen", "1.9.2")
+        val okhttp = Dependency("com.squareup.okhttp3", "okhttp", "4.5.0")
+        val retrofit = Dependency("com.squareup.retrofit2", "retrofit", "2.8.1")
+    }
+
+    object JakeWharton {
+
+        val retrofitKotlinSerializationConverter = Dependency(
+            "com.jakewharton.retrofit",
+            "retrofit2-kotlinx-serialization-converter",
+            "0.5.0"
+        )
     }
 }

plugin-core/build.gradle

@@ -14,6 +14,10 @@ dependencies {
     implementation Dependencies.TikXml.core
     implementation Dependencies.TikXml.coreAnnotation
     implementation Dependencies.SquareUp.okio
+    implementation Dependencies.SquareUp.okhttp
+    implementation Dependencies.SquareUp.retrofit
+    implementation Dependencies.JakeWharton.retrofitKotlinSerializationConverter
+
     kapt Dependencies.TikXml.processor
 }
 

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/LibrarianExtension.kt

@@ -31,6 +31,8 @@ open class LibrarianExtension @Inject constructor(
 
     var failOnOverrideUnMatchedLicense = true
 
+    var useBintray = true
+
     var additionalModules = mutableListOf<String>()
 
     val pages = objectFactory.domainObjectContainer(LibrarianPageExtension::class.java)

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/LibrarianPlugin.kt

@@ -13,6 +13,7 @@ open class LibrarianPlugin : Plugin<Project> {
         private const val librarianExtension = "librarian"
         const val generateArtifactsTask = "librarianGenerateArtifacts"
         const val generateGroupsTask = "librarianGenerateGroups"
+        const val generateBintrayGroupsTask = "librarianGenerateBintrayGroups"
         const val generatePagesTask = "librarianGeneratePages"
     }
 
@@ -28,11 +29,16 @@ open class LibrarianPlugin : Plugin<Project> {
             this.extension = extension
         }
 
-        generatePipelineDependTasks += project.createTask<GenerateGroupsTask>(generateGroupsTask).apply {
+        generatePipelineDependTasks += project.createTask<GenerateBintrayGroupsTask>(generateBintrayGroupsTask).apply {
             this.extension = extension
             mustRunAfter(generateArtifactsTask, "librarianGeneratePresetGroups")
         }
 
+        generatePipelineDependTasks += project.createTask<GenerateGroupsTask>(generateGroupsTask).apply {
+            this.extension = extension
+            mustRunAfter(generateArtifactsTask, generateBintrayGroupsTask, "librarianGeneratePresetGroups")
+        }
+
         generatePipelineDependTasks += project.createTask<GeneratePagesTask>(generatePagesTask).apply {
             this.extension = extension
             mustRunAfter(generateGroupsTask)

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/entities/BintrayLicense.kt

@@ -0,0 +1,16 @@
+package net.meilcli.librarian.plugin.entities
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class BintrayLicense(
+    @SerialName("name")
+    val name: String,
+
+    @SerialName("longname")
+    val fullName: String,
+
+    @SerialName("url")
+    val url: String
+)

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/entities/BintrayPackage.kt

@@ -0,0 +1,28 @@
+package net.meilcli.librarian.plugin.entities
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class BintrayPackage(
+    @SerialName("repo")
+    val repository: String,
+
+    @SerialName("owner")
+    val owner: String,
+
+    @SerialName("desc")
+    val description: String? = null,
+
+    @SerialName("licenses")
+    val licenses: List<String> = emptyList(),
+
+    @SerialName("github_repo")
+    val githubRepository: String? = null,
+
+    @SerialName("vcs_url")
+    val vcsUrl: String? = null,
+
+    @SerialName("website_url")
+    val websiteUrl: String? = null
+)

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/entities/BintrayRepository.kt

@@ -0,0 +1,3 @@
+package net.meilcli.librarian.plugin.entities
+
+data class BintrayRepository(val url: String, val repository: String, val owner: String)

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/entities/BintraySearchResult.kt

@@ -0,0 +1,19 @@
+package net.meilcli.librarian.plugin.entities
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class BintraySearchResult(
+    @SerialName("name")
+    val packageName: String,
+
+    @SerialName("owner")
+    val owner: String,
+
+    @SerialName("repo")
+    val repository: String
+) {
+
+    fun urlEscapedPackageName() = packageName.replace(":", "&3A")
+}

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/internal/artifacts/ArtifactsByLibraryGroupsFilter.kt

@@ -0,0 +1,14 @@
+package net.meilcli.librarian.plugin.internal.artifacts
+
+import net.meilcli.librarian.plugin.entities.Artifact
+import net.meilcli.librarian.plugin.entities.LibraryGroup
+import net.meilcli.librarian.plugin.internal.IFilter
+
+class ArtifactsByLibraryGroupsFilter(
+    private val libraryGroups: List<LibraryGroup>
+) : IFilter<Collection<Artifact>> {
+
+    override fun filter(source: Collection<Artifact>): Collection<Artifact> {
+        return source.filter { x -> libraryGroups.all { y -> y.artifacts.contains(x.artifact).not() } }
+    }
+}

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/internal/bintray/ArtifactAndBintrayRepositoryToBintrayPackageLoader.kt

@@ -0,0 +1,87 @@
+package net.meilcli.librarian.plugin.internal.bintray
+
+import net.meilcli.librarian.plugin.entities.Artifact
+import net.meilcli.librarian.plugin.entities.BintrayPackage
+import net.meilcli.librarian.plugin.entities.BintrayRepository
+import net.meilcli.librarian.plugin.internal.IParameterizedLoader
+import org.gradle.api.Project
+import org.slf4j.LoggerFactory
+
+class ArtifactAndBintrayRepositoryToBintrayPackageLoader(
+    private val project: Project
+) : IParameterizedLoader<Pair<Artifact, BintrayRepository>, BintrayPackage?> {
+
+    private val api = IBintrayApi.default
+    private val logger = LoggerFactory.getLogger(ArtifactAndBintrayRepositoryToBintrayPackageLoader::class.java)
+    private val resultCache = mutableMapOf<Pair<Artifact, BintrayRepository>, BintrayPackage?>()
+    private var notService = false
+
+    override fun load(parameter: Pair<Artifact, BintrayRepository>): BintrayPackage? {
+        if (notService) {
+            project.logger.quiet("not service hit")
+            return null
+        }
+        if (resultCache.containsKey(parameter)) {
+            project.logger.quiet("cache hit")
+            return resultCache[parameter]
+        }
+
+        try {
+            // search package name, because name convention is difference each repository
+            val searchResponse = api
+                .searchPackage(
+                    group = parameter.first.group,
+                    artifact = parameter.first.name,
+                    owner = parameter.second.owner,
+                    repository = parameter.second.repository
+                )
+                .execute()
+
+            if (500 <= searchResponse.code()) {
+                notService = true
+                return null
+            }
+            if (searchResponse.isSuccessful.not()) {
+                resultCache[parameter] = null
+                return null
+            }
+
+            val searchResults = searchResponse.body()
+            if (searchResults == null) {
+                resultCache[parameter] = null
+                return null
+            }
+            val searchResult = searchResults.find {
+                it.owner == parameter.second.owner && it.repository == parameter.second.repository
+            }
+            if (searchResult == null) {
+                resultCache[parameter] = null
+                return null
+            }
+
+            val packageResponse = api
+                .getPackage(
+                    owner = parameter.second.owner,
+                    repository = parameter.second.repository,
+                    packageName = searchResult.packageName
+                )
+                .execute()
+
+            if (500 <= packageResponse.code()) {
+                notService = true
+                return null
+            }
+            if (packageResponse.isSuccessful.not()) {
+                resultCache[parameter] = null
+                return null
+            }
+
+            resultCache[parameter] = packageResponse.body()
+            return resultCache[parameter]
+        } catch (exception: Exception) {
+            logger.warn("Librarian unknown exception", exception)
+            project.logger.quiet("exception")
+            return null
+        }
+    }
+}

plugin-core/src/main/kotlin/net/meilcli/librarian/plugin/internal/bintray/ArtifactToBintrayRepositoryLoader.kt

@@ -0,0 +1,64 @@
+package net.meilcli.librarian.plugin.internal.bintray
+
+import net.meilcli.librarian.plugin.entities.Artifact
+import net.meilcli.librarian.plugin.entities.BintrayRepository
+import net.meilcli.librarian.plugin.internal.ILoader
+import net.meilcli.librarian.plugin.internal.IParameterizedLoader
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import org.slf4j.LoggerFactory
+
+class ArtifactToBintrayRepositoryLoader(
+    bintrayRepositoriesLoader: ILoader<List<BintrayRepository>>
+) : IParameterizedLoader<Artifact, BintrayRepository?> {
+
+    private val client = OkHttpClient()
+    private val logger = LoggerFactory.getLogger(ArtifactToBintrayRepositoryLoader::class.java)
+    private val bintrayRepositories = bintrayRepositoriesLoader.load()
+    private val notServiceRepositories = mutableListOf<BintrayRepository>()
+    private val resultCache = mutableMapOf<Artifact, BintrayRepository?>()
+
+    override fun load(parameter: Artifact): BintrayRepository? {
+        if (resultCache.containsKey(parameter)) {
+            return resultCache[parameter]
+        }
+        for (bintrayRepository in bintrayRepositories) {
+            if (notServiceRepositories.contains(bintrayRepository)) {
+                continue
+            }
+
+            val httpStatusCode = try {
+                head(bintrayRepository, parameter)
+            } catch (exception: Exception) {
+                logger.warn("Librarian: unknown exception when head bintray", exception)
+                400
+            }
+            if (500 <= httpStatusCode) {
+                notServiceRepositories += bintrayRepository
+                continue
+            }
+            if (300 <= httpStatusCode) {
+                continue
+            }
+            resultCache[parameter] = bintrayRepository
+            return bintrayRepository
+        }
+
+        return null
+    }
+
+    private fun head(bintrayRepository: BintrayRepository, artifact: Artifact): Int {
+        fun String.replaceDot(): String {
+            return replace('.', '/')
+        }
+
+        val request = Request.Builder()
+            .head()
+            .url("${bintrayRepository.url}/${artifact.group.replaceDot()}/${artifact.name}/${artifact.version}/${artifact.name}-${artifact.version}.pom")
+            .build()
+
+        client.newCall(request).execute().use {
+            return it.code
+        }
+    }
+}