Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make updateInstances task compatible with configuration cache #915

Merged
merged 2 commits into from
Jun 30, 2023
Merged

Make updateInstances task compatible with configuration cache #915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3d5c0a3
Make updateInstances task compatible with configuration cache
AppearamidGuy Jun 29, 2023
5b76b42
Merge branch 'main' into config-cache
MV-GH Jun 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ It uses the fediverse api and filters on the monthly users.
You can run it by doing

```shell
./gradlew app:updateInstances --no-configuration-cache
./gradlew app:updateInstances
```

## Generate compose compiler metrics
Expand Down
197 changes: 108 additions & 89 deletions app/update_instances.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,37 +9,55 @@ import org.apache.groovy.json.internal.LazyMap
import groovy.json.JsonOutput
import groovy.json.JsonSlurper

// All lemmy instances with at least this amount of monthly active users will be included.
val minimumMAU = 50

// Run this as `./gradlew app:updateInstances`
tasks.register<UpdateInstancesTask>("updateInstances") {
description = "Fetches a list of popular Lemmy instances and writes it to the DefaultInstances.kt file"

val endpointUrl = "https://api.fediverse.observer/"
val instancesFilePath = "src/main/java/com/jerboa/DefaultInstances.kt"
val manifestPath = "src/main/AndroidManifest.xml"
val START_TAG = "<!--#AUTO_GEN_INSTANCE_LIST_DO_NOT_TOUCH#-->"
val END_TAG = "<!--#INSTANCE_LIST_END#-->"
val IDENT = 14
val nsfwList = listOf("lemmynsfw.com")
// All lemmy instances with at least this amount of monthly active users will be included.
minimumMAU.set(50)

// Some extension methods to make the JsonSlurper output easier to process
fun LazyMap.getMap(key: String): LazyMap {
return this[key] as LazyMap
endpointUrl.set("https://api.fediverse.observer/")
instancesFile.set(file("src/main/java/com/jerboa/DefaultInstances.kt"))
manifestFile.set(file("src/main/AndroidManifest.xml"))
nsfwList.set(listOf("lemmynsfw.com"))
}

fun LazyMap.getArray(key: String): ArrayList<*> {
return this[key] as ArrayList<*>
}
@UntrackedTask(because = "Output depends on api response")
abstract class UpdateInstancesTask: DefaultTask() {
private companion object {
const val START_TAG = "<!--#AUTO_GEN_INSTANCE_LIST_DO_NOT_TOUCH#-->"
const val END_TAG = "<!--#INSTANCE_LIST_END#-->"
const val INDENT = 14
}

@Suppress("UNCHECKED_CAST")
fun <T> LazyMap.getAs(key: String): T {
return this[key] as T
}
@get:Input
abstract val minimumMAU: Property<Int>
@get:Input
abstract val endpointUrl: Property<String>
@get:Input
abstract val nsfwList: ListProperty<String>

@get:OutputFile
abstract val instancesFile: RegularFileProperty
@get:OutputFile
abstract val manifestFile: RegularFileProperty

// Some extension methods to make the JsonSlurper output easier to process
fun LazyMap.getMap(key: String): LazyMap {
return this[key] as LazyMap
}

// Run this as `./gradlew app:updateInstances --no-configuration-cache`
tasks.register("updateInstances") {
description = "Fetches a list of popular Lemmy instances and writes it to the DefaultInstances.kt file"
fun LazyMap.getArray(key: String): ArrayList<*> {
return this[key] as ArrayList<*>
}

doFirst {
@Suppress("UNCHECKED_CAST")
fun <T> LazyMap.getAs(key: String): T {
return this[key] as T
}

@TaskAction
fun action() {
// Get sorted list of nodes
val nodes = getData()
.getMap("data")
Expand All @@ -51,7 +69,7 @@ tasks.register("updateInstances") {
Pair(name, users ?: 0)
}
.filter {
it.second >= minimumMAU && !nsfwList.contains(it.first)
it.second >= minimumMAU.get() && !nsfwList.get().contains(it.first)
}
.sortedBy {
it.second
Expand All @@ -61,82 +79,83 @@ tasks.register("updateInstances") {
updateInstanceList(nodes)
updateManifest(nodes.map { it.first })
}
}


fun getData(): LazyMap {
val url = URL(endpointUrl)
val query = """
{
nodes(softwarename: "lemmy") {
domain
active_users_monthly
}
}"""

// Format JSON request body
val body = JsonOutput.toJson(mapOf("query" to query))

// Create POST request
val req = url.openConnection() as HttpURLConnection
req.requestMethod = "POST"
req.doOutput = true
req.setRequestProperty("Content-Type", "application/json")

// Write body to request
OutputStreamWriter(req.outputStream, "UTF-8").use {
it.write(body)
}

// Get response and JSON parse it
return JsonSlurper().parse(req.inputStream.reader()) as LazyMap
}
private fun getData(): LazyMap {
val url = URL(endpointUrl.get())
val query = """
{
nodes(softwarename: "lemmy") {
domain
active_users_monthly
}
}""".trimIndent()

// Format JSON request body
val body = JsonOutput.toJson(mapOf("query" to query))

// Create POST request
val req = url.openConnection() as HttpURLConnection
req.requestMethod = "POST"
req.doOutput = true
req.setRequestProperty("Content-Type", "application/json")

// Write body to request
OutputStreamWriter(req.outputStream, "UTF-8").use {
it.write(body)
}

fun updateInstanceList(nodes: List<Pair<String, Int>>) {
// Create output file and write header
val outFile = file(instancesFilePath)
outFile.writeText(
"""package com.jerboa
// Get response and JSON parse it
return JsonSlurper().parse(req.inputStream.reader()) as LazyMap
}

val DEFAULT_LEMMY_INSTANCES = arrayOf(
"""
)
fun updateInstanceList(nodes: List<Pair<String, Int>>) {
// Create output file and write header
val outFile = instancesFile.get().asFile
outFile.writeText("""
package com.jerboa

val DEFAULT_LEMMY_INSTANCES = arrayOf(

""".trimIndent()
)

// Write each node's name, one per line
for (n in nodes) {
outFile.appendText(" \"${n.first}\", // ${n.second} monthly users\n")
}

// Write each node's name, one per line
for (n in nodes) {
outFile.appendText(" \"${n.first}\", // ${n.second} monthly users\n")
outFile.appendText(")\n")
}

outFile.appendText(")\n")
}


fun updateManifest(list: List<String>) {
val manifest = file(manifestPath)
val lines = manifest.readLines()
manifest.writeText("")
fun updateManifest(list: List<String>) {
val manifest = manifestFile.get().asFile
val lines = manifest.readLines()
manifest.writeText("")

var skip = false
var skip = false

for (line in lines) {
if (line.trim() == START_TAG) {
skip = true
manifest.appendText(" ".repeat(IDENT) + START_TAG)
manifest.appendText(genManifestHosts(list))
manifest.appendText(" ".repeat(IDENT) + END_TAG + System.lineSeparator())
} else if (line.trim() == END_TAG) {
skip = false
} else if (!skip) {
manifest.appendText(line + System.lineSeparator())
for (line in lines) {
if (line.trim() == START_TAG) {
skip = true
manifest.appendText(" ".repeat(INDENT) + START_TAG)
manifest.appendText(genManifestHosts(list))
manifest.appendText(" ".repeat(INDENT) + END_TAG + System.lineSeparator())
} else if (line.trim() == END_TAG) {
skip = false
} else if (!skip) {
manifest.appendText(line + System.lineSeparator())
}
}
}
}

fun genManifestHosts(list: List<String>): String {
return list.joinToString(
separator = System.lineSeparator(),
prefix = System.lineSeparator(),
postfix = System.lineSeparator(),
) { " ".repeat(IDENT) + "<data android:host=\"$it\"/>" }
fun genManifestHosts(list: List<String>): String {
return list.joinToString(
separator = System.lineSeparator(),
prefix = System.lineSeparator(),
postfix = System.lineSeparator(),
) { " ".repeat(INDENT) + "<data android:host=\"$it\"/>" }
}
}