diff --git a/README.md b/README.md index 553464bbb9..0d4ead4e66 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,6 @@ gcloud: ## Invoke a test asynchronously without waiting for test results. # async: false - ## The billing enabled Google Cloud Platform project name to use - # project: delta-essence-114723 - ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history @@ -151,6 +148,9 @@ flank: ## test targets - a list of tests to run. omit to run all tests. # test-targets: # - className/testName + + ## The billing enabled Google Cloud Platform project name to use + # project: delta-essence-114723 ``` ### Android example @@ -186,9 +186,6 @@ gcloud: ## Invoke a test asynchronously without waiting for test results. # async: false - ## The billing enabled Google Cloud Platform project name to use - # project: delta-essence-114723 - ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history @@ -285,6 +282,9 @@ flank: ## regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt # files-to-download: # - .*\.mp4$ + + ## The billing enabled Google Cloud Platform project name to use + # project: delta-essence-114723 ``` ### Android code coverage diff --git a/release_notes.md b/release_notes.md index 27e9e5320b..c72dc37f3a 100644 --- a/release_notes.md +++ b/release_notes.md @@ -4,7 +4,7 @@ - [#507](https://github.com/TestArmada/flank/pull/507) Improve error message when credentials fail to load, folder doesn't exist, and on bucket creation failure. Properly pass through user credential when checking the storage bucket. ([bootstraponline](https://github.com/bootstraponline)) - [#514](https://github.com/TestArmada/flank/pull/514) Rename `testShards` to `maxTestShards` ([miguelslemos](https://github.com/miguelslemos)) - [#518](https://github.com/TestArmada/flank/pull/518) Add deprecation warnings when old key names are used. `flank android doctor --fix` will auto fix the YAML file. ([bootstraponline](https://github.com/bootstraponline)) -- [#519](https://github.com/TestArmada/flank/pull/519) Rename `maxTestShards` to `max-test-shards`, `shardTime` to `shard-time`, `repeatTests` to `repeat-tests`, `smartFlankGcsPath` to `smart-flank-gcs-path`, `disableSharding` to `disable-sharding`. ([bootstraponline](https://github.com/bootstraponline)) +- [#519](https://github.com/TestArmada/flank/pull/519) Rename `maxTestShards` to `max-test-shards`, `shardTime` to `shard-time`, `repeatTests` to `repeat-tests`, `smartFlankGcsPath` to `smart-flank-gcs-path`, `disableSharding` to `disable-sharding`. Moved `project` from `gcloud` to `flank` ([bootstraponline](https://github.com/bootstraponline)) ## v4.4.0 diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index d55e37b04a..435bb8cb21 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -22,9 +22,6 @@ gcloud: ## Invoke a test asynchronously without waiting for test results. # async: false - ## The billing enabled Google Cloud Platform project name to use - # project: delta-essence-114723 - ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history @@ -102,3 +99,6 @@ flank: ## test targets - a list of tests to run. omit to run all tests. # test-targets: # - className/testName + + ## The billing enabled Google Cloud Platform project name to use + # project: delta-essence-114723 diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 1e58ac42ce..d5f0cce66f 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -22,9 +22,6 @@ gcloud: ## Invoke a test asynchronously without waiting for test results. # async: false - ## The billing enabled Google Cloud Platform project name to use - # project: delta-essence-114723 - ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history @@ -121,3 +118,6 @@ flank: ## regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt # files-to-download: # - .*\.mp4$ + + ## The billing enabled Google Cloud Platform project name to use + # project: delta-essence-114723 diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index cd7096a9a7..cf79898023 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -47,7 +47,6 @@ class AndroidArgs( override val recordVideo = cli?.recordVideo ?: cli?.noRecordVideo?.not() ?: gcloud.recordVideo override val testTimeout = cli?.timeout ?: gcloud.timeout override val async = cli?.async ?: gcloud.async - override val project = cli?.project ?: gcloud.project override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts @@ -72,6 +71,7 @@ class AndroidArgs( override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding + override val project = cli?.project ?: flank.project // computed properties not specified in yaml override val testShardChunks: List> by lazy { @@ -145,7 +145,6 @@ AndroidArgs record-video: $recordVideo timeout: $testTimeout async: $async - project: $project results-history-name: $resultsHistoryName # Android gcloud app: $appApk @@ -173,6 +172,7 @@ ${listToString(filesToDownload)} test-targets-always-run: ${listToString(testTargetsAlwaysRun)} disable-sharding: $disableSharding + project: $project """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index b92276516a..4c4b70b854 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -16,7 +16,9 @@ import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX import ftl.config.FtlConstants.JSON_FACTORY import ftl.config.FtlConstants.defaultCredentialPath +import ftl.config.FtlConstants.useMock import ftl.gc.GcStorage +import ftl.gc.GcToolResults import ftl.reports.xml.model.JUnitTestResult import ftl.shard.Shard import ftl.shard.StringShards @@ -105,9 +107,9 @@ object ArgsHelper { } fun createGcsBucket(projectId: String, bucket: String): String { - // com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list - // when testing, use a hard coded results bucket instead. - if (FtlConstants.useMock) return bucket + if (bucket.isEmpty()) return GcToolResults.getDefaultBucket(projectId) ?: throw RuntimeException("Failed to make bucket for $projectId") + if (useMock) return bucket + // test lab supports using a special free storage bucket // because we don't have access to the root account, it won't show up in the storage list. if (bucket.startsWith("test-lab-")) return bucket diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 2156a72225..49b7637875 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -38,7 +38,6 @@ class IosArgs( override val recordVideo = cli?.recordVideo ?: cli?.noRecordVideo?.not() ?: gcloud.recordVideo override val testTimeout = cli?.timeout ?: gcloud.timeout override val async = cli?.async ?: gcloud.async - override val project = cli?.project ?: gcloud.project override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts @@ -56,6 +55,7 @@ class IosArgs( override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload override val disableSharding = cli?.disableSharding ?: flank.disableSharding + override val project = cli?.project ?: flank.project private val iosFlank = iosFlankYml.flank val testTargets = cli?.testTargets ?: iosFlank.testTargets @@ -109,7 +109,6 @@ IosArgs record-video: $recordVideo timeout: $testTimeout async: $async - project: $project results-history-name: $resultsHistoryName # iOS gcloud test: $xctestrunZip @@ -132,6 +131,7 @@ ${listToString(filesToDownload)} test-targets: ${listToString(testTargets)} disable-sharding: $disableSharding + project: $project """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 058c9bb696..7cf41a6ae3 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -2,7 +2,10 @@ package ftl.args.yml import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import ftl.args.ArgsHelper +import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX +import ftl.util.Utils import ftl.util.Utils.fatalError /** Flank specific parameters for both iOS and Android */ @@ -27,16 +30,24 @@ class FlankYmlParams( val testTargetsAlwaysRun: List = emptyList(), @field:JsonProperty("files-to-download") - val filesToDownload: List = emptyList() + val filesToDownload: List = emptyList(), + + val project: String = ArgsHelper.getDefaultProjectId() ?: "" ) { companion object : IYmlKeys { override val keys = listOf( "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "disable-sharding", - "test-targets-always-run", "files-to-download" + "test-targets-always-run", "files-to-download", "project" ) } init { + Utils.assertNotEmpty( + project, "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + + "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" + ) + if (maxTestShards <= 0 && maxTestShards != -1) fatalError("max-test-shards must be >= 1 or -1") if (shardTime <= 0 && shardTime != -1) fatalError("shard-time must be >= 1 or -1") if (repeatTests < 1) fatalError("repeat-tests must be >= 1") diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt index 4f4235cd97..ffdec1219e 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt @@ -2,10 +2,6 @@ package ftl.args.yml import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty -import ftl.args.ArgsHelper.getDefaultProjectId -import ftl.config.FtlConstants.defaultCredentialPath -import ftl.gc.GcToolResults -import ftl.util.Utils.assertNotEmpty /** * Common Gcloud parameters shared between iOS and Android @@ -28,8 +24,6 @@ class GcloudYmlParams( val async: Boolean = false, - val project: String = getDefaultProjectId() ?: "", - @field:JsonProperty("results-history-name") val resultsHistoryName: String? = null, @@ -38,20 +32,9 @@ class GcloudYmlParams( ) { companion object : IYmlKeys { override val keys = - listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project", + listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "results-history-name", "flaky-test-attempts") } - - init { - assertNotEmpty( - project, "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + - "or save service account credential to $defaultCredentialPath\n" + - " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" - ) - - if (resultsBucket.isEmpty()) resultsBucket = GcToolResults.getDefaultBucket(project) ?: "" - assertNotEmpty(resultsBucket, "results-bucket is not set") - } } @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/YamlDeprecated.kt b/test_runner/src/main/kotlin/ftl/args/yml/YamlDeprecated.kt index e4d8da7bab..4fc3f59957 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/YamlDeprecated.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/YamlDeprecated.kt @@ -1,6 +1,7 @@ package ftl.args.yml import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory import com.fasterxml.jackson.databind.node.ObjectNode import ftl.args.ArgsHelper.yamlMapper import ftl.util.Utils @@ -52,6 +53,11 @@ object YamlDeprecated { Key(Parent.flank, "disableSharding"), Key(Parent.flank, "disable-sharding"), Level.Warning + ), + ModifiedKey( + Key(Parent.gcloud, "project"), + Key(Parent.flank, "project"), + Level.Warning ) ) @@ -65,7 +71,10 @@ object YamlDeprecated { } private fun JsonNode.replace(parent: Parent, child: String, value: JsonNode) { - (this[parent.toString()] as ObjectNode).replace(child, value) + val parentKey = parent.toString() + // if the parent node ('flank:') doesn't exist then add it ('flank: {}') to the YAML + if (this[parentKey] == null) (this as ObjectNode).set(parentKey, JsonNodeFactory.instance.objectNode()) + (this[parentKey] as ObjectNode).replace(child, value) } private fun mutate(parsed: JsonNode, changes: List) { diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index f14502c1b1..14d928611d 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -162,8 +162,7 @@ class AndroidArgsFileTest { val config = AndroidArgs( GcloudYml( GcloudYmlParams( - resultsBucket = oldConfig.resultsBucket, - project = "delta-essence-114723" + resultsBucket = oldConfig.resultsBucket ) ), AndroidGcloudYml( @@ -172,7 +171,11 @@ class AndroidArgsFileTest { test = oldConfig.testApk ) ), - FlankYml(), + FlankYml( + FlankYmlParams( + project = "delta-essence-114723" + ) + ), "" ) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index db88e5673d..c9bec657e9 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -186,7 +186,6 @@ AndroidArgs record-video: false timeout: 70m async: true - project: projectFoo results-history-name: android-history # Android gcloud app: $appApkAbsolutePath @@ -226,6 +225,7 @@ AndroidArgs - class example.Test#grantPermission - class example.Test#grantPermission2 disable-sharding: true + project: projectFoo """.trimIndent() ) } diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt index 4371cc78b1..04704c118b 100644 --- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt @@ -39,7 +39,7 @@ class ArgsHelperTest { fun mergeYmlMaps_succeeds() { val merged = mergeYmlMaps(GcloudYml, IosGcloudYml) assertThat(merged.keys.size).isEqualTo(1) - assertThat(merged["gcloud"]?.size).isEqualTo(12) + assertThat(merged["gcloud"]?.size).isEqualTo(11) } @Test diff --git a/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt b/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt index 8aef1b7dc9..b86798f031 100644 --- a/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt @@ -12,6 +12,7 @@ class GcloudYmlTest { @Test fun gcloudYml() { val gcloud = GcloudYml().gcloud + gcloud.resultsBucket = "mockBucket" assertThat(gcloud.resultsBucket) .isEqualTo("mockBucket") diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index dbea2fd4d8..df5d9460fd 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -146,7 +146,6 @@ IosArgs record-video: false timeout: 70m async: true - project: projectFoo results-history-name: ios-history # iOS gcloud test: $testAbsolutePath @@ -178,6 +177,7 @@ IosArgs - b/testBasicSelection - b/testBasicSelection2 disable-sharding: true + project: projectFoo """.trimIndent() ) } diff --git a/test_runner/src/test/kotlin/ftl/args/yml/YamlDeprecatedTest.kt b/test_runner/src/test/kotlin/ftl/args/yml/YamlDeprecatedTest.kt index 7aa288d050..310a208b53 100644 --- a/test_runner/src/test/kotlin/ftl/args/yml/YamlDeprecatedTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/yml/YamlDeprecatedTest.kt @@ -30,10 +30,37 @@ class YamlDeprecatedTest { assertThat(output).isEqualTo(input) } + @Test + fun `Transform missing Flank object node`() { + // Verify input with null flank ObjectNode is successfully replaced + val input = """ + --- + gcloud: + project: 0 + + """.trimIndent() + + val expected = """ + --- + gcloud: {} + flank: + project: 0 + + """.trimIndent() + + val (error, output) = YamlDeprecated.modify(input) + + assertThat(error).isFalse() + assertThat(output).isEqualTo(expected) + } + @Test fun `Flank old keys renamed to new keys`() { val input = """ --- + gcloud: + project: 0 + flank: testShards: 1 shardTime: 2 @@ -41,12 +68,13 @@ class YamlDeprecatedTest { smartFlankGcsPath: 4 disableSharding: 5 - """.trimIndent() val expected = """ --- + gcloud: {} flank: + project: 0 max-test-shards: 1 shard-time: 2 repeat-tests: 3 diff --git a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt index 3753ac09a3..1e5b427cbd 100644 --- a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt +++ b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt @@ -28,7 +28,6 @@ gcloud: record-video: . timeout: . async: . - project: . results-history-name: . app: . @@ -55,6 +54,7 @@ flank: test-targets-always-run: - . three: . + project: . """.trimIndent() ) assertThat(lint).isEqualTo( @@ -74,6 +74,7 @@ Unknown keys in flank -> [three] gcloud: app: . test: . +flank: project: . """.trimIndent() ) @@ -98,7 +99,6 @@ gcloud: record-video: . timeout: . async: . - project: . results-history-name: . test: . @@ -118,6 +118,7 @@ flank: test-targets: - . three: . + project: . """.trimIndent() ) assertThat(lint).isEqualTo( @@ -135,9 +136,10 @@ Unknown keys in flank -> [three] val lint = Doctor.validateYaml( IosArgs, """ gcloud: - project: . test: . xctestrun-file: . +flank: + project: . """.trimIndent() ) assertThat(lint).isEqualTo("")