feat: allow application default credentials when running integration …

…tests

feat: allow application default credentials when running integration tests

CONTRIBUTING.md

@@ -15,32 +15,89 @@ under a directory `ansible_collections`. Clone ensuring that hierarchy:
 
 ```shell
 mkdir -p $TARGET_DIR/ansible_collections/google
-git clone <url> $TARGET_DIR/collections/google/cloud
+git clone <url> $TARGET_DIR/ansible_collections/google/cloud
+```
+
+Then set up your Python virtual environment:
+
+```shell
+cd $TARGET_DIR/ansible_collections/google
+python3 -m venv venv
+. ./venv/bin/activate
+pip3 install -r requirements.txt
+pip3 install -r requirements-test.txt
+pip3 install ansible
 ```
 
 ## Running tests
 
-### prequisites for all tests
+### Prequisites for all tests
 
+- Install `gcloud` following [these instructions](https://cloud.google.com/sdk/docs/install).
 - Install the `ansible` package.
 - Some container runtime is necessary (e.g. `podman` or `docker`). The instructions use podman.
 
 ## Running integration tests
 
 ### Integration testing prequisites
 
-#### Installing personal GCP credentials
+#### Authentication with personal GCP credentials
+
+If you are running the integration tests locally the easiest way to
+authenticate to GCP is using [application default credentials](https://cloud.google.com/sdk/docs/authorizing#adc).
+Once you have installed `gcloud` and performed basic initialization (via `gcloud init`) run:
+
+```shell
+gcloud auth application-default login
+```
+
+#### Authentication with service account credentials
+
+A service account may also be used to run the integration tests. You can create one using `gcloud`:
+
+```shell
+gcloud iam service-accounts create ansible-test-account \
+    --description="For running Anisble integration tests" \
+    --display-name="Ansible Test Account"
+```
+
+You'll also need to export a key file. Here and below `$SERVICE_ACCOUNT_NAME`
+is the full email address of the service account, in the form
+`EMAIL@PROJECT_ID.iam.gserviceaccount.com`, e.g., if you used the
+account name `ansible-test-account` as suggested above and your project
+ID is `my-test-project`, use `[email protected]`.
+
+```shell
+gcloud iam service-accounts keys create /path/to/cred/file.json \
+    --iam-account=ansible-test-account@my-test-project.iam.gserviceaccount.com
+chmod 0600 /path/to/cred/file.json
+```
+
+Read the [best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys)
+to learn how to keep your service account key and your GCP resources safe.
+
+#### Configuring test credentials
 
 The integration tests for this module require the use of real GCP credentials, and must provide
-ansible-test those values. They can be added by authoring the following in `tests/integration/cloud-config-gcp.ini`:
+ansible-test those values. They can be added by creating the file `tests/integration/cloud-config-gcp.ini`.
+
+If you are using personal (i.e., application default) credentials, add:
+
+```
+[default]
+gcp_project: $PROJECT_ID
+gcp_cred_kind: application
+gcp_folder_id: $TEST_FOLDER (to create test projects)
+```
+
+If you are using a service account for credentials, add:
 
 ```
 [default]
-gcp_project: @PROJECT_ID
-gcp_cred_file: @CRED_FILE
-gcp_cred_kind: @CRED_KIND
-gcp_cred_email: @EMAIL
-gcp_folder_id: @TEST_FOLDER (to create test projects)
+gcp_project: $PROJECT_ID
+gcp_cred_file: /path/to/cred/file.json
+gcp_cred_kind: serviceaccount
+gcp_folder_id: $TEST_FOLDER (to create test projects)
 ```
 
 #### Setting up the project for testing
@@ -51,7 +108,8 @@ and is expected to be configured beforehand.
 For convenience, a bootstrap script is provided.
 
 NOTE: running this script will make irreversible changes in your
-GCP project (e.g. create an AppEngine project):
+GCP project (e.g. create an AppEngine project). You can omit
+`$SERVICE_ACCOUNT_NAME` is you are using application default credentials.
 
 ```bash
 bash ./scripts/bootstrap-project.sh $PROJECT_ID $SERVICE_ACCOUNT_NAME
@@ -92,7 +150,7 @@ ansible-lint
 
 ## Specific Tasks
 
-The following enumerates detailed documentation for specific tasks related tot
+The following enumerates detailed documentation for specific tasks related to
 the codebase.
 
 ### Updating the supported ansible-core version

changelogs/fragments/app-default-creds.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test - add support for GCP application default credentials (https://github.com/ansible-collections/google.cloud/issues/359).

scripts/bootstrap-project.sh

@@ -40,12 +40,15 @@ for SERVICE in "${SERVICE_LIST[@]}"; do
     gcloud services enable "$SERVICE" --project="$PROJECT_ID"
 done
 
-for ROLE in "${REQUIRED_ROLE_LIST[@]}"; do
-    echo "enabling role $ROLE..."
-    gcloud projects add-iam-policy-binding "$PROJECT_ID" \
-        --member="serviceAccount:$SERVICE_ACCOUNT_NAME" \
-        --role="$ROLE"
-done
+if [ -n "$SERVICE_ACCOUNT_NAME" ]
+then
+    for ROLE in "${REQUIRED_ROLE_LIST[@]}"; do
+        echo "enabling role $ROLE..."
+        gcloud projects add-iam-policy-binding "$PROJECT_ID" \
+            --member="serviceAccount:$SERVICE_ACCOUNT_NAME" \
+            --role="$ROLE"
+    done
+fi
 
 if ! gcloud app describe --project="$PROJECT_ID" > /dev/null; then
     echo "creating appengine project..."

tests/integration/targets/gcp_appengine_firewall_rule/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-resource_name: "{{ resource_prefix }}"
+resource_name: "{{ resource_prefix }}"

tests/integration/targets/gcp_appengine_firewall_rule/tasks/autogen.yml

@@ -20,7 +20,7 @@
     action: ALLOW
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
 #----------------------------------------------------------
 - name: create a firewall rule
@@ -30,7 +30,7 @@
     action: ALLOW
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is true
@@ -41,7 +41,7 @@
   google.cloud.gcp_appengine_firewall_rule_info:
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/cloud-platform
   register: results
@@ -57,7 +57,7 @@
     action: ALLOW
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is false
@@ -72,7 +72,7 @@
     action: ALLOW
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is true
@@ -83,7 +83,7 @@
   google.cloud.gcp_appengine_firewall_rule_info:
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/cloud-platform
   register: results
@@ -101,7 +101,7 @@
     action: ALLOW
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is false

tests/integration/targets/gcp_bigquery_dataset/tasks/autogen.yml

@@ -20,7 +20,7 @@
       dataset_id: my_example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
 #----------------------------------------------------------
 - name: create a dataset
@@ -30,7 +30,7 @@
       dataset_id: my_example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is true
@@ -41,7 +41,7 @@
   google.cloud.gcp_bigquery_dataset_info:
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/bigquery
   register: results
@@ -57,7 +57,7 @@
       dataset_id: my_example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is false
@@ -72,7 +72,7 @@
       dataset_id: my_example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is true
@@ -83,7 +83,7 @@
   google.cloud.gcp_bigquery_dataset_info:
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/bigquery
   register: results
@@ -99,7 +99,7 @@
       dataset_id: my_example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is false

tests/integration/targets/gcp_bigquery_table/tasks/autogen.yml

@@ -20,7 +20,7 @@
       dataset_id: example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: dataset
 - name: delete a table
@@ -33,7 +33,7 @@
       table_id: example_table
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
 #----------------------------------------------------------
 - name: create a table
@@ -46,7 +46,7 @@
       table_id: example_table
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is true
@@ -58,7 +58,7 @@
       dataset: example_dataset
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/bigquery
   register: results
@@ -77,7 +77,7 @@
       table_id: example_table
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: present
   register: result
 - name: assert changed is false
@@ -95,7 +95,7 @@
       table_id: example_table
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is true
@@ -107,7 +107,7 @@
       dataset: example_dataset
       project: "{{ gcp_project }}"
       auth_kind: "{{ gcp_cred_kind }}"
-      service_account_file: "{{ gcp_cred_file }}"
+      service_account_file: "{{ gcp_cred_file | default(omit) }}"
       scopes:
         - https://www.googleapis.com/auth/bigquery
   register: results
@@ -126,7 +126,7 @@
       table_id: example_table
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: result
 - name: assert changed is false
@@ -143,7 +143,7 @@
       dataset_id: example_dataset
     project: "{{ gcp_project }}"
     auth_kind: "{{ gcp_cred_kind }}"
-    service_account_file: "{{ gcp_cred_file }}"
+    service_account_file: "{{ gcp_cred_file | default(omit) }}"
     state: absent
   register: dataset
   ignore_errors: true