Improve Docs And Support For Contributors (#353)

* Create deploy_resources.yml

* Update deploy_resources.yml

* Update deploy_resources.yml

* Update deploy_resources.yml

* Add locking table

* Delete deploy_resources.yml

* fix spelling

* Create PULL_REQUEST_TEMPLATE.md

* Create e2e_tests.md

* Update Makefile

* Update PULL_REQUEST_TEMPLATE.md

* Update PULL_REQUEST_TEMPLATE.md

* Update PULL_REQUEST_TEMPLATE.md

* Update e2e_tests.md

* Update e2e_tests.md

* Update Makefile

* Update Makefile

* Update e2e_tests.md

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+<!---
+Thank you very much for your contributions!
+--->
+
+
+<!--- If your PR fully resolves and should automatically close the linked issue, use Closes. Otherwise, use Relates --->
+Relates OR Closes #0000
+
+Output from acceptance testing:
+More information about running the tests [here](../docs/contributing/e2e_tests.md)
+<!--
+Replace TestAccXXX with a pattern that matches the tests affected by this PR.
+
+For more information on the `-run` flag, see the `go test` documentation at https://tip.golang.org/cmd/go/#hdr-Testing_flags.
+
+
+More information about running the tests [here](../docs/contributing/e2e_tests.md)
+-->
+```
+$ make testName=TestAccXXX e2e-test-with-apply
+...
+```

.github/workflows/cq_fetch.yml

@@ -12,7 +12,7 @@ jobs:
       matrix:
         dbversion: [ "postgres:latest" ]
         go: [ "1.17" ]
-        platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS
+        platform: [ ubuntu-latest ] # can not run in macOS and windowsOS
     runs-on: ${{ matrix.platform }}
     services:
       postgres:
@@ -92,7 +92,7 @@ jobs:
       matrix:
         dbversion: [ "postgres:latest" ]
         go: [ "1.17" ]
-        platform: [ ubuntu-latest ] # can not run in macOS and widnowsOS
+        platform: [ ubuntu-latest ] # can not run in macOS and windowsOS
     runs-on: ${{ matrix.platform }}
     services:
       postgres:

Makefile

@@ -33,9 +33,22 @@ pg-connect:
 
 # Run an integration test
 # you can pass in a specific test to run by specifying the testName:
-# make testName=TestIntegrationElasticbeanstal e2e-test
+# make testName=TestIntegrationElasticbeanstal$ e2e-test
 e2e-test:
-	INTEGRATION_TESTS=1 TF_VAR_PREFIX=cq-testing TF_APPLY_RESOURCES=0 TF_VAR_SUFFIX=integration go test -timeout 30s -v -run ^$(testName)$$  github.com/cloudquery/cq-provider-aws/resources/integration_tests
+	@if [[ "$(testName)" == "" ]]; then echo "\n testName must be set \n"; exit 1; fi
+	INTEGRATION_TESTS=1 TF_VAR_PREFIX=cq-testing TF_APPLY_RESOURCES=0 TF_VAR_SUFFIX=integration go test -timeout 180m -v -run ^$(testName)  github.com/cloudquery/cq-provider-aws/resources/integration_tests
+
+
+e2e-test-read-only:
+	@if [[ "$(testName)" == "" ]]; then echo "\n testName must be set \n"; exit 1; fi
+	@if [[ "$(TF_VAR_PREFIX)" == "" ]]; then echo "\n TF_VAR_PREFIX must be set \n"; exit 1; fi
+	@if [[ "$(TF_VAR_SUFFIX)" == "" ]]; then echo "\n TF_VAR_SUFFIX must be set \n"; exit 1; fi
+	INTEGRATION_TESTS=1TF_APPLY_RESOURCES=0 go test -timeout 180m -v -run ^$(testName)  github.com/cloudquery/cq-provider-aws/resources/integration_tests
+
+e2e-test-with-apply:
+	@if [[ "$(testName)" == "" ]]; then echo "\n testName must be set \n"; exit 1; fi
+	INTEGRATION_TESTS=1 TF_VAR_PREFIX=cq-testing TF_APPLY_RESOURCES=1 TF_VAR_SUFFIX=integration go test -timeout 180m -v -run ^$(testName)  github.com/cloudquery/cq-provider-aws/resources/integration_tests
+
 
 # Generate mocks for mock/unit testing 
 create-mocks:

docs/contributing/e2e_tests.md

@@ -0,0 +1,36 @@
+# Running cq-provider-aws E2E Tests:
+
+
+## Prerequisites:
+
+1. Go
+2. [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli)
+3. [Terraform Prerequisites](https://learn.hashicorp.com/tutorials/terraform/aws-build)
+
+
+
+## Running Tests
+
+All of the tests in the `resources/integration_tests` directory use a wrapper to dynamically call terraform in order to provision resources at test time and then tear down the resources when the testing is complete. You do not have to deploy the entire infrastructure suite in order to run the tests, you can pick and choose exactly which tests you want to run
+
+Using the Makefile in the root of the repository is the simplest way to start running your tests locally. 
+
+(hint: You can include regular expressions in the test name to match multiple tests)
+
+example of running multiple tests (`TestIntegrationCognitoUserPools` and `TestIntegrationCognitoIdentityPools`):
+``` bash
+make testName=^TestIntegrationCognito$ e2e-test-with-apply
+```
+
+
+### Running Read Only Tests
+
+
+If you have another way of deploying the terraform defined in the `resources/integration_tests/infra` directory you can do that and then run the tests against the deployed resources like this:
+
+
+``` bash
+export TF_VAR_PREFIX=WhatEverValueYouSpecified
+export TF_VAR_SUFFIX=WhatEverValueYouSpecified
+make testName=^TestIntegrationCognito$ e2e-test-with-apply
+```

resources/integration_tests/infra/README.md

@@ -8,6 +8,7 @@ Export the current s3 bucket for tfstate file
 ```shell
 export TF_BACKEND_BUCKET=cq-integration-tests-tf
 export TF_BACKEND_KEY=terraform.tfstate
+export TF_DYNAMODB_TABLE=cq-integration-tests-tf
 ```
 
 Init terraform

resources/integration_tests/infra/scripts/init.sh

@@ -14,6 +14,13 @@ else
   TF_BACKEND_KEY="${TF_BACKEND_KEY}"
 fi
 
+if [[ -z "${TF_DYNAMODB_TABLE}" ]]; then
+  echo "Missing env=TF_DYNAMODB_TABLE"
+  exit 1;
+else
+  TF_DYNAMODB_TABLE="${TF_DYNAMODB_TABLE}"
+fi
+
 if [[ -z "${TF_BACKEND_REGION}" ]]; then
   TF_BACKEND_REGION="us-east-1"
 else
@@ -30,4 +37,5 @@ terraform {
 
 terraform init -backend-config="bucket=$TF_BACKEND_BUCKET" \
    -backend-config="key=$TF_BACKEND_KEY" \
+   -backend-config="dynamodb_table=$TF_DYNAMODB_TABLE" \
    -backend-config="region=$TF_BACKEND_REGION"