e2e: added e2e workflows

.github/e2e-test/Dockerfile

@@ -0,0 +1,2 @@
+FROM node:16-slim
+RUN echo "E2E test of Notation Github Actions"

.github/e2e-test/trustpolicy/invalid-trustpolicy.json

@@ -0,0 +1,16 @@
+{
+    "version": "1.0",
+    "trustPolicies": [
+        {
+            "name": "e2e-test-invalid",
+            "registryScopes": [ "invalid", "*" ],
+            "signatureVerification": {
+                "level" : "strict" 
+            },
+            "trustStores": [ "ca:e2e-test"],
+            "trustedIdentities": [
+                "*"
+            ]
+        }
+    ]
+}

.github/e2e-test/trustpolicy/trustpolicy.json

@@ -0,0 +1,16 @@
+{
+    "version": "1.0",
+    "trustPolicies": [
+        {
+            "name": "e2e-test",
+            "registryScopes": [ "*" ],
+            "signatureVerification": {
+                "level" : "strict" 
+            },
+            "trustStores": [ "ca:e2e-test"],
+            "trustedIdentities": [
+                "*"
+            ]
+        }
+    ]
+}

.github/e2e-test/truststore/invalid-trust-store/x509/ca/e2e-test/invalid.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDPjCCAiagAwIBAgIBeTANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJVUzEL
+MAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxDzANBgNVBAoTBk5vdGFyeTEP
+MA0GA1UEAxMGYWxwaW5lMB4XDTIzMDUwOTA0NTUxMloXDTMzMDUxMDA0NTUxMlow
+TjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMQ8w
+DQYDVQQKEwZOb3RhcnkxDzANBgNVBAMTBmFscGluZTCCASIwDQYJKoZIhvcNAQEB
+BQADggEPADCCAQoCggEBAK5hpq1229GGLjMK6i9KZhuUO+SV7rUFnWIDiIPO5yWx
+YDkl+bGroeAvJYu6MVCMQ6FMRXD9jhnG6R+sAHwY7gVgcJ1OXak87PkLp/Ii1Cr7
+XkkySZeD+Br1vSQzfxs3pFG+iBCeVVkeZdsg+xqwnAlqAILXwIbTGRyJP1Xiu9nw
+OeuX1YmxPl2m29Pt1EtfVCL9COsVKt5LgOVyWP/9ISWevOBqSCU9bk35HFo9VTeU
+f6+ffhSMjv0Y9uwkFFOKXpcV8Sa3ArqyBmgQlUfGg1iwYlqiDE0fTYxiB3gLgETA
+lmTm50J+WB9LoDrnrQpbXFLoegm+JV+uSD8J8H7DL2sCAwEAAaMnMCUwDgYDVR0P
+AQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMA0GCSqGSIb3DQEBCwUAA4IB
+AQAt0Nvna1c4pPn8kzoN5VvmFmeIgdO/BJpmdhdg0WIQ9aeN/xPXXaVjPp1Mk7ed
+XHAvBwQr0Gyzqyy7g/h0gdnAFG7f6blrRNzbrRBCq6cNqX8iwgK/9+2OYKxk1QWj
+8Gx0cvu1DN1aXjPPGgQ2j3tHjJvJv32J/zuZa8gU40RPPSLaBlc5ZjpFmyi29sKl
+TeeZ+F/Ssic51qXXw2CsYGGWK5yQ3xSCxbw6bb2G/s/YI7/KlWg9BktBJHzRu04Z
+NR77W7/dyJ3Lj17PlW1XKmMOFHsQivagXeRCbmYZ43fX4ugFRFKL7KE0EgmGOWpJ
+0xv+6ig93sqHzQ/0uv1YgFov
+-----END CERTIFICATE-----

.github/licenserc.yml

@@ -39,6 +39,7 @@ header:
     - 'tsconfig.json'
     - '**/data/**'
     - 'dist'
+    - '**/e2e-test/**'
 
   comment: on-failure
 

.github/workflows/e2e-test-setup.yml

@@ -0,0 +1,85 @@
+# Copyright The Notary Project Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: e2e-test-setup
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  e2e-test-setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      # E2E test cases on Notation: Setup
+      - name: Setup Notation
+        uses: notaryproject/notation-action/setup@main
+
+      - name: Setup Notation using unsupported version
+        id: unsupported-version
+        continue-on-error: true
+        uses: notaryproject/notation-action/setup@main
+        with:
+            version: unsupported
+      - name: 'Should Fail: Setup Notation using unsupported version'
+        if: steps.unsupported-version.outcome != 'failure'
+        run: |
+            echo "Setup Notation using unsupported version should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using URL
+        uses: notaryproject/notation-action/setup@main
+        with:
+            url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+            checksum: bca21002cfb611f9ffc2930e88ecdb74471806770d0c64d45e39f4971692bcf5
+
+      - name: Setup Notation using URL without checksum
+        id: no-checksum
+        continue-on-error: true
+        uses: notaryproject/notation-action/setup@main
+        with:
+          url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+      - name: 'Should Fail: Setup Notation using URL without checksum'
+        if: steps.no-checksum.outcome != 'failure'
+        run: |
+            echo "Setup Notation using URL without checksum should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using URL and invalid checksum
+        id: invalid-checksum
+        continue-on-error: true
+        uses: notaryproject/notation-action/setup@main
+        with:
+          url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+          checksum: abcedf
+      - name: 'Should Fail: Setup Notation using URL and invalid checksum'
+        if: steps.invalid-checksum.outcome != 'failure'
+        run: |
+            echo "Setup Notation using URL and invalid checksum should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using invalid URL
+        id: invalid-url
+        continue-on-error: true
+        uses: notaryproject/notation-action/setup@main
+        with:
+          url: invalid-url
+          checksum: test
+      - name: 'Should Fail: Setup Notation using invalid URL'
+        if: steps.invalid-url.outcome != 'failure'
+        run: |
+            echo "Setup Notation using invalid URL should fail, but succeeded."
+            exit 1

.github/workflows/e2e-test-sign.yml

@@ -0,0 +1,198 @@
+# Copyright The Notary Project Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: e2e-test-sign
+
+on:
+  push:
+  pull_request:
+
+env:
+  E2E_KEY: /home/runner/.config/notation/localkeys/e2e-test.key
+  E2E_CERT: /home/runner/.config/notation/localkeys/e2e-test.crt
+
+jobs:
+  e2e-test-sign:
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build and push to local registry
+        id: prepare
+        uses: docker/build-push-action@v4
+        with:
+          context: .github/e2e-test
+          push: true
+          tags: localhost:5000/e2e:latest
+      - name: Retrieve digest
+        run: |
+          echo "target_artifact_reference=localhost:5000/e2e@${{ steps.prepare.outputs.digest }}" >> "$GITHUB_ENV"
+
+      # Setting up Notation CLI on the runner
+      - name: Setup Notation
+        uses: notaryproject/notation-action/setup@main
+
+      # Generate test
+      - name: Notation generate-test
+        run: |
+          notation cert generate-test "e2e-test"
+
+      # E2E test cases on Notation: Sign
+      - name: Sign artifact using notation plugin
+        uses: notaryproject/notation-action/sign@main
+        with:
+          plugin_name: e2e-test-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+
+      - name: Sign artifact with multiple plugin_config
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: e2e-test-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+            key=val
+            foo=bar
+
+      - name: Sign artifact with invalid plugin name
+        continue-on-error: true
+        id: invalid-plugin-name
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: invalid-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin name'
+        if: steps.invalid-plugin-name.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin name should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact with invalid plugin url
+        continue-on-error: true
+        id: invalid-plugin-url
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: invalid-url
+          plugin_checksum: test
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin url'
+        if: steps.invalid-plugin-url.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin url should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing plugin checksum
+        continue-on-error: true
+        id: missing-plugin-checksum
+        uses: notaryproject/notation-action/sign@main
+        with:
+          plugin_name: test
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact missing plugin checksum'
+        if: steps.missing-plugin-checksum.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing plugin checksum should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact with invalid plugin checksum
+        continue-on-error: true
+        id: invalid-plugin-checksum
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: abcdef
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin checksum'
+        if: steps.invalid-plugin-checksum.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin checksum should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing key id
+        continue-on-error: true
+        id: missing-key-id
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://test
+          plugin_checksum: abcdef
+          target_artifact_reference: test
+      - name: 'Should Fail: Sign artifact missing key id'
+        if: steps.missing-key-id.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing key id should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing target artifact reference
+        continue-on-error: true
+        id: missing-artifact-reference
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://test
+          plugin_checksum: abcdef
+          key_id: test
+      - name: 'Should Fail: Sign artifact missing target artifact reference'
+        if: steps.missing-artifact-reference.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing target artifact reference should fail, but succeeded."
+            exit 1
+