e2e: added e2e workflows

.github/licenserc.yml

@@ -39,6 +39,7 @@ header:
     - 'tsconfig.json'
     - '**/data/**'
     - 'dist'
+    - 'tests'
 
   comment: on-failure
 

.github/workflows/e2e-test-setup.yml

@@ -0,0 +1,85 @@
+# Copyright The Notary Project Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: e2e-test-setup
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  e2e-test-setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      # E2E test cases on Notation: Setup
+      - name: Setup Notation
+        uses: ./setup
+
+      - name: Setup Notation using unsupported version
+        id: unsupported-version
+        continue-on-error: true
+        uses: ./setup
+        with:
+            version: unsupported
+      - name: 'Should Fail: Setup Notation using unsupported version'
+        if: steps.unsupported-version.outcome != 'failure'
+        run: |
+            echo "Setup Notation using unsupported version should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using URL
+        uses: ./setup
+        with:
+            url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+            checksum: bca21002cfb611f9ffc2930e88ecdb74471806770d0c64d45e39f4971692bcf5
+
+      - name: Setup Notation using URL without checksum
+        id: no-checksum
+        continue-on-error: true
+        uses: ./setup
+        with:
+          url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+      - name: 'Should Fail: Setup Notation using URL without checksum'
+        if: steps.no-checksum.outcome != 'failure'
+        run: |
+            echo "Setup Notation using URL without checksum should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using URL and invalid checksum
+        id: invalid-checksum
+        continue-on-error: true
+        uses: ./setup
+        with:
+          url: https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.6/notation_1.0.0-rc.6_linux_amd64.tar.gz
+          checksum: abcedf
+      - name: 'Should Fail: Setup Notation using URL and invalid checksum'
+        if: steps.invalid-checksum.outcome != 'failure'
+        run: |
+            echo "Setup Notation using URL and invalid checksum should fail, but succeeded."
+            exit 1
+
+      - name: Setup Notation using invalid URL
+        id: invalid-url
+        continue-on-error: true
+        uses: ./setup
+        with:
+          url: invalid-url
+          checksum: test
+      - name: 'Should Fail: Setup Notation using invalid URL'
+        if: steps.invalid-url.outcome != 'failure'
+        run: |
+            echo "Setup Notation using invalid URL should fail, but succeeded."
+            exit 1

.github/workflows/e2e-test-sign.yml

@@ -0,0 +1,198 @@
+# Copyright The Notary Project Authors.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: e2e-test-sign
+
+on:
+  push:
+  pull_request:
+
+env:
+  E2E_KEY: /home/runner/.config/notation/localkeys/e2e-test.key
+  E2E_CERT: /home/runner/.config/notation/localkeys/e2e-test.crt
+
+jobs:
+  e2e-test-sign:
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build and push to local registry
+        id: prepare
+        uses: docker/build-push-action@v4
+        with:
+          context: ./tests/e2e
+          push: true
+          tags: localhost:5000/e2e:latest
+      - name: Retrieve digest
+        run: |
+          echo "target_artifact_reference=localhost:5000/e2e@${{ steps.prepare.outputs.digest }}" >> "$GITHUB_ENV"
+
+      # Setting up Notation CLI on the runner
+      - name: Setup Notation
+        uses: ./setup
+
+      # Generate test
+      - name: Notation generate-test
+        run: |
+          notation cert generate-test "e2e-test"
+
+      # E2E test cases on Notation: Sign
+      - name: Sign artifact using notation plugin
+        uses: ./sign
+        with:
+          plugin_name: e2e-test-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+
+      - name: Sign artifact with multiple plugin_config
+        uses: ./sign
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: e2e-test-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+            key=val
+            foo=bar
+
+      - name: Sign artifact with invalid plugin name
+        continue-on-error: true
+        id: invalid-plugin-name
+        uses: ./sign
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: invalid-plugin
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: be8d035024d3a96afb4118af32f2e201f126c7254b02f7bcffb3e3149d744fd2
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin name'
+        if: steps.invalid-plugin-name.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin name should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact with invalid plugin url
+        continue-on-error: true
+        id: invalid-plugin-url
+        uses: ./sign
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: invalid-url
+          plugin_checksum: test
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin url'
+        if: steps.invalid-plugin-url.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin url should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing plugin checksum
+        continue-on-error: true
+        id: missing-plugin-checksum
+        uses: ./sign
+        with:
+          plugin_name: test
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact missing plugin checksum'
+        if: steps.missing-plugin-checksum.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing plugin checksum should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact with invalid plugin checksum
+        continue-on-error: true
+        id: invalid-plugin-checksum
+        uses: ./sign
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://github.com/notaryproject/notation-action/raw/e2e-test-plugin/tests/plugin_binaries/notation-e2e-test-plugin_0.1.0_linux_amd64.tar.gz
+          plugin_checksum: abcdef
+          key_id: ${{ env.E2E_CERT }}
+          target_artifact_reference: ${{ env.target_artifact_reference }}
+          signature_format: cose
+          plugin_config: |-
+            keyFile=${{ env.E2E_KEY }}
+      - name: 'Should Fail: Sign artifact with invalid plugin checksum'
+        if: steps.invalid-plugin-checksum.outcome != 'failure'
+        run: |
+            echo "Sign artifact with invalid plugin checksum should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing key id
+        continue-on-error: true
+        id: missing-key-id
+        uses: ./sign
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://test
+          plugin_checksum: abcdef
+          target_artifact_reference: test
+      - name: 'Should Fail: Sign artifact missing key id'
+        if: steps.missing-key-id.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing key id should fail, but succeeded."
+            exit 1
+
+      - name: Sign artifact missing target artifact reference
+        continue-on-error: true
+        id: missing-artifact-reference
+        uses: notaryproject/notation-action/sign@main
+        env:
+          E2E_SIGNING_KEY: ${{ secrets.E2E_SIGNING_KEY }}
+        with:
+          plugin_name: test
+          plugin_url: https://test
+          plugin_checksum: abcdef
+          key_id: test
+      - name: 'Should Fail: Sign artifact missing target artifact reference'
+        if: steps.missing-artifact-reference.outcome != 'failure'
+        run: |
+            echo "Sign artifact missing target artifact reference should fail, but succeeded."
+            exit 1
+