Remove github secrets requirements (flyteorg#136)

* Remove github secrets requirements * Fix end2end * Fix path typo * more fixes * Cache always * Trigger a cache refresh * attempting a different cache key * more fixes * dummy commit * maybe a better SHA * Dump github context * use 'before SHA' * Add other restore keys * dummy commit * dummy commit * Prime cache fix * dummy commit * Prime cache fix * Prime cache fix * dummy commit * Use builder image for integration tests * Fix tags * remove unused build arg * Fix master workflow * more cleanup * Dump github context * Attempt a more generic workflow * Fixes * fix integration script * Add goreleaser
eapolinario · Oct 30, 2020 · f39a4e0 · f39a4e0
1 parent dfab3c4
commit f39a4e0
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 114 deletions.
diff --git a/flyteadmin/.github/workflows/master.yml b/flyteadmin/.github/workflows/master.yml
@@ -6,8 +6,60 @@ on:
       - master
 
 jobs:
+  # Duplicated from pull request workflow because sharing is not yet supported
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
+
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
+
+  # Duplicated from pull request workflow because sharing is not yet supported
+  endtoend:
+    name: End to End tests
+    runs-on: ubuntu-latest
+    needs: [build-docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true
+      - uses: engineerd/[email protected]
+      - name: End2End
+        run: |
+          kubectl cluster-info
+          kubectl get pods -n kube-system
+          echo "current-context:" $(kubectl config current-context)
+          echo "environment-kubeconfig:" ${KUBECONFIG}
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute
+
   bump-version:
+    if: github.actor != 'goreleaserbot'
     runs-on: ubuntu-latest
+    needs: build-docker # Only to ensure it can successfully build
     outputs:
       version: ${{ steps.bump-version.outputs.tag }}
     steps:
@@ -18,6 +70,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           WITH_V: true
           DEFAULT_BUMP: patch
+
   push-github-end2end:
     runs-on: ubuntu-latest
     needs: bump-version
@@ -30,22 +83,12 @@ jobs:
         with:
           username: "${{ github.actor }}"
           password: "${{ secrets.GITHUB_TOKEN }}"
-          image_name: ${{ secrets.flytegithub_repo }}/flyteadmin
+          image_name: ${{ github.repository }}
           image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
           push_git_tag: true
           registry: docker.pkg.github.com
           build_extra_args: "--compress=true"
-      - uses: engineerd/[email protected]
-      - name: End2End
-        env:
-          DOCKER_USERNAME: ${{ github.actor }}
-          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
-        run: |
-          kubectl cluster-info
-          kubectl get pods -n kube-system
-          echo "current-context:" $(kubectl config current-context)
-          echo "environment-kubeconfig:" ${KUBECONFIG}
-          ADMIN=${{ secrets.flytegithub_repo }}/flyteadmin:${{ github.sha }} make end2end_execute
+
   push-dockerhub:
     runs-on: ubuntu-latest
     needs: bump-version
@@ -62,6 +105,7 @@ jobs:
           image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
           push_git_tag: true
           build_extra_args: "--compress=true"
+
   tests-lint:
     runs-on: ubuntu-latest
     steps:

diff --git a/flyteadmin/.github/workflows/pull_request.yml b/flyteadmin/.github/workflows/pull_request.yml
@@ -4,72 +4,82 @@ on:
   pull_request
 
 jobs:
-  build-and-end2end:
+  build-docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Push Docker Image to Github Registry
-        uses: whoan/docker-build-with-cache-action@v5
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
         with:
-          username: "${{ github.actor }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-          image_name: ${{ secrets.flytegithub_repo }}/flyteadmin
-          image_tag: ${{ github.sha }}
-          push_git_tag: true
-          registry: docker.pkg.github.com
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
+
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
+
+  endtoend:
+    name: End to End tests
+    runs-on: ubuntu-latest
+    needs: [build-docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot.tar || true
       - uses: engineerd/[email protected]
       - name: End2End
-        env:
-          DOCKER_USERNAME: ${{ github.actor }}
-          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           kubectl cluster-info
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
-          ADMIN=${{ secrets.flytegithub_repo }}/flyteadmin:${{ github.sha }} make end2end_execute
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:latest make end2end_execute
+
   integration:
     runs-on: ubuntu-latest
+    needs: [build-docker]
     steps:
-      - uses: actions/checkout@v2
-      - name: Push Docker Image to Github Registry
-        uses: whoan/docker-build-with-cache-action@v5
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
         with:
-          username: "${{ github.actor }}"
-          password: "${{ secrets.GITHUB_TOKEN }}"
-          image_name: ${{ secrets.flytegithub_repo }}/flyteadmin
-          image_tag: ${{ github.sha }}-builder
-          push_git_tag: true
-          registry: docker.pkg.github.com
-          build_extra_args: "--target=builder"
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+      - name: Prime docker cache
+        run: docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true
       - uses: engineerd/[email protected]
       - name: Integration
-        env:
-          DOCKER_USERNAME: ${{ github.actor }}
-          DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           kubectl cluster-info
           kubectl get pods -n kube-system
           echo "current-context:" $(kubectl config current-context)
           echo "environment-kubeconfig:" ${KUBECONFIG}
-          make k8s_integration_execute
-  push-dockerhub:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Push Docker Image to Dockerhub
-        uses: whoan/docker-build-with-cache-action@v5
-        with:
-          username: "${{ secrets.DOCKERHUB_USERNAME }}"
-          password: "${{ secrets.DOCKERHUB_PASSWORD }}"
-          image_name: ${{ secrets.DOCKERHUB_IMAGE_NAME }}
-          image_tag: ${{ github.sha }}
-          push_git_tag: true
-          build_extra_args: "--compress=true"
+          IMAGE_NAME=${{ github.event.repository.name }} IMAGE=lyft/${{ github.event.repository.name }}:builder make k8s_integration_execute
+
   tests-lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
       - name: Unit Tests
         uses: cedrickring/[email protected]
         env:

diff --git a/flyteadmin/.goreleaser.yml b/flyteadmin/.goreleaser.yml
@@ -0,0 +1,40 @@
+project_name: flyteadmin
+before:
+  hooks:
+    - go mod download
+builds:
+  - id: flyteadmin
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/main.go
+    binary: flyteadmin
+    goos:
+      - linux
+      - windows
+      - darwin
+archives:
+  - id: flyteadmin-archive
+    name_template: |-
+      kyverno-cli_{{ .Tag }}_{{ .Os }}_{{ .Arch -}}
+      {{- with .Arm -}}
+      {{- if (eq . "6") -}}hf
+      {{- else -}}v{{- . -}}
+      {{- end -}}
+      {{- end -}}
+    builds:
+      - flyteadmin
+    replacements:
+      386: i386
+      amd64: x86_64
+    format_overrides:
+      - goos: windows
+        format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
diff --git a/flyteadmin/Dockerfile b/flyteadmin/Dockerfile
@@ -27,6 +27,7 @@ ENV PATH="/artifacts:${PATH}"
 FROM alpine:3.10
 COPY --from=builder /artifacts /bin
 
+# Ensure the latest CA certs are present to authenticate SSL connections.
 RUN apk --update add ca-certificates
 
 CMD ["flyteadmin"]
diff --git a/flyteadmin/boilerplate/lyft/end2end/end2end.sh b/flyteadmin/boilerplate/lyft/end2end/end2end.sh
@@ -13,21 +13,20 @@ OUT="${DIR}/tmp"
 rm -rf ${OUT}
 git clone https://github.com/lyft/flyte.git "${OUT}"
 
-echo "Loading github docker images into 'kind' cluster to workaround this issue: https://github.com/containerd/containerd/issues/3291#issuecomment-631746985"
-docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} docker.pkg.github.com
-
 pushd ${OUT}
 
-if [ ! -z "$PROPELLER" ]; then
-  docker pull docker.pkg.github.com/${PROPELLER}
-  kind load docker-image docker.pkg.github.com/${PROPELLER}
-  sed -i.bak -e "s_docker.io/lyft/flytepropeller:.*_docker.pkg.github.com/${PROPELLER}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
-fi
-
-if [ ! -z "$ADMIN" ]; then
-  docker pull docker.pkg.github.com/${ADMIN}
-  kind load docker-image docker.pkg.github.com/${ADMIN}
-  sed -i.bak -e "s_docker.io/lyft/flyteadmin:.*_docker.pkg.github.com/${ADMIN}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
+if [ ! -z "$IMAGE" ]; 
+then
+  kind load docker-image ${IMAGE}
+  if [ ${IMAGE_NAME} -eq "flytepropeller" ]
+  then
+    sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/propeller/deployment.yaml
+  fi
+
+  if [ ${IMAGE} -eq "flyteadmin" ]
+  then
+    sed -i.bak -e "s_${IMAGE_NAME}:.*_${IMAGE}_g" ${OUT}/kustomize/base/admindeployment/deployment.yaml
+  fi
 fi
 
 make kustomize