From f64f297208c67feed3815aef130096726a789c11 Mon Sep 17 00:00:00 2001 From: Philip Harrison Date: Thu, 1 Apr 2021 14:32:04 +0100 Subject: [PATCH 1/2] CI: Simplify Dockerfile.ci Use ghcr.io for ci image builds and leve leverage Docker's buildkit caching when building it. Using docker volumes instead of copying all the files over to the ci image. The caching change was previously [reverted in this commit](https://github.com/dependabot/dependabot-core/commit/a89aca9f82cfbdf56e229e6dec77bfda4aa343fb#diff-b803fcb7f17ed9235f1e5cb1fcd2f5d3b2838429d4368ae4c57ce4436577f03f). I believe this will work with [https://github.com/dependabot/dependabot-core/pull/3426](this CI change) as I have vague recalls that the failures happened because we where running all the bundle installs in `Dockerfile.ci`. --- .github/workflows/ci.yml | 149 +++++++++++++++++++++++++++++++++++---- Dockerfile.ci | 67 ------------------ bin/docker-dev-shell | 36 ++++++++++ 3 files changed, 172 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfcce156a7..99ba9bcf70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: echo "BASE_IMAGE=ubuntu:18.04" >> $GITHUB_ENV echo "CORE_IMAGE=dependabot/dependabot-core" >> $GITHUB_ENV echo "CORE_CI_IMAGE=dependabot/dependabot-core-ci" >> $GITHUB_ENV + echo "CODE_DIR=/home/dependabot/dependabot-core" >> $GITHUB_ENV - name: Log in to Docker registry run: | if [ -n "${{ secrets.DOCKER_USERNAME }}" ] && [ -n "${{ secrets.DOCKER_PASSWORD }}" ]; then @@ -58,40 +59,162 @@ jobs: - name: Pull Docker base images & warm Docker cache run: | docker pull "$BASE_IMAGE" - docker pull "$CORE_CI_IMAGE:core--$BRANCH_REF" || - docker pull "$CORE_IMAGE:latest" - docker pull "$CORE_CI_IMAGE:ci--$BRANCH_REF" || - "$CORE_CI_IMAGE:latest" || true + docker pull "$CORE_IMAGE:branch--$BRANCH_REF" || + docker pull "$CORE_IMAGE:latest" || true + docker pull "$CORE_CI_IMAGE:branch--$BRANCH_REF" || + docker pull "$CORE_CI_IMAGE:latest" || true - name: Build dependabot-core image run: | DOCKER_BUILDKIT=1 docker build \ -t "$CORE_IMAGE:latest" \ - -t "$CORE_CI_IMAGE:core--$BRANCH_REF" \ + -t "$CORE_IMAGE:branch--$BRANCH_REF" \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from "$BASE_IMAGE" \ - --cache-from "$CORE_CI_IMAGE:core--$BRANCH_REF" \ + --cache-from "$CORE_IMAGE:branch--$BRANCH_REF" \ --cache-from "$CORE_IMAGE:latest" \ . - name: Push dependabot-core image to Docker registry if: env.DOCKER_LOGGED_IN == 'true' run: | - docker push "$CORE_CI_IMAGE:core--$BRANCH_REF" + docker push "$CORE_IMAGE:branch--$BRANCH_REF" - name: Build dependabot-core-ci image run: | - rm .dockerignore - docker build \ + DOCKER_BUILDKIT=1 docker build \ -t "$CORE_CI_IMAGE:latest" \ - -t "$CORE_CI_IMAGE:ci--$BRANCH_REF" \ + -t "$CORE_CI_IMAGE:branch--$BRANCH_REF" \ -f Dockerfile.ci \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from "$CORE_CI_IMAGE:latest" \ - --cache-from "$CORE_CI_IMAGE:ci--$BRANCH_REF" \ + --cache-from "$CORE_CI_IMAGE:branch--$BRANCH_REF" \ . - name: Push dependabot-core-ci image to Docker registry if: env.DOCKER_LOGGED_IN == 'true' run: | docker push "$CORE_CI_IMAGE:latest" - docker push "$CORE_CI_IMAGE:ci--$BRANCH_REF" + docker push "$CORE_CI_IMAGE:branch--$BRANCH_REF" - name: Run ${{ matrix.suite.name }} tests run: | - docker run --env "CI=true" --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" --env "SUITE_NAME=${{ matrix.suite.name }}" --rm "$CORE_CI_IMAGE" bash -c \ + docker run \ + -v "$(pwd)/.rubocop.yml:$CODE_DIR/.rubocop.yml" \ + -v "$(pwd)/bin:$CODE_DIR/bin" \ + -v "$(pwd)/common/.rubocop.yml:$CODE_DIR/common/.rubocop.yml" \ + -v "$(pwd)/common/Gemfile:$CODE_DIR/common/Gemfile" \ + -v "$(pwd)/common/dependabot-common.gemspec:$CODE_DIR/common/dependabot-common.gemspec" \ + -v "$(pwd)/common/bin:$CODE_DIR/common/bin" \ + -v "$(pwd)/common/lib:$CODE_DIR/common/lib" \ + -v "$(pwd)/common/spec:$CODE_DIR/common/spec" \ + -v "$(pwd)/common/script:$CODE_DIR/common/script" \ + -v "$(pwd)/terraform/.rubocop.yml:$CODE_DIR/terraform/.rubocop.yml" \ + -v "$(pwd)/terraform/Gemfile:$CODE_DIR/terraform/Gemfile" \ + -v "$(pwd)/terraform/dependabot-terraform.gemspec:$CODE_DIR/terraform/dependabot-terraform.gemspec" \ + -v "$(pwd)/terraform/lib:$CODE_DIR/terraform/lib" \ + -v "$(pwd)/terraform/spec:$CODE_DIR/terraform/spec" \ + -v "$(pwd)/terraform/script:$CODE_DIR/terraform/script" \ + -v "$(pwd)/elm/.rubocop.yml:$CODE_DIR/elm/.rubocop.yml" \ + -v "$(pwd)/elm/Gemfile:$CODE_DIR/elm/Gemfile" \ + -v "$(pwd)/elm/dependabot-elm.gemspec:$CODE_DIR/elm/dependabot-elm.gemspec" \ + -v "$(pwd)/elm/lib:$CODE_DIR/elm/lib" \ + -v "$(pwd)/elm/spec:$CODE_DIR/elm/spec" \ + -v "$(pwd)/elm/script:$CODE_DIR/elm/script" \ + -v "$(pwd)/docker/.rubocop.yml:$CODE_DIR/docker/.rubocop.yml" \ + -v "$(pwd)/docker/Gemfile:$CODE_DIR/docker/Gemfile" \ + -v "$(pwd)/docker/dependabot-docker.gemspec:$CODE_DIR/docker/dependabot-docker.gemspec" \ + -v "$(pwd)/docker/lib:$CODE_DIR/docker/lib" \ + -v "$(pwd)/docker/spec:$CODE_DIR/docker/spec" \ + -v "$(pwd)/docker/script:$CODE_DIR/docker/script" \ + -v "$(pwd)/git_submodules/.rubocop.yml:$CODE_DIR/git_submodules/.rubocop.yml" \ + -v "$(pwd)/git_submodules/Gemfile:$CODE_DIR/git_submodules/Gemfile" \ + -v "$(pwd)/git_submodules/dependabot-git_submodules.gemspec:$CODE_DIR/git_submodules/dependabot-core.gemspec" \ + -v "$(pwd)/git_submodules/lib:$CODE_DIR/git_submodules/lib" \ + -v "$(pwd)/git_submodules/spec:$CODE_DIR/git_submodules/spec" \ + -v "$(pwd)/git_submodules/script:$CODE_DIR/git_submodules/script" \ + -v "$(pwd)/github_actions/.rubocop.yml:$CODE_DIR/github_actions/.rubocop.yml" \ + -v "$(pwd)/github_actions/Gemfile:$CODE_DIR/github_actions/Gemfile" \ + -v "$(pwd)/github_actions/dependabot-github_actions.gemspec:$CODE_DIR/github_actions/dependabot-core.gemspec" \ + -v "$(pwd)/github_actions/lib:$CODE_DIR/github_actions/lib" \ + -v "$(pwd)/github_actions/spec:$CODE_DIR/github_actions/spec" \ + -v "$(pwd)/github_actions/script:$CODE_DIR/github_actions/script" \ + -v "$(pwd)/python/.rubocop.yml:$CODE_DIR/python/.rubocop.yml" \ + -v "$(pwd)/python/Gemfile:$CODE_DIR/python/Gemfile" \ + -v "$(pwd)/python/dependabot-python.gemspec:$CODE_DIR/python/dependabot-python.gemspec" \ + -v "$(pwd)/python/helpers:/opt/python" \ + -v "$(pwd)/python/helpers:/opt/python/helpers" \ + -v "$(pwd)/python/lib:$CODE_DIR/python/lib" \ + -v "$(pwd)/python/spec:$CODE_DIR/python/spec" \ + -v "$(pwd)/python/script:$CODE_DIR/python/script" \ + -v "$(pwd)/nuget/.rubocop.yml:$CODE_DIR/nuget/.rubocop.yml" \ + -v "$(pwd)/nuget/Gemfile:$CODE_DIR/nuget/Gemfile" \ + -v "$(pwd)/nuget/dependabot-nuget.gemspec:$CODE_DIR/nuget/dependabot-core.gemspec" \ + -v "$(pwd)/nuget/lib:$CODE_DIR/nuget/lib" \ + -v "$(pwd)/nuget/spec:$CODE_DIR/nuget/spec" \ + -v "$(pwd)/nuget/script:$CODE_DIR/nuget/script" \ + -v "$(pwd)/maven/.rubocop.yml:$CODE_DIR/maven/.rubocop.yml" \ + -v "$(pwd)/maven/Gemfile:$CODE_DIR/maven/Gemfile" \ + -v "$(pwd)/maven/dependabot-maven.gemspec:$CODE_DIR/maven/dependabot-core.gemspec" \ + -v "$(pwd)/maven/lib:$CODE_DIR/maven/lib" \ + -v "$(pwd)/maven/spec:$CODE_DIR/maven/spec" \ + -v "$(pwd)/maven/script:$CODE_DIR/maven/script" \ + -v "$(pwd)/gradle/.rubocop.yml:$CODE_DIR/gradle/.rubocop.yml" \ + -v "$(pwd)/gradle/Gemfile:$CODE_DIR/gradle/Gemfile" \ + -v "$(pwd)/gradle/dependabot-gradle.gemspec:$CODE_DIR/gradle/dependabot-gradle.gemspec" \ + -v "$(pwd)/gradle/lib:$CODE_DIR/gradle/lib" \ + -v "$(pwd)/gradle/spec:$CODE_DIR/gradle/spec" \ + -v "$(pwd)/gradle/script:$CODE_DIR/gradle/script" \ + -v "$(pwd)/hex/.rubocop.yml:$CODE_DIR/hex/.rubocop.yml" \ + -v "$(pwd)/hex/Gemfile:$CODE_DIR/hex/Gemfile" \ + -v "$(pwd)/hex/dependabot-hex.gemspec:$CODE_DIR/hex/dependabot-hex.gemspec" \ + -v "$(pwd)/hex/lib:$CODE_DIR/hex/lib" \ + -v "$(pwd)/hex/spec:$CODE_DIR/hex/spec" \ + -v "$(pwd)/hex/script:$CODE_DIR/hex/script" \ + -v "$(pwd)/cargo/.rubocop.yml:$CODE_DIR/cargo/.rubocop.yml" \ + -v "$(pwd)/cargo/Gemfile:$CODE_DIR/cargo/Gemfile" \ + -v "$(pwd)/cargo/dependabot-cargo.gemspec:$CODE_DIR/cargo/dependabot-core.gemspec" \ + -v "$(pwd)/cargo/lib:$CODE_DIR/cargo/lib" \ + -v "$(pwd)/cargo/spec:$CODE_DIR/cargo/spec" \ + -v "$(pwd)/cargo/script:$CODE_DIR/cargo/script" \ + -v "$(pwd)/dep/.rubocop.yml:$CODE_DIR/dep/.rubocop.yml" \ + -v "$(pwd)/dep/Gemfile:$CODE_DIR/dep/Gemfile" \ + -v "$(pwd)/dep/dependabot-dep.gemspec:$CODE_DIR/dep/dependabot-dep.gemspec" \ + -v "$(pwd)/dep/lib:$CODE_DIR/dep/lib" \ + -v "$(pwd)/dep/spec:$CODE_DIR/dep/spec" \ + -v "$(pwd)/dep/script:$CODE_DIR/dep/script" \ + -v "$(pwd)/go_modules/.rubocop.yml:$CODE_DIR/go_modules/.rubocop.yml" \ + -v "$(pwd)/go_modules/Gemfile:$CODE_DIR/go_modules/Gemfile" \ + -v "$(pwd)/go_modules/dependabot-go_modules.gemspec:$CODE_DIR/go_modules/dependabot-go_modules.gemspec" \ + -v "$(pwd)/go_modules/lib:$CODE_DIR/go_modules/lib" \ + -v "$(pwd)/go_modules/spec:$CODE_DIR/go_modules/spec" \ + -v "$(pwd)/go_modules/script:$CODE_DIR/go_modules/script" \ + -v "$(pwd)/npm_and_yarn/.rubocop.yml:$CODE_DIR/npm_and_yarn/.rubocop.yml" \ + -v "$(pwd)/npm_and_yarn/Gemfile:$CODE_DIR/npm_and_yarn/Gemfile" \ + -v "$(pwd)/npm_and_yarn/dependabot-npm_and_yarn.gemspec:$CODE_DIR/npm_and_yarn/dependabot-npm_and_yarn.gemspec" \ + -v "$(pwd)/npm_and_yarn/lib:$CODE_DIR/npm_and_yarn/lib" \ + -v "$(pwd)/npm_and_yarn/spec:$CODE_DIR/npm_and_yarn/spec" \ + -v "$(pwd)/npm_and_yarn/script:$CODE_DIR/npm_and_yarn/script" \ + -v "$(pwd)/composer/.rubocop.yml:$CODE_DIR/composer/.rubocop.yml" \ + -v "$(pwd)/composer/Gemfile:$CODE_DIR/composer/Gemfile" \ + -v "$(pwd)/composer/dependabot-composer.gemspec:$CODE_DIR/composer/dependabot-composer.gemspec" \ + -v "$(pwd)/composer/lib:$CODE_DIR/composer/lib" \ + -v "$(pwd)/composer/spec:$CODE_DIR/composer/spec" \ + -v "$(pwd)/composer/script:$CODE_DIR/composer/script" \ + -v "$(pwd)/bundler/.rubocop.yml:$CODE_DIR/bundler/.rubocop.yml" \ + -v "$(pwd)/bundler/Gemfile:$CODE_DIR/bundler/Gemfile" \ + -v "$(pwd)/bundler/dependabot-bundler.gemspec:$CODE_DIR/bundler/dependabot-bundler.gemspec" \ + -v "$(pwd)/bundler/lib:$CODE_DIR/bundler/lib" \ + -v "$(pwd)/bundler/helpers:$CODE_DIR/bundler/helpers" \ + -v "$(pwd)/bundler/spec:$CODE_DIR/bundler/spec" \ + -v "$(pwd)/bundler/helpers:/opt/bundler" \ + -v "$(pwd)/bundler/helpers:/opt/bundler/helpers" \ + -v "$(pwd)/bundler/script:$CODE_DIR/bundler/script" \ + -v "$(pwd)/omnibus/.rubocop.yml:$CODE_DIR/omnibus/.rubocop.yml" \ + -v "$(pwd)/omnibus/Gemfile:$CODE_DIR/omnibus/Gemfile" \ + -v "$(pwd)/omnibus/dependabot-omnibus.gemspec:$CODE_DIR/omnibus/dependabot-omnibus.gemspec" \ + -v "$(pwd)/omnibus/lib:$CODE_DIR/omnibus/lib" \ + -v "$(pwd)/omnibus/spec:$CODE_DIR/omnibus/spec" \ + -v "$(pwd)/omnibus/script:$CODE_DIR/omnibus/script" \ + --env "CI=true" \ + --env "DEPENDABOT_TEST_ACCESS_TOKEN=$DEPENDABOT_TEST_ACCESS_TOKEN" \ + --env "SUITE_NAME=${{ matrix.suite.name }}" \ + --env "BUNDLE_PATH=/home/dependabot/.bundle" \ + -w "$CODE_DIR" \ + --rm "$CORE_CI_IMAGE" bash -c \ "cd /home/dependabot/dependabot-core/${{ matrix.suite.path }} && ./script/ci-test" diff --git a/Dockerfile.ci b/Dockerfile.ci index 8ca7d5f92e..aedb4bb2f9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,74 +1,7 @@ FROM dependabot/dependabot-core -ARG CODE_DIR=/home/dependabot/dependabot-core -WORKDIR ${CODE_DIR} - -ENV BUNDLE_PATH="/home/dependabot/.bundle" \ - BUNDLE_BIN=".bundle/binstubs" \ - PATH=".bundle/binstubs:$PATH:/home/dependabot/.bundle/bin" RUN pyenv install -s 3.6.13 RUN pyenv install -s 3.7.1 RUN pyenv install -s 3.7.10 -COPY .rubocop.yml /home/dependabot/dependabot-core/ - -RUN mkdir -p \ - ${CODE_DIR}/bundler \ - ${CODE_DIR}/cargo \ - ${CODE_DIR}/common \ - ${CODE_DIR}/composer \ - ${CODE_DIR}/dep \ - ${CODE_DIR}/docker \ - ${CODE_DIR}/elm \ - ${CODE_DIR}/git_submodules \ - ${CODE_DIR}/github_actions \ - ${CODE_DIR}/go_modules \ - ${CODE_DIR}/gradle \ - ${CODE_DIR}/hex \ - ${CODE_DIR}/maven \ - ${CODE_DIR}/npm_and_yarn \ - ${CODE_DIR}/nuget \ - ${CODE_DIR}/omnibus \ - ${CODE_DIR}/python \ - ${CODE_DIR}/terraform - -COPY common/lib/dependabot/version.rb ${CODE_DIR}/common/lib/dependabot/version.rb -COPY common/Gemfile common/dependabot-common.gemspec ${CODE_DIR}/common/ -COPY bundler/Gemfile bundler/dependabot-bundler.gemspec ${CODE_DIR}/bundler/ -COPY cargo/Gemfile cargo/dependabot-cargo.gemspec ${CODE_DIR}/cargo/ -COPY composer/Gemfile composer/dependabot-composer.gemspec ${CODE_DIR}/composer/ -COPY dep/Gemfile dep/dependabot-dep.gemspec ${CODE_DIR}/dep/ -COPY docker/Gemfile docker/dependabot-docker.gemspec ${CODE_DIR}/docker/ -COPY elm/Gemfile elm/dependabot-elm.gemspec ${CODE_DIR}/elm/ -COPY git_submodules/Gemfile git_submodules/dependabot-git_submodules.gemspec ${CODE_DIR}/git_submodules/ -COPY github_actions/Gemfile github_actions/dependabot-github_actions.gemspec ${CODE_DIR}/github_actions/ -COPY go_modules/Gemfile go_modules/dependabot-go_modules.gemspec ${CODE_DIR}/go_modules/ -COPY gradle/Gemfile gradle/dependabot-gradle.gemspec ${CODE_DIR}/gradle/ -COPY hex/Gemfile hex/dependabot-hex.gemspec ${CODE_DIR}/hex/ -COPY maven/Gemfile maven/dependabot-maven.gemspec ${CODE_DIR}/maven/ -COPY npm_and_yarn/Gemfile npm_and_yarn/dependabot-npm_and_yarn.gemspec ${CODE_DIR}/npm_and_yarn/ -COPY nuget/Gemfile nuget/dependabot-nuget.gemspec ${CODE_DIR}/nuget/ -COPY python/Gemfile python/dependabot-python.gemspec ${CODE_DIR}/python/ -COPY terraform/Gemfile terraform/dependabot-terraform.gemspec ${CODE_DIR}/terraform/ -COPY omnibus/Gemfile omnibus/dependabot-omnibus.gemspec ${CODE_DIR}/omnibus/ - -COPY common/ ${CODE_DIR}/common/ -COPY bundler/ ${CODE_DIR}/bundler/ -COPY cargo/ ${CODE_DIR}/cargo/ -COPY composer/ ${CODE_DIR}/composer/ -COPY dep/ ${CODE_DIR}/dep/ -COPY docker/ ${CODE_DIR}/docker/ -COPY elm/ ${CODE_DIR}/elm/ -COPY git_submodules/ ${CODE_DIR}/git_submodules/ -COPY github_actions/ ${CODE_DIR}/github_actions/ -COPY go_modules/ ${CODE_DIR}/go_modules/ -COPY gradle/ ${CODE_DIR}/gradle/ -COPY hex/ ${CODE_DIR}/hex/ -COPY maven/ ${CODE_DIR}/maven/ -COPY npm_and_yarn/ ${CODE_DIR}/npm_and_yarn/ -COPY nuget/ ${CODE_DIR}/nuget/ -COPY python/ ${CODE_DIR}/python/ -COPY terraform/ ${CODE_DIR}/terraform/ -COPY omnibus/ ${CODE_DIR}/omnibus/ - RUN git config --global user.name dependabot-ci && git config --global user.email no-reply@github.com diff --git a/bin/docker-dev-shell b/bin/docker-dev-shell index a6038dde50..a69d7b6ebe 100755 --- a/bin/docker-dev-shell +++ b/bin/docker-dev-shell @@ -90,73 +90,106 @@ docker run --rm -ti \ -v "$(pwd)/.core-bash_history:/home/dependabot/.bash_history" \ -v "$(pwd)/.rubocop.yml:$CODE_DIR/.rubocop.yml" \ -v "$(pwd)/bin:$CODE_DIR/bin" \ + -v "$(pwd)/common/.rubocop.yml:$CODE_DIR/common/.rubocop.yml" \ -v "$(pwd)/common/Gemfile:$CODE_DIR/common/Gemfile" \ -v "$(pwd)/common/dependabot-common.gemspec:$CODE_DIR/common/dependabot-common.gemspec" \ -v "$(pwd)/common/bin:$CODE_DIR/common/bin" \ -v "$(pwd)/common/lib:$CODE_DIR/common/lib" \ -v "$(pwd)/common/spec:$CODE_DIR/common/spec" \ + -v "$(pwd)/common/script:$CODE_DIR/common/script" \ + -v "$(pwd)/terraform/.rubocop.yml:$CODE_DIR/terraform/.rubocop.yml" \ -v "$(pwd)/terraform/Gemfile:$CODE_DIR/terraform/Gemfile" \ -v "$(pwd)/terraform/dependabot-terraform.gemspec:$CODE_DIR/terraform/dependabot-terraform.gemspec" \ -v "$(pwd)/terraform/lib:$CODE_DIR/terraform/lib" \ -v "$(pwd)/terraform/spec:$CODE_DIR/terraform/spec" \ + -v "$(pwd)/terraform/script:$CODE_DIR/terraform/script" \ + -v "$(pwd)/elm/.rubocop.yml:$CODE_DIR/elm/.rubocop.yml" \ -v "$(pwd)/elm/Gemfile:$CODE_DIR/elm/Gemfile" \ -v "$(pwd)/elm/dependabot-elm.gemspec:$CODE_DIR/elm/dependabot-elm.gemspec" \ -v "$(pwd)/elm/lib:$CODE_DIR/elm/lib" \ -v "$(pwd)/elm/spec:$CODE_DIR/elm/spec" \ + -v "$(pwd)/elm/script:$CODE_DIR/elm/script" \ + -v "$(pwd)/docker/.rubocop.yml:$CODE_DIR/docker/.rubocop.yml" \ -v "$(pwd)/docker/Gemfile:$CODE_DIR/docker/Gemfile" \ -v "$(pwd)/docker/dependabot-docker.gemspec:$CODE_DIR/docker/dependabot-docker.gemspec" \ -v "$(pwd)/docker/lib:$CODE_DIR/docker/lib" \ -v "$(pwd)/docker/spec:$CODE_DIR/docker/spec" \ + -v "$(pwd)/docker/script:$CODE_DIR/docker/script" \ + -v "$(pwd)/git_submodules/.rubocop.yml:$CODE_DIR/git_submodules/.rubocop.yml" \ -v "$(pwd)/git_submodules/Gemfile:$CODE_DIR/git_submodules/Gemfile" \ -v "$(pwd)/git_submodules/dependabot-git_submodules.gemspec:$CODE_DIR/git_submodules/dependabot-core.gemspec" \ -v "$(pwd)/git_submodules/lib:$CODE_DIR/git_submodules/lib" \ -v "$(pwd)/git_submodules/spec:$CODE_DIR/git_submodules/spec" \ + -v "$(pwd)/git_submodules/script:$CODE_DIR/git_submodules/script" \ + -v "$(pwd)/github_actions/.rubocop.yml:$CODE_DIR/github_actions/.rubocop.yml" \ -v "$(pwd)/github_actions/Gemfile:$CODE_DIR/github_actions/Gemfile" \ -v "$(pwd)/github_actions/dependabot-github_actions.gemspec:$CODE_DIR/github_actions/dependabot-core.gemspec" \ -v "$(pwd)/github_actions/lib:$CODE_DIR/github_actions/lib" \ -v "$(pwd)/github_actions/spec:$CODE_DIR/github_actions/spec" \ + -v "$(pwd)/github_actions/script:$CODE_DIR/github_actions/script" \ + -v "$(pwd)/python/.rubocop.yml:$CODE_DIR/python/.rubocop.yml" \ -v "$(pwd)/python/Gemfile:$CODE_DIR/python/Gemfile" \ -v "$(pwd)/python/dependabot-python.gemspec:$CODE_DIR/python/dependabot-python.gemspec" \ -v "$(pwd)/python/helpers:/opt/python" \ -v "$(pwd)/python/helpers:/opt/python/helpers" \ -v "$(pwd)/python/lib:$CODE_DIR/python/lib" \ -v "$(pwd)/python/spec:$CODE_DIR/python/spec" \ + -v "$(pwd)/python/script:$CODE_DIR/python/script" \ + -v "$(pwd)/nuget/.rubocop.yml:$CODE_DIR/nuget/.rubocop.yml" \ -v "$(pwd)/nuget/Gemfile:$CODE_DIR/nuget/Gemfile" \ -v "$(pwd)/nuget/dependabot-nuget.gemspec:$CODE_DIR/nuget/dependabot-core.gemspec" \ -v "$(pwd)/nuget/lib:$CODE_DIR/nuget/lib" \ -v "$(pwd)/nuget/spec:$CODE_DIR/nuget/spec" \ + -v "$(pwd)/nuget/script:$CODE_DIR/nuget/script" \ + -v "$(pwd)/maven/.rubocop.yml:$CODE_DIR/maven/.rubocop.yml" \ -v "$(pwd)/maven/Gemfile:$CODE_DIR/maven/Gemfile" \ -v "$(pwd)/maven/dependabot-maven.gemspec:$CODE_DIR/maven/dependabot-core.gemspec" \ -v "$(pwd)/maven/lib:$CODE_DIR/maven/lib" \ -v "$(pwd)/maven/spec:$CODE_DIR/maven/spec" \ + -v "$(pwd)/maven/script:$CODE_DIR/maven/script" \ + -v "$(pwd)/gradle/.rubocop.yml:$CODE_DIR/gradle/.rubocop.yml" \ -v "$(pwd)/gradle/Gemfile:$CODE_DIR/gradle/Gemfile" \ -v "$(pwd)/gradle/dependabot-gradle.gemspec:$CODE_DIR/gradle/dependabot-gradle.gemspec" \ -v "$(pwd)/gradle/lib:$CODE_DIR/gradle/lib" \ -v "$(pwd)/gradle/spec:$CODE_DIR/gradle/spec" \ + -v "$(pwd)/gradle/script:$CODE_DIR/gradle/script" \ + -v "$(pwd)/hex/.rubocop.yml:$CODE_DIR/hex/.rubocop.yml" \ -v "$(pwd)/hex/Gemfile:$CODE_DIR/hex/Gemfile" \ -v "$(pwd)/hex/dependabot-hex.gemspec:$CODE_DIR/hex/dependabot-hex.gemspec" \ -v "$(pwd)/hex/lib:$CODE_DIR/hex/lib" \ -v "$(pwd)/hex/spec:$CODE_DIR/hex/spec" \ + -v "$(pwd)/hex/script:$CODE_DIR/hex/script" \ + -v "$(pwd)/cargo/.rubocop.yml:$CODE_DIR/cargo/.rubocop.yml" \ -v "$(pwd)/cargo/Gemfile:$CODE_DIR/cargo/Gemfile" \ -v "$(pwd)/cargo/dependabot-cargo.gemspec:$CODE_DIR/cargo/dependabot-core.gemspec" \ -v "$(pwd)/cargo/lib:$CODE_DIR/cargo/lib" \ -v "$(pwd)/cargo/spec:$CODE_DIR/cargo/spec" \ + -v "$(pwd)/cargo/script:$CODE_DIR/cargo/script" \ + -v "$(pwd)/dep/.rubocop.yml:$CODE_DIR/dep/.rubocop.yml" \ -v "$(pwd)/dep/Gemfile:$CODE_DIR/dep/Gemfile" \ -v "$(pwd)/dep/dependabot-dep.gemspec:$CODE_DIR/dep/dependabot-dep.gemspec" \ -v "$(pwd)/dep/lib:$CODE_DIR/dep/lib" \ -v "$(pwd)/dep/spec:$CODE_DIR/dep/spec" \ + -v "$(pwd)/dep/script:$CODE_DIR/dep/script" \ + -v "$(pwd)/go_modules/.rubocop.yml:$CODE_DIR/go_modules/.rubocop.yml" \ -v "$(pwd)/go_modules/Gemfile:$CODE_DIR/go_modules/Gemfile" \ -v "$(pwd)/go_modules/dependabot-go_modules.gemspec:$CODE_DIR/go_modules/dependabot-go_modules.gemspec" \ -v "$(pwd)/go_modules/lib:$CODE_DIR/go_modules/lib" \ -v "$(pwd)/go_modules/spec:$CODE_DIR/go_modules/spec" \ + -v "$(pwd)/go_modules/script:$CODE_DIR/go_modules/script" \ + -v "$(pwd)/npm_and_yarn/.rubocop.yml:$CODE_DIR/npm_and_yarn/.rubocop.yml" \ -v "$(pwd)/npm_and_yarn/Gemfile:$CODE_DIR/npm_and_yarn/Gemfile" \ -v "$(pwd)/npm_and_yarn/dependabot-npm_and_yarn.gemspec:$CODE_DIR/npm_and_yarn/dependabot-npm_and_yarn.gemspec" \ -v "$(pwd)/npm_and_yarn/lib:$CODE_DIR/npm_and_yarn/lib" \ -v "$(pwd)/npm_and_yarn/spec:$CODE_DIR/npm_and_yarn/spec" \ + -v "$(pwd)/npm_and_yarn/script:$CODE_DIR/npm_and_yarn/script" \ + -v "$(pwd)/composer/.rubocop.yml:$CODE_DIR/composer/.rubocop.yml" \ -v "$(pwd)/composer/Gemfile:$CODE_DIR/composer/Gemfile" \ -v "$(pwd)/composer/dependabot-composer.gemspec:$CODE_DIR/composer/dependabot-composer.gemspec" \ -v "$(pwd)/composer/lib:$CODE_DIR/composer/lib" \ -v "$(pwd)/composer/spec:$CODE_DIR/composer/spec" \ + -v "$(pwd)/composer/script:$CODE_DIR/composer/script" \ + -v "$(pwd)/bundler/.rubocop.yml:$CODE_DIR/bundler/.rubocop.yml" \ -v "$(pwd)/bundler/Gemfile:$CODE_DIR/bundler/Gemfile" \ -v "$(pwd)/bundler/dependabot-bundler.gemspec:$CODE_DIR/bundler/dependabot-bundler.gemspec" \ -v "$(pwd)/bundler/lib:$CODE_DIR/bundler/lib" \ @@ -164,10 +197,13 @@ docker run --rm -ti \ -v "$(pwd)/bundler/spec:$CODE_DIR/bundler/spec" \ -v "$(pwd)/bundler/helpers:/opt/bundler" \ -v "$(pwd)/bundler/helpers:/opt/bundler/helpers" \ + -v "$(pwd)/bundler/script:$CODE_DIR/bundler/script" \ + -v "$(pwd)/omnibus/.rubocop.yml:$CODE_DIR/omnibus/.rubocop.yml" \ -v "$(pwd)/omnibus/Gemfile:$CODE_DIR/omnibus/Gemfile" \ -v "$(pwd)/omnibus/dependabot-omnibus.gemspec:$CODE_DIR/omnibus/dependabot-omnibus.gemspec" \ -v "$(pwd)/omnibus/lib:$CODE_DIR/omnibus/lib" \ -v "$(pwd)/omnibus/spec:$CODE_DIR/omnibus/spec" \ + -v "$(pwd)/omnibus/script:$CODE_DIR/omnibus/script" \ -v "$(pwd)/tmp:/$CODE_DIR/tmp" \ --name "$CONTAINER_NAME" \ --env "LOCAL_GITHUB_ACCESS_TOKEN=$LOCAL_GITHUB_ACCESS_TOKEN" \ From 4aa0b921efc808e56826788196c67ac9bf6e7f78 Mon Sep 17 00:00:00 2001 From: Philip Harrison Date: Tue, 6 Apr 2021 11:35:28 +0100 Subject: [PATCH 2/2] Build base image from cache --- .github/workflows/docker.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 103a77f6c9..2f4821f7b9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,27 +13,33 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Prepare environment variables + run: | + echo "BASE_IMAGE=ubuntu:18.04" >> $GITHUB_ENV + echo "CORE_IMAGE=dependabot/dependabot-core" >> $GITHUB_ENV - name: Pull Docker base image & warm Docker cache run: | - docker pull "dependabot/dependabot-core:latest" + docker pull "$BASE_IMAGE" + docker pull "$CORE_IMAGE:latest" - name: Build dependabot-core image env: DOCKER_BUILDKIT: 1 run: | docker build \ - -t "dependabot/dependabot-core:latest" \ + -t "$CORE_IMAGE:latest" \ --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from "$BASE_IMAGE" \ + --cache-from "$CORE_IMAGE:latest" \ . - name: Log in to the Docker registry run: | echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - - name: Push image to packages (latest) - if: "!contains(github.ref, 'refs/tags')" + - name: Push latest image run: | - docker push "dependabot/dependabot-core:latest" - - name: Push image to packages (tagged) + docker push "$CORE_IMAGE:latest" + - name: Push tagged image if: "contains(github.ref, 'refs/tags')" run: | VERSION="$(grep -Eo "[0-9]+\.[0-9]+\.[0-9]+" common/lib/dependabot/version.rb)" - docker tag "dependabot/dependabot-core:latest" "dependabot/dependabot-core:$VERSION" - docker push "dependabot/dependabot-core:$VERSION" + docker tag "$CORE_IMAGE:latest" "$CORE_IMAGE:$VERSION" + docker push "$CORE_IMAGE:$VERSION"