From 50de9628be8d995f946bff3ff7fb6ebd7d454a9d Mon Sep 17 00:00:00 2001 From: Cotton Seed Date: Mon, 18 Mar 2019 00:58:32 -0400 Subject: [PATCH 1/5] add docker base image, use in scorecard --- docker/Dockerfile.base | 26 ++++++++++++++++++++++++++ docker/Dockerfile.spark-base | 11 +++++++++++ docker/Makefile | 21 +++++++++++++++++++++ docker/core-site.xml | 16 ++++++++++++++++ docker/hail-ci-build.sh | 4 ++++ docker/hail-ci-deploy.sh | 9 +++++++++ projects.yaml | 2 ++ scorecard/Dockerfile | 10 ++-------- scorecard/Makefile | 15 ++++++++------- scorecard/environment.yml | 9 --------- 10 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 docker/Dockerfile.base create mode 100644 docker/Dockerfile.spark-base create mode 100644 docker/Makefile create mode 100644 docker/core-site.xml create mode 100644 docker/hail-ci-build.sh create mode 100644 docker/hail-ci-deploy.sh delete mode 100644 scorecard/environment.yml diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base new file mode 100644 index 00000000000..dd7c2a26ad1 --- /dev/null +++ b/docker/Dockerfile.base @@ -0,0 +1,26 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get -y install \ + htop \ + unzip bzip2 \ + wget curl \ + emacs25-nox \ + default-jdk \ + python3 python3-pip && \ + rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install -U \ + pip decorator pylint pytest flake8 \ + requests \ + jinja2 \ + aiohttp aiodns aiohttp_jinja2 uvloop>=0.12 \ + werkzeug flask flask-cors Flask_Sockets \ + kubernetes google-cloud-storage \ + PyGithub cerberus humanize libsass authlib + +# source: https://cloud.google.com/storage/docs/gsutil_install#linux +RUN /bin/sh -c 'curl https://sdk.cloud.google.com | bash' && \ + mv /root/google-cloud-sdk / && \ + /google-cloud-sdk/bin/gcloud components install beta kubectl +ENV PATH $PATH:/google-cloud-sdk/bin diff --git a/docker/Dockerfile.spark-base b/docker/Dockerfile.spark-base new file mode 100644 index 00000000000..1de71db4863 --- /dev/null +++ b/docker/Dockerfile.spark-base @@ -0,0 +1,11 @@ +FROM base + +RUN wget -O spark-2.2.0-bin-hadoop2.7.tgz https://archive.apache.org/dist/spark/spark-2.2.0/spark-2.2.0-bin-hadoop2.7.tgz && \ + tar xzf spark-2.2.0-bin-hadoop2.7.tgz && \ + rm spark-2.2.0-bin-hadoop2.7.tgz + +RUN wget -O /spark-2.2.0-bin-hadoop2.7/jars/gcs-connector-hadoop2-latest.jar https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar +COPY core-site.xml /spark-2.2.0-bin-hadoop2.7/conf/core-site.xml + +ENV SPARK_HOME /spark-2.2.0-bin-hadoop2.7 +ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin" diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000000..417cf324aa0 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,21 @@ +.PHONY: build push deploy + +PROJECT = $(shell gcloud config get-value project) + +BASE_IMAGE = gcr.io/$(PROJECT)/base:$(shell docker images -q --no-trunc base:latest | sed -e 's,[^:]*:,,') +SPARK_BASE_IMAGE = gcr.io/$(PROJECT)/spark-base:$(shell docker images -q --no-trunc spark-base:latest | sed -e 's,[^:]*:,,') + +build: + -docker pull ubuntu:18.04 + -docker pull gcr.io/$(PROJECT)/base + -docker pull gcr.io/$(PROJECT)/spark-base + docker build . -t base -f Dockerfile.base --cache-from base,ubuntu:18.04 + docker build . -t spark-base -f Dockerfile.spark-base --cache-from spark-base,base,ubuntu:18.04 + +push: build + docker tag base $(BASE_IMAGE) + docker push $(BASE_IMAGE) + docker tag spark-base $(SPARK_BASE_IMAGE) + docker push $(SPARK_BASE_IMAGE) + +deploy: push diff --git a/docker/core-site.xml b/docker/core-site.xml new file mode 100644 index 00000000000..25775d28fef --- /dev/null +++ b/docker/core-site.xml @@ -0,0 +1,16 @@ + + + + + + + google.cloud.auth.service.account.enable + true + + + + google.cloud.auth.service.account.json.keyfile + /hail-vdc-sa-key/hail-vdc-sa-key.json + + + diff --git a/docker/hail-ci-build.sh b/docker/hail-ci-build.sh new file mode 100644 index 00000000000..07249a78a86 --- /dev/null +++ b/docker/hail-ci-build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -ex + +make build diff --git a/docker/hail-ci-deploy.sh b/docker/hail-ci-deploy.sh new file mode 100644 index 00000000000..567aeccc52a --- /dev/null +++ b/docker/hail-ci-deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex + +gcloud -q auth activate-service-account \ + --key-file=/secrets/gcr-push-service-account-key.json + +gcloud -q auth configure-docker + +make deploy diff --git a/projects.yaml b/projects.yaml index 60ec4bf45f9..6ec758e87cc 100644 --- a/projects.yaml +++ b/projects.yaml @@ -1,3 +1,4 @@ +- project: docker - project: auth-gateway - project: batch - project: ci @@ -15,6 +16,7 @@ - project: pipeline dependencies: ["batch"] - project: scorecard + dependencies: ["docker"] - project: site - project: upload - project: vdc diff --git a/scorecard/Dockerfile b/scorecard/Dockerfile index 4f23f5315ea..6c2e22b0713 100644 --- a/scorecard/Dockerfile +++ b/scorecard/Dockerfile @@ -1,13 +1,7 @@ -FROM continuumio/miniconda -MAINTAINER Hail Team - -COPY environment.yml . -RUN conda env create scorecard -f environment.yml && \ - rm -f environment.yml && \ - rm -rf /home/root/.conda/pkgs/* +FROM base COPY scorecard /scorecard EXPOSE 5000 -CMD ["bash", "-c", "source activate scorecard; python /scorecard/scorecard.py"] +CMD ["bash", "-c", "python3 /scorecard/scorecard.py"] diff --git a/scorecard/Makefile b/scorecard/Makefile index ece5d21bb04..88e57864ccc 100644 --- a/scorecard/Makefile +++ b/scorecard/Makefile @@ -2,23 +2,24 @@ PROJECT = $(shell gcloud config get-value project) +SCORECARD_IMAGE = gcr.io/$(PROJECT)/scorecard:$(shell docker images -q --no-trunc scorecard:latest | sed -e 's,[^:]*:,,') + build: - docker build . -t scorecard + -docker pull gcr.io/$(PROJECT)/scorecard + docker build . -t scorecard --cache-from scorecard,base,ubuntu:18.04 -push: IMAGE = gcr.io/$(PROJECT)/scorecard:$(shell docker images -q --no-trunc scorecard | sed -e 's,[^:]*:,,') push: build - echo $(IMAGE) > scorecard-image - docker tag scorecard $(IMAGE) - docker push $(IMAGE) + docker tag scorecard $(SCORECARD_IMAGE) + docker push $(SCORECARD_IMAGE) run-docker: build - docker run -i -p 5000:5000 -v secrets:/secrets -t scorecard + docker run -i -p 5000:5000 -v `pwd`/secrets:/secrets -t scorecard run: GITHUB_TOKEN_PATH=secrets/scorecard-github-access-token.txt python scorecard/scorecard.py deploy: push sed -e "s,@sha@,$(shell git rev-parse --short=12 HEAD)," \ - -e "s,@image@,$(shell cat scorecard-image)," \ + -e "s,@image@,$(SCORECARD_IMAGE)," \ < deployment.yaml.in > deployment.yaml kubectl -n default apply -f deployment.yaml diff --git a/scorecard/environment.yml b/scorecard/environment.yml deleted file mode 100644 index 8e5e8105d85..00000000000 --- a/scorecard/environment.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: scorecard -dependencies: -- python=3.7 -- flask -- flask-cors -- humanize -- pip -- pip: - - PyGithub From 69154ac657d3ddf897b1521b6453cf0c122cca3d Mon Sep 17 00:00:00 2001 From: Cotton Seed Date: Mon, 18 Mar 2019 01:06:24 -0400 Subject: [PATCH 2/5] fix permissions --- docker/hail-ci-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/hail-ci-build.sh b/docker/hail-ci-build.sh index 07249a78a86..9c64f46308c 100644 --- a/docker/hail-ci-build.sh +++ b/docker/hail-ci-build.sh @@ -1,4 +1,9 @@ #!/bin/bash set -ex +gcloud -q auth activate-service-account \ + --key-file=/secrets/gcr-push-service-account-key.json + +gcloud -q auth configure-docker + make build From a386b8e27a4ee98280444b22d34d834bcdb76f96 Mon Sep 17 00:00:00 2001 From: Cotton Seed Date: Mon, 18 Mar 2019 01:31:59 -0400 Subject: [PATCH 3/5] remove build which doesn't have pull permissions --- docker/hail-ci-build.sh | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 docker/hail-ci-build.sh diff --git a/docker/hail-ci-build.sh b/docker/hail-ci-build.sh deleted file mode 100644 index 9c64f46308c..00000000000 --- a/docker/hail-ci-build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -ex - -gcloud -q auth activate-service-account \ - --key-file=/secrets/gcr-push-service-account-key.json - -gcloud -q auth configure-docker - -make build From 8625aed55c0cc9acc17c984ad547f938c20457ec Mon Sep 17 00:00:00 2001 From: Cotton Seed Date: Mon, 18 Mar 2019 11:58:50 -0400 Subject: [PATCH 4/5] added pyasyncinit --- docker/Dockerfile.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index dd7c2a26ad1..1501f14204f 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -14,7 +14,7 @@ RUN python3 -m pip install -U \ pip decorator pylint pytest flake8 \ requests \ jinja2 \ - aiohttp aiodns aiohttp_jinja2 uvloop>=0.12 \ + pyasyncinit aiohttp aiodns aiohttp_jinja2 uvloop>=0.12 \ werkzeug flask flask-cors Flask_Sockets \ kubernetes google-cloud-storage \ PyGithub cerberus humanize libsass authlib From 2407da5b5d8dbe55d847bf1cccb667cda9722b8e Mon Sep 17 00:00:00 2001 From: Cotton Seed Date: Mon, 18 Mar 2019 18:49:22 -0400 Subject: [PATCH 5/5] reorder, install jdk-headless. --- docker/Dockerfile.base | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 1501f14204f..fe33fa55508 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -6,10 +6,16 @@ RUN apt-get update && \ unzip bzip2 \ wget curl \ emacs25-nox \ - default-jdk \ + default-jdk-headless \ python3 python3-pip && \ rm -rf /var/lib/apt/lists/* +# source: https://cloud.google.com/storage/docs/gsutil_install#linux +RUN /bin/sh -c 'curl https://sdk.cloud.google.com | bash' && \ + mv /root/google-cloud-sdk / && \ + /google-cloud-sdk/bin/gcloud components install beta kubectl +ENV PATH $PATH:/google-cloud-sdk/bin + RUN python3 -m pip install -U \ pip decorator pylint pytest flake8 \ requests \ @@ -18,9 +24,3 @@ RUN python3 -m pip install -U \ werkzeug flask flask-cors Flask_Sockets \ kubernetes google-cloud-storage \ PyGithub cerberus humanize libsass authlib - -# source: https://cloud.google.com/storage/docs/gsutil_install#linux -RUN /bin/sh -c 'curl https://sdk.cloud.google.com | bash' && \ - mv /root/google-cloud-sdk / && \ - /google-cloud-sdk/bin/gcloud components install beta kubectl -ENV PATH $PATH:/google-cloud-sdk/bin