diff --git a/.travis.yml b/.travis.yml index 98cea2fd..9f8a4963 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,12 +33,20 @@ before_deploy: - export PATH="$PATH:$HOME/.rvm/bin" - export PHANTOMJS_CDNURL=http://cnpmjs.org/downloads deploy: - provider: script - script: ".travis/deploy.sh" - skip_cleanup: true - on: - repo: WorldBank-Transport/DRIVER - branch: develop + - provider: script + script: ".travis/deploy_staging.sh" + skip_cleanup: true + on: + repo: WorldBank-Transport/DRIVER + branch: develop + - provider: script + script: ".travis/deploy.sh" + skip_cleanup: true + on: + repo: WorldBank-Transport/DRIVER + branch: master + tags: true + after_deploy: - rm deployment/driver.pem - rm gradle/data/driver.keystore diff --git a/.travis/deploy.sh b/.travis/deploy.sh index be611c36..90fcd478 100755 --- a/.travis/deploy.sh +++ b/.travis/deploy.sh @@ -5,35 +5,7 @@ set -x for image in app editor web gradle analysis; do - docker push "quay.io/azavea/driver-${image}:${TRAVIS_COMMIT:0:7}" - docker tag "quay.io/azavea/driver-${image}:${TRAVIS_COMMIT:0:7}" "quay.io/azavea/driver-${image}:latest" - docker push "quay.io/azavea/driver-${image}:latest" + docker push "quay.io/azavea/driver-${image}:${TRAVIS_TAG}" + docker tag "quay.io/azavea/driver-${image}:${TRAVIS_TAG}" "quay.io/azavea/driver-${image}:${TRAVIS_TAG}" + docker push "quay.io/azavea/driver-${image}:${TRAVIS_TAG}" done - -# TODO: The below was for deploying to our staging environment, which we no longer have. -# If staging is ever resurrected, we can re-enable this. -# Set up group vars -#echo "setting up group vars..." -#set +x -#grep -v nominatim_key deployment/ansible/group_vars/all.example \ -# > deployment/ansible/group_vars/all -#echo "web_js_nominatim_key: \"${NOMINATIM_API_KEY}\"" \ -# >> deployment/ansible/group_vars/all -#echo "oauth_client_id: \"${OAUTH_CLIENT_ID}\"" \ -# >> deployment/ansible/group_vars/all -#echo "oauth_client_secret: \"${OAUTH_CLIENT_SECRET}\"" \ -# >> deployment/ansible/group_vars/all -#echo "forecast_io_api_key: \"${FORECAST_IO_API_KEY}\"" \ -# >> deployment/ansible/group_vars/all -#echo "keystore_password: \"${DRIVER_KEYSTORE_PASSWORD}\"" \ -# >> deployment/ansible/group_vars/all -#set -x - -#ansible-galaxy install -f -r deployment/ansible/roles.yml -p deployment/ansible/roles - -#chmod 0600 deployment/driver.pem - -#ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook \ -# -i deployment/ansible/inventory/staging \ -# --private-key="${TRAVIS_BUILD_DIR}/deployment/driver.pem" \ -# deployment/ansible/database.yml deployment/ansible/app.yml deployment/ansible/celery.yml diff --git a/.travis/deploy_staging.sh b/.travis/deploy_staging.sh new file mode 100755 index 00000000..be611c36 --- /dev/null +++ b/.travis/deploy_staging.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -e +set -x + +for image in app editor web gradle analysis; +do + docker push "quay.io/azavea/driver-${image}:${TRAVIS_COMMIT:0:7}" + docker tag "quay.io/azavea/driver-${image}:${TRAVIS_COMMIT:0:7}" "quay.io/azavea/driver-${image}:latest" + docker push "quay.io/azavea/driver-${image}:latest" +done + +# TODO: The below was for deploying to our staging environment, which we no longer have. +# If staging is ever resurrected, we can re-enable this. +# Set up group vars +#echo "setting up group vars..." +#set +x +#grep -v nominatim_key deployment/ansible/group_vars/all.example \ +# > deployment/ansible/group_vars/all +#echo "web_js_nominatim_key: \"${NOMINATIM_API_KEY}\"" \ +# >> deployment/ansible/group_vars/all +#echo "oauth_client_id: \"${OAUTH_CLIENT_ID}\"" \ +# >> deployment/ansible/group_vars/all +#echo "oauth_client_secret: \"${OAUTH_CLIENT_SECRET}\"" \ +# >> deployment/ansible/group_vars/all +#echo "forecast_io_api_key: \"${FORECAST_IO_API_KEY}\"" \ +# >> deployment/ansible/group_vars/all +#echo "keystore_password: \"${DRIVER_KEYSTORE_PASSWORD}\"" \ +# >> deployment/ansible/group_vars/all +#set -x + +#ansible-galaxy install -f -r deployment/ansible/roles.yml -p deployment/ansible/roles + +#chmod 0600 deployment/driver.pem + +#ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook \ +# -i deployment/ansible/inventory/staging \ +# --private-key="${TRAVIS_BUILD_DIR}/deployment/driver.pem" \ +# deployment/ansible/database.yml deployment/ansible/app.yml deployment/ansible/celery.yml diff --git a/deployment/ansible/group_vars/all.example b/deployment/ansible/group_vars/all.example index 48e05332..99f9ea21 100644 --- a/deployment/ansible/group_vars/all.example +++ b/deployment/ansible/group_vars/all.example @@ -1,4 +1,7 @@ --- +# This is used ONLY when deploying to production to determine which docker image is used. +# To use a specific version locally, you'll need to `git checkout tags/` +app_version: "1.0.0" developing: "{{ 'development' in group_names }}" developing_or_staging: "{{ 'development' in group_names or 'staging' in group_names }}" not_developing: "{{ 'development' not in group_names }}" @@ -9,6 +12,9 @@ production: "{{ 'production' in group_names }}" docker_version: "1.10.*" docker_py_version: "1.2.3" docker_options: "--storage-driver=aufs" +# In local development, always use the 'latest' tag regardless of what app_version says; +# users need to checkout a tag in order to get a specific version. +docker_image_tag: "{% if developing %}latest{% else %}{{app_version}}{% endif %}" python_version: "2.7.*" pip_version: "1.*" diff --git a/deployment/ansible/roles/driver.app/tasks/main.yml b/deployment/ansible/roles/driver.app/tasks/main.yml index 2eed71df..da29ab50 100644 --- a/deployment/ansible/roles/driver.app/tasks/main.yml +++ b/deployment/ansible/roles/driver.app/tasks/main.yml @@ -17,20 +17,20 @@ command: > docker build -f {{ root_app_dir }}/Dockerfile.base - -t quay.io/azavea/driver-app:latest + -t quay.io/azavea/driver-app:{{ docker_image_tag }} {{ root_app_dir }} when: developing -- name: Build application Docker image +- name: Build application Docker image (dev tools) command: > docker build -f {{ root_app_dir }}/Dockerfile.development - -t quay.io/azavea/driver-app:latest + -t quay.io/azavea/driver-app:{{ docker_image_tag }} {{ root_app_dir }} when: developing - name: Pull application Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-app:latest + command: /usr/bin/docker pull quay.io/azavea/driver-app:{{ docker_image_tag }} when: staging or production notify: - Restart driver-app diff --git a/deployment/ansible/roles/driver.app/templates/upstart-app.conf.j2 b/deployment/ansible/roles/driver.app/templates/upstart-app.conf.j2 index ba709a23..1dd46154 100644 --- a/deployment/ansible/roles/driver.app/templates/upstart-app.conf.j2 +++ b/deployment/ansible/roles/driver.app/templates/upstart-app.conf.j2 @@ -16,7 +16,7 @@ pre-start script /usr/bin/docker rm driver-app || true {% if 'development' not in group_names -%} - /usr/bin/docker pull quay.io/azavea/driver-app:latest + /usr/bin/docker pull quay.io/azavea/driver-app:{{ docker_image_tag }} {% endif %} end script @@ -41,7 +41,7 @@ exec /usr/bin/docker run \ --env DRIVER_ADMIN_GROUP='{{ driver_admin_group }}' \ --log-driver syslog \ {% if celery %} --entrypoint '/bin/bash -c' \ -{% endif %} quay.io/azavea/driver-app:latest {% if celery %} \ +{% endif %} quay.io/azavea/driver-app:{{ docker_image_tag }} {% if celery %} \ celery -A driver worker --queue=path --loglevel=info --concurrency=4 {% endif %} diff --git a/deployment/ansible/roles/driver.celery/tasks/main.yml b/deployment/ansible/roles/driver.celery/tasks/main.yml index 382f41ea..157582c1 100644 --- a/deployment/ansible/roles/driver.celery/tasks/main.yml +++ b/deployment/ansible/roles/driver.celery/tasks/main.yml @@ -3,20 +3,20 @@ command: > docker build -f {{ root_app_dir }}/Dockerfile.base - -t quay.io/azavea/driver-app:latest + -t quay.io/azavea/driver-app:{{ docker_image_tag }} {{ root_app_dir }} when: developing -- name: Build application Docker image +- name: Build application Docker image (dev tools) command: > docker build -f {{ root_app_dir }}/Dockerfile.development - -t quay.io/azavea/driver-app:latest + -t quay.io/azavea/driver-app:{{ docker_image_tag }} {{ root_app_dir }} when: developing - name: Pull application Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-app:latest + command: /usr/bin/docker pull quay.io/azavea/driver-app:{{ docker_image_tag}} when: staging or production notify: - Restart driver-celery diff --git a/deployment/ansible/roles/driver.celery/templates/upstart-celery.conf.j2 b/deployment/ansible/roles/driver.celery/templates/upstart-celery.conf.j2 index d12ffbc5..5ed8fd34 100644 --- a/deployment/ansible/roles/driver.celery/templates/upstart-celery.conf.j2 +++ b/deployment/ansible/roles/driver.celery/templates/upstart-celery.conf.j2 @@ -16,7 +16,7 @@ pre-start script /usr/bin/docker rm driver-celery || true {% if 'development' not in group_names -%} - /usr/bin/docker pull quay.io/azavea/driver-app:latest + /usr/bin/docker pull quay.io/azavea/driver-app:{{ docker_image_tag }} {% endif %} end script @@ -36,7 +36,7 @@ exec /usr/bin/docker run \ {% endfor -%} --log-driver syslog \ --entrypoint 'celery' \ - quay.io/azavea/driver-app:latest \ + quay.io/azavea/driver-app:{{ docker_image_tag }} \ -A driver worker --queue=taskworker \ --loglevel={% if 'development' in group_names %}debug{% else %}info{% endif %} \ --concurrency=4 \ diff --git a/deployment/ansible/roles/driver.gradle/tasks/main.yml b/deployment/ansible/roles/driver.gradle/tasks/main.yml index 72f93fb1..a0bd7d2f 100644 --- a/deployment/ansible/roles/driver.gradle/tasks/main.yml +++ b/deployment/ansible/roles/driver.gradle/tasks/main.yml @@ -17,12 +17,12 @@ command: > docker build -f {{ gradle_dir }}/Dockerfile - -t quay.io/azavea/driver-gradle:latest + -t quay.io/azavea/driver-gradle:{{ docker_image_tag }} {{ gradle_dir }} when: developing - name: Pull gradle Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-gradle:latest + command: /usr/bin/docker pull quay.io/azavea/driver-gradle:{{ docker_image_tag }} when: staging or production - name: Configure gradle service definition diff --git a/deployment/ansible/roles/driver.gradle/templates/upstart-gradle.conf.j2 b/deployment/ansible/roles/driver.gradle/templates/upstart-gradle.conf.j2 index 3b72594a..2401d65d 100644 --- a/deployment/ansible/roles/driver.gradle/templates/upstart-gradle.conf.j2 +++ b/deployment/ansible/roles/driver.gradle/templates/upstart-gradle.conf.j2 @@ -16,7 +16,7 @@ pre-start script /usr/bin/docker rm driver-gradle || true {% if 'development' not in group_names -%} - /usr/bin/docker pull quay.io/azavea/driver-gradle:latest + /usr/bin/docker pull quay.io/azavea/driver-gradle:{{ docker_image_tag }} {% endif %} end script @@ -28,7 +28,7 @@ exec /usr/bin/docker run \ --env {{ k }}="{{ v }}" \ {% endfor -%} --log-driver syslog \ - quay.io/azavea/driver-gradle:latest + quay.io/azavea/driver-gradle:{{ docker_image_tag }} post-stop script /usr/bin/docker kill driver-gradle diff --git a/deployment/ansible/roles/driver.r-docker/tasks/main.yml b/deployment/ansible/roles/driver.r-docker/tasks/main.yml index d49b3b33..721f93bf 100644 --- a/deployment/ansible/roles/driver.r-docker/tasks/main.yml +++ b/deployment/ansible/roles/driver.r-docker/tasks/main.yml @@ -3,10 +3,10 @@ command: > docker build -f {{ root_analysis_dir }}/Dockerfile - -t quay.io/azavea/driver-analysis:latest + -t quay.io/azavea/driver-analysis:{{ docker_image_tag }} {{ root_analysis_dir }} when: developing - name: Pull application Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-analysis:latest + command: /usr/bin/docker pull quay.io/azavea/driver-analysis:{{ docker_image_tag }} when: staging or production diff --git a/deployment/ansible/roles/driver.web/tasks/development.yml b/deployment/ansible/roles/driver.web/tasks/development.yml index e0aa5eb2..427f6927 100644 --- a/deployment/ansible/roles/driver.web/tasks/development.yml +++ b/deployment/ansible/roles/driver.web/tasks/development.yml @@ -3,7 +3,7 @@ command: > /usr/bin/docker build -f {{ editor_dir }}/Dockerfile - -t quay.io/azavea/driver-editor:latest + -t quay.io/azavea/driver-editor:{{ docker_image_tag }} {{ editor_dir }} - name: Build production editor app @@ -13,14 +13,14 @@ --name driver-editor --volume {{ editor_dir }}:{{ editor_dir }} --log-driver syslog - quay.io/azavea/driver-editor:latest + quay.io/azavea/driver-editor:{{ docker_image_tag }} build - name: Build web Docker image command: > /usr/bin/docker build -f {{ web_dir }}/Dockerfile - -t quay.io/azavea/driver-web:latest + -t quay.io/azavea/driver-web:{{ docker_image_tag }} {{ web_dir }} - name: Build production web app @@ -31,5 +31,5 @@ --volume {{ editor_dir }}:{{ editor_dir }} --volume {{ web_dir }}:{{ web_dir }} --log-driver syslog - quay.io/azavea/driver-web:latest + quay.io/azavea/driver-web:{{ docker_image_tag }} build diff --git a/deployment/ansible/roles/driver.web/tasks/staging_or_production.yml b/deployment/ansible/roles/driver.web/tasks/staging_or_production.yml index 845fe071..b30e856c 100644 --- a/deployment/ansible/roles/driver.web/tasks/staging_or_production.yml +++ b/deployment/ansible/roles/driver.web/tasks/staging_or_production.yml @@ -1,6 +1,6 @@ --- - name: Pull editor Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-editor:latest + command: /usr/bin/docker pull quay.io/azavea/driver-editor:{{ docker_image_tag }} - name: Build production editor app command: > @@ -10,7 +10,7 @@ --volume {{ editor_scripts_dir }}/config.js:{{ editor_scripts_dir }}/config.js --volume {{ editor_build_dir }}:{{ editor_build_dir }} --log-driver syslog - quay.io/azavea/driver-editor:latest + quay.io/azavea/driver-editor:{{ docker_image_tag }} build - name: Create editor data container @@ -18,10 +18,10 @@ /usr/bin/docker create --volume {{ editor_dir }} --name driver-editor - quay.io/azavea/driver-editor:latest + quay.io/azavea/driver-editor:{{ docker_image_tag }} - name: Pull web Docker image - command: /usr/bin/docker pull quay.io/azavea/driver-web:latest + command: /usr/bin/docker pull quay.io/azavea/driver-web:{{ docker_image_tag }} - name: Build production web app command: > @@ -34,7 +34,7 @@ --volume {{ web_dir }}/app/index.html:{{ web_dir }}/app/index.html --volume {{ web_build_dir }}:{{ web_build_dir }} --log-driver syslog - quay.io/azavea/driver-web:latest + quay.io/azavea/driver-web:{{ docker_image_tag }} build - name: Remove editor data container diff --git a/doc/system-administration.md b/doc/system-administration.md index 91dba535..098e1760 100644 --- a/doc/system-administration.md +++ b/doc/system-administration.md @@ -28,9 +28,11 @@ Each of these servers also includes: ## Deploying updates -Deploying updates to production is done using [Ansible](https://www.ansible.com/) (version must be at minimum 1.8). An `ansible-playbook` command is run, which uses the local configuration of the application files in the `deployment` directory to deploy updates to the remote servers. It is not necessary to have the application running locally, but it is necessary to have the latest source code, so make sure to pull down the latest version via `git pull`. In addition to the latest source code, there are four files not checked in to the repository that are needed in order to successfully deploy. These files are as follows +Deploying updates to production is done using [Ansible](https://www.ansible.com/) (version must be at minimum 1.8). An `ansible-playbook` command is run, which uses the local configuration of the application files in the `deployment` directory to deploy updates to the remote servers. It is not necessary to have the application running locally, but it is necessary to have the source code for the version you want to deploy, so make sure to pull down the latest version via `git pull` and then `git checkout tags/`. In addition to the latest source code, there are four files not checked in to the repository that are needed in order to successfully deploy. These files are as follows 1. `production` group_vars - Defines the configured settings for the system. Must be placed in: `deployment/ansible/group_vars/`. + - Make sure to edit the `app_version` flag at the top of the `production` group_vars file to specify the version of the app you want to deploy + - Check out that tag with `git checkout tags/` 2. `production` inventory - Defines the remote servers where code will be deployed. Must be placed in: `deployment/ansible/inventory/`. 3. `driver.keystore` - The signing keystore required for building Android APKs. Must be placed in: `gradle/data/`. 4. ssh identity file - The private key used for logging into the servers. This does not need to be placed in a particular location, but must be added via `ssh-add` before starting the deploy, so commands may be run on remote machines.