diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6fbd475..fabd7456 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,29 +17,26 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ "$(python -c "import platform;print(platform.python_version())")" - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache linting environments + uses: actions/cache@v2 with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ + path: | + ${{ env.PIP_CACHE_DIR }} + ${{ env.PRE_COMMIT_CACHE_DIR }} + key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -61,12 +58,15 @@ jobs: uses: crazy-max/ghaction-docker-buildx@v1 with: version: latest + - name: Create cross-platform support Dockerfile-x + run: ./buildx-dockerfile.sh - name: Build docker image run: | mkdir -p dist IFS='.' read -r -a version_array \ <<< "${{ steps.get_ver.outputs.version }}" docker buildx build \ + --file Dockerfile-x \ --platform linux/amd64 \ --output "type=docker,dest=dist/image.tar" \ --tag "$IMAGE_NAME:latest" \ @@ -89,18 +89,17 @@ jobs: needs: [build] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache testing environments + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ + key: "test-${{ runner.os }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -111,9 +110,8 @@ jobs: name: dist - name: Load docker image run: docker load < dist/image.tar.gz - - name: Create data volume mount + - name: Prepare data volume mount run: | - mkdir data chmod a+rwx data - name: Generate test configuration run: docker-compose run weewx --gen-test-config @@ -121,3 +119,9 @@ jobs: env: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest + - name: Upload data artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v1 + with: + name: data + path: data diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a5caa649..2b5edcec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Determine image version @@ -32,17 +32,20 @@ jobs: version: latest - name: Log into docker registry run: docker login --username "$DOCKER_USER" --password "$DOCKER_PW" + - name: Create cross-platform support Dockerfile-x + run: ./buildx-dockerfile.sh - name: Build Docker images, tag, and publish run: | - IFS='.' read -r -a version_array \ + IFS='.' read -r v_major v_minor v_patch \ <<< "${{ steps.get_ver.outputs.version }}" docker buildx build \ + --file Dockerfile-x \ --platform $PLATFORMS \ --output "type=image,push=true" \ --tag "$IMAGE_NAME:latest" \ --tag "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" \ - --tag "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" \ - --tag "${IMAGE_NAME}:${version_array[0]}" \ + --tag "${IMAGE_NAME}:${v_major}.${v_minor}" \ + --tag "${IMAGE_NAME}:${v_major}" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ diff --git a/.gitignore b/.gitignore index dc1a2bf0..bceb4ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ +__pycache__ .mypy_cache .pytest_cache .python-version -__pycache__ -data/* +Dockerfile-x diff --git a/.isort.cfg b/.isort.cfg index ade9bd8d..31813cdb 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -10,3 +10,6 @@ import_heading_firstparty=Local Libraries known_third_party=pytest # These must be manually set to correctly separate them from third party libraries known_first_party= + +# Run isort under the black profile to align with our other Python linting +profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a3268f5..ebfe4cdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.2.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -28,13 +28,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.23.0 + rev: v1.24.2 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -42,13 +42,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.4.1 + rev: v2.7.2 hooks: - id: pyupgrade # Run bandit on "tests" tree with a configuration @@ -68,23 +68,23 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/python/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 + rev: v2.2.0 hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.5.0 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a0 + rev: v4.3.4 hooks: - id: ansible-lint - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.30.0 + rev: v1.37.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -104,14 +104,14 @@ repos: # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.1 + rev: v2.0.0 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.5 + rev: 2.1.1 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.770 + rev: v0.782 hooks: - id: mypy diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae3c5f9c..78ed6b42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,10 +46,22 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```console +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/Dockerfile b/Dockerfile index 60caf076..9dc0dac5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified ARG GIT_REMOTE=unspecified ARG VERSION=unspecified -FROM --platform=$TARGETPLATFORM python:3-alpine as stage-1 +FROM python:3-alpine as stage-1 ARG WEEWX_UID=421 ENV WEEWX_HOME="/home/weewx" @@ -36,7 +36,7 @@ WORKDIR ${WEEWX_HOME} RUN bin/wee_extension --install /tmp/weewx-mqtt.zip COPY src/entrypoint.sh src/version.txt ./ -FROM --platform=$TARGETPLATFORM python:3-alpine as stage-2 +FROM python:3-slim as stage-2 ARG GIT_COMMIT ARG GIT_REMOTE @@ -58,7 +58,7 @@ ENV WEEWX_VERSION="4.1.1" RUN addgroup --system --gid ${WEEWX_UID} weewx \ && adduser --system --uid ${WEEWX_UID} --ingroup weewx weewx -RUN apk --no-cache add su-exec tzdata +RUN apt-get update && apt-get install -y libusb-1.0-0 gosu busybox-syslogd tzdata WORKDIR ${WEEWX_HOME} diff --git a/README.md b/README.md index 978a4f6b..2c916f0c 100644 --- a/README.md +++ b/README.md @@ -128,25 +128,48 @@ services: | WEEWX_UID | `uid` the daemon will be run under | weewx | | WEEWX_GID | `gid` the deamon will be run under | weewx | -## Building ## +## Building from source ## -This Docker container has multi-platform support and requires -the use of the -[`buildx` experimental feature](https://docs.docker.com/buildx/working-with-buildx/). -Make sure to enable experimental features in your environment. - -To build the container from source: +Build the image locally using this git repository as the [build context](https://docs.docker.com/engine/reference/commandline/build/#git-repositories): ```console -git clone https://github.com/felddy/weewx-docker.git -cd weewx-docker -docker buildx build \ - --platform linux/amd64 \ +docker build \ --build-arg VERSION=4.1.1 \ - --output type=docker \ - --tag felddy/weewx . + --tag felddy/weewx:4.1.1 \ + https://github.com/felddy/weewx-docker.git#develop ``` +## Cross-platform builds ## + +To create images that are compatible with other platforms you can use the +[`buildx`](https://docs.docker.com/buildx/working-with-buildx/) feature of +Docker: + +1. Copy the project to your machine using the `Clone` button above + or the command line: + + ```console + git clone https://github.com/felddy/weewx-docker.git + cd weewx-docker + ``` + +1. Create the `Dockerfile-x` file with `buildx` platform support: + + ```console + ./buildx-dockerfile.sh + ``` + +1. Build the image using `buildx`: + + ```console + docker buildx build \ + --file Dockerfile-x \ + --platform linux/amd64 \ + --build-arg VERSION=4.1.1 \ + --output type=docker \ + --tag felddy/weewx:4.1.1 . + ``` + ## Debugging ## There are a few helper arguments that can be used to diagnose container issues diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh new file mode 100755 index 00000000..46710e9a --- /dev/null +++ b/buildx-dockerfile.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Create a Dockerfile suitable for a multi-platform build using buildx +# See: https://docs.docker.com/buildx/working-with-buildx/ + +set -o nounset +set -o errexit +set -o pipefail + +DOCKERFILE=Dockerfile +DOCKERFILEX=Dockerfile-x + +# We don't want this expression to expand. +# shellcheck disable=SC2016 +sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/docker-compose-pytest.yml b/docker-compose-pytest.yml new file mode 100644 index 00000000..fb199923 --- /dev/null +++ b/docker-compose-pytest.yml @@ -0,0 +1,31 @@ +--- +version: "3.7" + +volumes: + data: + +# This docker-compose file is used to build and test the container +services: + weewx: + # Run the container normally + image: felddy/weewx:4.1.1 + init: true + restart: "no" + volumes: + - type: bind + source: ./data + target: /data + environment: + - TIMEZONE=US/Eastern + - WEEWX_UID=weewx + - WEEWX_GID=dialout + # devices: + # - "/dev/ttyUSB0:/dev/ttyUSB0" + + + weewx-version: + # Run the container to collect version information + image: felddy/weewx:4.1.1 + init: true + restart: "no" + command: --version diff --git a/docker-compose.yml b/docker-compose.yml index 018ce5c7..60bab865 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,16 +4,14 @@ version: "3.7" volumes: data: -# This docker-compose file is used to build and test the container services: weewx: - # Run the container normally build: args: - VERSION=4.1.1 context: . dockerfile: Dockerfile - image: felddy/weewx + image: felddy/weewx:4.1.1 init: true restart: "no" volumes: @@ -26,11 +24,3 @@ services: - WEEWX_GID=dialout # devices: # - "/dev/ttyUSB0:/dev/ttyUSB0" - - - weewx-version: - # Run the container to collect version information - image: felddy/weewx - init: true - restart: "no" - command: --version diff --git a/pytest.ini b/pytest.ini index d302749a..a0e33e52 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = --verbose -ra +addopts = --verbose -ra --dockerc-filepath=docker-compose-pytest.yml diff --git a/requirements-test.txt b/requirements-test.txt index fc5504e1..5f3337c0 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ +--requirement requirements.txt pre-commit pytest pytest-dockerc diff --git a/requirements.txt b/requirements.txt index 8ff8b2b9..cb0e03fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ configobj paho-mqtt pyserial +pyusb +setuptools +wheel diff --git a/setup-env b/setup-env new file mode 100755 index 00000000..4d822c47 --- /dev/null +++ b/setup-env @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +Configure a developement environment for this repository. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements needed for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +Usage: + setup-env [options] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virtual environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. +pyenv local "${env_name}" + +# Upgrade pip and friends +python3 -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]]; then + pip install --requirement $req_file + break + fi +done + +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print("No lineage configuration found.", file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 5761b4bd..8c5e7adb 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -1,17 +1,14 @@ -#!/bin/sh +#!/bin/bash set -o nounset set -o errexit -# Sha-bang cannot be /bin/bash (not available), but -# the container's /bin/sh does support pipefail. -# shellcheck disable=SC2039 set -o pipefail CONF_FILE="/data/weewx.conf" # echo version before starting syslog so we don't confound our tests if [ "$1" = "--version" ]; then - su-exec weewx:weewx ./bin/weewxd --version + gosu weewx:weewx ./bin/weewxd --version exit 0 fi @@ -20,9 +17,12 @@ if [ "$(id -u)" = 0 ]; then ln -snf /usr/share/zoneinfo/"${TIMEZONE:-UTC}" /etc/localtime # start the syslog daemon as root /sbin/syslogd -n -S -O - & - # drop privileges and restart this script as weewx user - su-exec "${WEEWX_UID:-weewx}:${WEEWX_GID:-weewx}" "$(readlink -f "$0")" "$@" - exit 0 + if [ "${WEEWX_UID:-weewx}" != 0 ]; then + # drop privileges and restart this script + echo "Switching uid:gid to ${WEEWX_UID:-weewx}:${WEEWX_GID:-weewx}" + gosu "${WEEWX_UID:-weewx}:${WEEWX_GID:-weewx}" "$(readlink -f "$0")" "$@" + exit 0 + fi fi copy_default_config() {