Merge pull request #17 from felddy/improvement/pyusb

Improvement/pyusb
felddy · Sep 21, 2020 · ef8a59c · ef8a59c
2 parents a52183c + 55621e7
commit ef8a59c
Show file tree

Hide file tree

Showing 17 changed files with 357 additions and 80 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -17,29 +17,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: 3.8
       - name: Store installed Python version
         run: |
           echo "::set-env name=PY_VERSION::"\
           "$(python -c "import platform;print(platform.python_version())")"
-      - name: Cache pip test requirements
-        uses: actions/cache@v1
+      - name: Cache linting environments
+        uses: actions/cache@v2
         with:
-          path: ${{ env.PIP_CACHE_DIR }}
-          key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\
-            ${{ hashFiles('**/requirements-test.txt') }}"
-          restore-keys: |
-            ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-
-            ${{ runner.os }}-pip-test-
-            ${{ runner.os }}-pip-
-      - name: Cache pre-commit hooks
-        uses: actions/cache@v1
-        with:
-          path: ${{ env.PRE_COMMIT_CACHE_DIR }}
-          key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\
+          path: |
+            ${{ env.PIP_CACHE_DIR }}
+            ${{ env.PRE_COMMIT_CACHE_DIR }}
+          key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\
+            ${{ hashFiles('**/requirements-test.txt') }}-\
+            ${{ hashFiles('**/requirements.txt') }}-\
             ${{ hashFiles('**/.pre-commit-config.yaml') }}"
+          restore-keys: |
+            lint-${{ runner.os }}-py${{ env.PY_VERSION }}-
+            lint-${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -61,12 +58,15 @@ jobs:
         uses: crazy-max/ghaction-docker-buildx@v1
         with:
           version: latest
+      - name: Create cross-platform support Dockerfile-x
+        run: ./buildx-dockerfile.sh
       - name: Build docker image
         run: |
           mkdir -p dist
           IFS='.' read -r -a version_array \
             <<< "${{ steps.get_ver.outputs.version }}"
           docker buildx build \
+            --file Dockerfile-x \
             --platform linux/amd64 \
             --output "type=docker,dest=dist/image.tar" \
             --tag "$IMAGE_NAME:latest" \
@@ -89,18 +89,17 @@ jobs:
     needs: [build]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: 3.8
-      - name: Cache pip test requirements
-        uses: actions/cache@v1
+      - name: Cache testing environments
+        uses: actions/cache@v2
         with:
           path: ${{ env.PIP_CACHE_DIR }}
-          key: "${{ runner.os }}-pip-test-\
+          key: "test-${{ runner.os }}-\
             ${{ hashFiles('**/requirements-test.txt') }}"
           restore-keys: |
-            ${{ runner.os }}-pip-test-
-            ${{ runner.os }}-pip-
+            test-${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -111,13 +110,18 @@ jobs:
           name: dist
       - name: Load docker image
         run: docker load < dist/image.tar.gz
-      - name: Create data volume mount
+      - name: Prepare data volume mount
         run: |
-          mkdir data
           chmod a+rwx data
       - name: Generate test configuration
         run: docker-compose run weewx --gen-test-config
       - name: Run tests
         env:
           RELEASE_TAG: ${{ github.event.release.tag_name }}
         run: pytest
+      - name: Upload data artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v1
+        with:
+          name: data
+          path: data
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: 3.8
       - name: Determine image version
@@ -32,17 +32,20 @@ jobs:
           version: latest
       - name: Log into docker registry
         run: docker login --username "$DOCKER_USER" --password "$DOCKER_PW"
+      - name: Create cross-platform support Dockerfile-x
+        run: ./buildx-dockerfile.sh
       - name: Build Docker images, tag, and publish
         run: |
-          IFS='.' read -r -a version_array \
+          IFS='.' read -r v_major v_minor v_patch \
             <<< "${{ steps.get_ver.outputs.version }}"
           docker buildx build \
+            --file Dockerfile-x \
             --platform $PLATFORMS \
             --output "type=image,push=true" \
             --tag "$IMAGE_NAME:latest" \
             --tag "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" \
-            --tag "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" \
-            --tag "${IMAGE_NAME}:${version_array[0]}" \
+            --tag "${IMAGE_NAME}:${v_major}.${v_minor}" \
+            --tag "${IMAGE_NAME}:${v_major}" \
             --build-arg GIT_COMMIT=$(git log -1 --format=%H) \
             --build-arg GIT_REMOTE=$(git remote get-url origin) \
             --build-arg VERSION=${{ steps.get_ver.outputs.version }} \

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,5 @@
+__pycache__
 .mypy_cache
 .pytest_cache
 .python-version
-__pycache__
-data/*
+Dockerfile-x
diff --git a/.isort.cfg b/.isort.cfg
@@ -10,3 +10,6 @@ import_heading_firstparty=Local Libraries
 known_third_party=pytest
 # These must be manually set to correctly separate them from third party libraries
 known_first_party=
+
+# Run isort under the black profile to align with our other Python linting
+profile=black
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v3.2.0
     hooks:
       - id: check-executables-have-shebangs
       - id: check-json
@@ -28,27 +28,27 @@ repos:
       - id: requirements-txt-fixer
       - id: trailing-whitespace
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.23.0
+    rev: v0.23.2
     hooks:
       - id: markdownlint
         args:
           - --config=.mdl_config.json
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.23.0
+    rev: v1.24.2
     hooks:
       - id: yamllint
   - repo: https://github.com/detailyang/pre-commit-shell
     rev: 1.0.5
     hooks:
       - id: shell-lint
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.1
+    rev: 3.8.3
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-docstrings
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.4.1
+    rev: v2.7.2
     hooks:
       - id: pyupgrade
   # Run bandit on "tests" tree with a configuration
@@ -68,23 +68,23 @@ repos:
         name: bandit (everything else)
         exclude: tests
   - repo: https://github.com/python/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
       - id: black
   - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.1
+    rev: v2.2.0
     hooks:
       - id: seed-isort-config
   - repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.5.0
     hooks:
       - id: isort
   - repo: https://github.com/ansible/ansible-lint.git
-    rev: v4.3.0a0
+    rev: v4.3.4
     hooks:
       - id: ansible-lint
   - repo: https://github.com/antonbabenko/pre-commit-terraform.git
-    rev: v1.30.0
+    rev: v1.37.0
     hooks:
       - id: terraform_fmt
       # There are ongoing issues with how this command works. This issue
@@ -104,14 +104,14 @@ repos:
       # Terraform 0.13.
       # - id: terraform_validate
   - repo: https://github.com/IamTheFij/docker-pre-commit
-    rev: v1.0.1
+    rev: v2.0.0
     hooks:
       - id: docker-compose-check
   - repo: https://github.com/prettier/prettier
-    rev: 2.0.5
+    rev: 2.1.1
     hooks:
       - id: prettier
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.770
+    rev: v0.782
     hooks:
       - id: mypy
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -46,10 +46,22 @@ There are a few ways to do this, but we prefer to use
 create and manage a Python virtual environment specific to this
 project.
 
+If you already have `pyenv` and `pyenv-virtualenv` configured you can
+take advantage of the `setup-env` tool in this repo to automate the
+entire environment configuration process.
+
+```console
+./setup-env
+```
+
+Otherwise, follow the steps below to manually configure your
+environment.
+
 #### Installing and using `pyenv` and `pyenv-virtualenv` ####
 
-On the Mac, installation is as simple as `brew install pyenv
-pyenv-virtualenv` and adding this to your profile:
+On the Mac, we recommend installing [brew](https://brew.sh/).  Then
+installation is as simple as `brew install pyenv pyenv-virtualenv` and
+adding this to your profile:
 
 ```bash
 eval "$(pyenv init -)"

diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified
 ARG GIT_REMOTE=unspecified
 ARG VERSION=unspecified
 
-FROM --platform=$TARGETPLATFORM python:3-alpine as stage-1
+FROM python:3-alpine as stage-1
 
 ARG WEEWX_UID=421
 ENV WEEWX_HOME="/home/weewx"
@@ -36,7 +36,7 @@ WORKDIR ${WEEWX_HOME}
 RUN bin/wee_extension --install /tmp/weewx-mqtt.zip
 COPY src/entrypoint.sh src/version.txt ./
 
-FROM --platform=$TARGETPLATFORM python:3-alpine as stage-2
+FROM python:3-slim as stage-2
 
 ARG GIT_COMMIT
 ARG GIT_REMOTE
@@ -58,7 +58,7 @@ ENV WEEWX_VERSION="4.1.1"
 RUN addgroup --system --gid ${WEEWX_UID} weewx \
   && adduser --system --uid ${WEEWX_UID} --ingroup weewx weewx
 
-RUN apk --no-cache add su-exec tzdata
+RUN apt-get update && apt-get install -y libusb-1.0-0 gosu busybox-syslogd tzdata
 
 WORKDIR ${WEEWX_HOME}
 

diff --git a/README.md b/README.md
@@ -128,25 +128,48 @@ services:
 | WEEWX_UID    | `uid` the daemon will be run under | weewx |
 | WEEWX_GID    | `gid` the deamon will be run under | weewx |
 
-## Building ##
+## Building from source ##
 
-This Docker container has multi-platform support and requires
-the use of the
-[`buildx` experimental feature](https://docs.docker.com/buildx/working-with-buildx/).
-Make sure to enable experimental features in your environment.
-
-To build the container from source:
+Build the image locally using this git repository as the [build context](https://docs.docker.com/engine/reference/commandline/build/#git-repositories):
 
 ```console
-git clone https://github.com/felddy/weewx-docker.git
-cd weewx-docker
-docker buildx build \
-  --platform linux/amd64 \
+docker build \
   --build-arg VERSION=4.1.1 \
-  --output type=docker \
-  --tag felddy/weewx .
+  --tag felddy/weewx:4.1.1 \
+  https://github.com/felddy/weewx-docker.git#develop
 ```
 
+## Cross-platform builds ##
+
+To create images that are compatible with other platforms you can use the
+[`buildx`](https://docs.docker.com/buildx/working-with-buildx/) feature of
+Docker:
+
+1. Copy the project to your machine using the `Clone` button above
+   or the command line:
+
+    ```console
+    git clone https://github.com/felddy/weewx-docker.git
+    cd weewx-docker
+    ```
+
+1. Create the `Dockerfile-x` file with `buildx` platform support:
+
+    ```console
+    ./buildx-dockerfile.sh
+    ```
+
+1. Build the image using `buildx`:
+
+    ```console
+    docker buildx build \
+      --file Dockerfile-x \
+      --platform linux/amd64 \
+      --build-arg VERSION=4.1.1 \
+      --output type=docker \
+      --tag felddy/weewx:4.1.1 .
+    ```
+
 ## Debugging ##
 
 There are a few helper arguments that can be used to diagnose container issues

diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Create a Dockerfile suitable for a multi-platform build using buildx
+# See: https://docs.docker.com/buildx/working-with-buildx/
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+DOCKERFILE=Dockerfile
+DOCKERFILEX=Dockerfile-x
+
+# We don't want this expression to expand.
+# shellcheck disable=SC2016
+sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX
diff --git a/data/.gitignore b/data/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore