Skip to content

Commit

Permalink
Merge pull request #109 from RADAR-base/oura-release
Browse files Browse the repository at this point in the history
Update Docker files for Oura release
  • Loading branch information
mpgxvii authored Nov 28, 2023
2 parents da6a657 + 0673b47 commit 02c79f3
Show file tree
Hide file tree
Showing 13 changed files with 393 additions and 5 deletions.
1 change: 1 addition & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ jobs:
- name: Build docker
uses: docker/build-push-action@v3
with:
file: ./kafka-connect-fitbit-source/Dockerfile
context: .
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
Expand Down
142 changes: 142 additions & 0 deletions .github/workflows/oura.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
# Continuous integration, including test and integration test
name: Main Oura test

# Run in master and dev branches and in all pull requests to those branches
on:
push:
branches: [ master, dev ]
pull_request:
branches: [ master, dev ]

env:
DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source

jobs:
# Build and test the code
kotlin:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3

- uses: actions/setup-java@v3
with:
distribution: temurin
java-version: 11

- name: Setup Gradle
uses: gradle/gradle-build-action@v2

- name: Compile code
run: gradle assemble
working-directory: ./kafka-connect-oura-source

# Gradle check
- name: Check
run: gradle check
working-directory: ./kafka-connect-oura-source

docker:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3

- name: Docker build parameters
id: docker_params
run: |
HAS_DOCKER_LOGIN=${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
echo "has_docker_login=$HAS_DOCKER_LOGIN" >> $GITHUB_OUTPUT
if [ "${{ github.event_name == 'pull_request' }}" = "true" ] || [ "$HAS_DOCKER_LOGIN" = "false" ]; then
echo "push=false" >> $GITHUB_OUTPUT
echo "load=true" >> $GITHUB_OUTPUT
echo "platforms=linux/amd64" >> $GITHUB_OUTPUT
else
echo "push=true" >> $GITHUB_OUTPUT
echo "load=false" >> $GITHUB_OUTPUT
echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT
fi
- name: Cache Docker layers
id: cache_buildx
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-${{ hashFiles('**/Dockerfile', '**/*.gradle', 'gradle.properties', '.dockerignore', '*/src/main/**', 'docker/**') }}
restore-keys: |
${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-
${{ runner.os }}-buildx-
- name: Login to Docker Hub
if: steps.docker_params.outputs.has_docker_login == 'true'
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

# Add Docker labels and tags
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v4
with:
images: ${{ env.DOCKER_IMAGE }}

# Setup docker build environment
- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Cache parameters
id: cache-parameters
run: |
if [ "${{ steps.cache_buildx.outputs.cache-hit }}" = "true" ]; then
echo "::set-output name=cache-to::"
else
echo "::set-output name=cache-to::type=local,dest=/tmp/.buildx-cache-new,mode=max"
fi
- name: Build docker
uses: docker/build-push-action@v3
with:
file: ./kafka-connect-oura-source/Dockerfile
context: .
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
platforms: ${{ steps.docker_params.outputs.platforms }}
load: ${{ steps.docker_params.outputs.load }}
push: ${{ steps.docker_params.outputs.push }}
tags: ${{ steps.docker_meta.outputs.tags }}
# Use runtime labels from docker_meta as well as fixed labels
labels: |
${{ steps.docker_meta.outputs.labels }}
maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
org.opencontainers.image.description=RADAR-base upload connector backend application
org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
org.opencontainers.image.vendor=RADAR-base
org.opencontainers.image.licenses=Apache-2.0
- name: Pull docker image
if: steps.docker_params.outputs.load == 'false'
run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}

- name: Inspect docker image
run: |
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} curl --version
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move docker build cache
if: steps.cache_buildx.outputs.cache-hit != 'true'
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ jobs:
- name: Build docker
uses: docker/build-push-action@v3
with:
file: ./kafka-connect-fitbit-source/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
Expand Down
99 changes: 99 additions & 0 deletions .github/workflows/release-oura.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# Create release files
name: Release

on:
release:
types: [ published ]

env:
DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source

jobs:
uploadBackend:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
with:
distribution: temurin
java-version: 11

- name: Gradle cache
uses: actions/cache@v3
with:
# Cache gradle directories
path: |
~/.gradle/caches
~/.gradle/wrapper
# An explicit key for restoring and saving the cache
key: ${{ runner.os }}-gradle-${{ hashFiles('gradlew', '**/*.gradle', 'gradle.properties', 'gradle/**') }}
restore-keys: |
${{ runner.os }}-gradle-
# Compile code
- name: Compile code
run: ./gradlew jar

# Upload it to GitHub
- name: Upload to GitHub
uses: AButler/[email protected]
with:
files: "*/build/libs/*"
repo-token: ${{ secrets.GITHUB_TOKEN }}

# Build and push tagged release docker image
docker:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- uses: actions/checkout@v3

# Setup docker build environment
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2

- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

# Add Docker labels and tags
- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v4
with:
images: ${{ env.DOCKER_IMAGE }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Build docker
uses: docker/build-push-action@v3
with:
file: ./kafka-connect-oura-source/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
# Use runtime labels from docker_meta_backend as well as fixed labels
labels: |
${{ steps.docker_meta.outputs.labels }}
maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
org.opencontainers.image.description=RADAR-base upload connector backend application
org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
org.opencontainers.image.vendor=RADAR-base
org.opencontainers.image.licenses=Apache-2.0
- name: Inspect image
run: |
docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@ out/
.gradle/
docker/users
docker/source-fitbit.properties
docker/source-oura.properties
bin/
.DS_Store
47 changes: 45 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
version: '2.4'
version: "2.4"

volumes:
fitbit-logs: {}
Expand Down Expand Up @@ -143,7 +143,9 @@ services:
# RADAR Fitbit connector #
#---------------------------------------------------------------------------#
radar-fitbit-connector:
build: .
build:
context: .
dockerfile: ./kafka-connect-fitbit-source/Dockerfile
image: radarbase/radar-connect-fitbit-source
restart: on-failure
volumes:
Expand Down Expand Up @@ -178,3 +180,44 @@ services:
KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
KAFKA_BROKERS: 3
CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"

#---------------------------------------------------------------------------#
# RADAR Oura connector #
#---------------------------------------------------------------------------#
radar-oura-connector:
build:
context: .
dockerfile: ./kafka-connect-oura-source/Dockerfile
image: radarbase/radar-connect-oura-source
restart: on-failure
volumes:
- ./docker/source-oura.properties:/etc/kafka-connect/source-oura.properties
- ./docker/users:/var/lib/kafka-connect-oura-source/users
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
- kafka-1
- kafka-2
- kafka-3
- schema-registry-1
environment:
CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: "default"
CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
CONNECT_STATUS_STORAGE_TOPIC: "default.status"
CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-oura-source/logs/connect.offsets"
CONNECT_REST_ADVERTISED_HOST_NAME: "radar-oura-connector"
CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
CONNECTOR_PROPERTY_FILE_PREFIX: "source-oura"
KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
KAFKA_BROKERS: 3
CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"
12 changes: 12 additions & 0 deletions docker/source-oura.properties.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name=radar-oura-source
connector.class=org.radarbase.connect.rest.oura.OuraSourceConnector
tasks.max=4
rest.source.base.url=https://api.ouraring.com
rest.source.poll.interval.ms=5000
oura.api.client=?
oura.api.secret=?
oura.user.repository.class=org.radarbase.connect.rest.oura.user.OuraServiceUserRepository
oura.user.repository.url=
oura.user.repository.client.id=radar_oura_connector
oura.user.repository.client.secret=
oura.user.repository.oauth2.token.url=
File renamed without changes.
55 changes: 55 additions & 0 deletions kafka-connect-oura-source/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Copyright 2018 The Hyve
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

FROM --platform=$BUILDPLATFORM gradle:8.4-jdk11 as builder

RUN mkdir /code
WORKDIR /code

ENV GRADLE_USER_HOME=/code/.gradlecache \
GRADLE_OPTS="-Dorg.gradle.vfs.watch=false -Djdk.lang.Process.launchMechanism=vfork"

COPY buildSrc /code/buildSrc
COPY ./build.gradle.kts ./settings.gradle.kts ./gradle.properties /code/
COPY kafka-connect-oura-source/build.gradle.kts /code/kafka-connect-oura-source/
COPY oura-library/build.gradle /code/oura-library/

RUN gradle downloadDependencies copyDependencies

COPY ./kafka-connect-oura-source/src/ /code/kafka-connect-oura-source/src
COPY ./oura-library/src/ /code/oura-library/src

RUN gradle jar

FROM confluentinc/cp-kafka-connect-base:7.5.0

MAINTAINER Pauline Conde <[email protected]>

LABEL description="Kafka Oura REST API Source connector"

ENV CONNECT_PLUGIN_PATH="/usr/share/java/kafka-connect/plugins" \
WAIT_FOR_KAFKA="1"

# To isolate the classpath from the plugin path as recommended
COPY --from=builder /code/kafka-connect-oura-source/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/
COPY --from=builder /code/oura-library/build/third-party/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/

COPY --from=builder /code/kafka-connect-oura-source/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/
COPY --from=builder /code/oura-library/build/libs/*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-oura-source/

# Load topics validator
COPY --chown=appuser:appuser ./docker/kafka-wait /usr/bin/kafka-wait

# Load modified launcher
COPY --chown=appuser:appuser ./docker/launch /etc/confluent/docker/launch
Loading

0 comments on commit 02c79f3

Please sign in to comment.