Merge pull request #110 from RADAR-base/release-0.5.0

Release 0.5.0

.github/workflows/main.yml

@@ -104,6 +104,7 @@ jobs:
       - name: Build docker
         uses: docker/build-push-action@v3
         with:
+          file: ./kafka-connect-fitbit-source/Dockerfile
           context: .
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: ${{ steps.cache-parameters.outputs.cache-to }}

.github/workflows/oura.yml

@@ -0,0 +1,142 @@
+# Continuous integration, including test and integration test
+name: Main Oura test
+
+# Run in master and dev branches and in all pull requests to those branches
+on:
+  push:
+    branches: [ master, dev ]
+  pull_request:
+    branches: [ master, dev ]
+
+env:
+  DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
+
+jobs:
+  # Build and test the code
+  kotlin:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 11
+
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
+      - name: Compile code
+        run: gradle assemble
+        working-directory: ./kafka-connect-oura-source
+
+      # Gradle check
+      - name: Check
+        run: gradle check
+        working-directory: ./kafka-connect-oura-source
+
+  docker:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - name: Docker build parameters
+        id: docker_params
+        run: |
+          HAS_DOCKER_LOGIN=${{ secrets.DOCKERHUB_USERNAME != ''  && secrets.DOCKERHUB_TOKEN != '' }}
+          echo "has_docker_login=$HAS_DOCKER_LOGIN" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name == 'pull_request' }}" = "true" ] || [ "$HAS_DOCKER_LOGIN" = "false" ]; then
+            echo "push=false" >> $GITHUB_OUTPUT
+            echo "load=true" >> $GITHUB_OUTPUT
+            echo "platforms=linux/amd64" >> $GITHUB_OUTPUT
+          else
+            echo "push=true" >> $GITHUB_OUTPUT
+            echo "load=false" >> $GITHUB_OUTPUT
+            echo "platforms=linux/amd64,linux/arm64" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Cache Docker layers
+        id: cache_buildx
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-${{ hashFiles('**/Dockerfile', '**/*.gradle', 'gradle.properties', '.dockerignore', '*/src/main/**', 'docker/**') }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ steps.docker_params.outputs.push }}-
+            ${{ runner.os }}-buildx-
+
+      - name: Login to Docker Hub
+        if: steps.docker_params.outputs.has_docker_login == 'true'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Add Docker labels and tags
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_IMAGE }}
+
+      # Setup docker build environment
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache parameters
+        id: cache-parameters
+        run: |
+          if [ "${{ steps.cache_buildx.outputs.cache-hit }}" = "true" ]; then
+            echo "::set-output name=cache-to::"
+          else
+            echo "::set-output name=cache-to::type=local,dest=/tmp/.buildx-cache-new,mode=max"
+          fi
+
+      - name: Build docker
+        uses: docker/build-push-action@v3
+        with:
+          file: ./kafka-connect-oura-source/Dockerfile
+          context: .
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
+          platforms: ${{ steps.docker_params.outputs.platforms }}
+          load: ${{ steps.docker_params.outputs.load }}
+          push: ${{ steps.docker_params.outputs.push }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          # Use runtime labels from docker_meta as well as fixed labels
+          labels: |
+            ${{ steps.docker_meta.outputs.labels }}
+            maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
+            org.opencontainers.image.description=RADAR-base upload connector backend application
+            org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
+            org.opencontainers.image.vendor=RADAR-base
+            org.opencontainers.image.licenses=Apache-2.0
+
+      - name: Pull docker image
+        if: steps.docker_params.outputs.load == 'false'
+        run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+
+      - name: Inspect docker image
+        run: |
+          docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} curl --version
+
+
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move docker build cache
+        if: steps.cache_buildx.outputs.cache-hit != 'true'
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/release.yml → .github/workflows/release-fitbit.yml

@@ -79,6 +79,7 @@ jobs:
       - name: Build docker
         uses: docker/build-push-action@v3
         with:
+          file: ./kafka-connect-fitbit-source/Dockerfile
           context: .
           platforms: linux/amd64,linux/arm64
           push: true

.github/workflows/release-oura.yml

@@ -0,0 +1,99 @@
+# Create release files
+name: Release
+
+on:
+  release:
+    types: [ published ]
+
+env:
+  DOCKER_IMAGE: radarbase/kafka-connect-rest-oura-source
+
+jobs:
+  uploadBackend:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 11
+
+      - name: Gradle cache
+        uses: actions/cache@v3
+        with:
+          # Cache gradle directories
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          # An explicit key for restoring and saving the cache
+          key: ${{ runner.os }}-gradle-${{ hashFiles('gradlew', '**/*.gradle', 'gradle.properties', 'gradle/**') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      # Compile code
+      - name: Compile code
+        run: ./gradlew jar
+
+      # Upload it to GitHub
+      - name: Upload to GitHub
+        uses: AButler/[email protected]
+        with:
+          files: "*/build/libs/*"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+  # Build and push tagged release docker image
+  docker:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - uses: actions/checkout@v3
+
+      # Setup docker build environment
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Add Docker labels and tags
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_IMAGE }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build docker
+        uses: docker/build-push-action@v3
+        with:
+          file: ./kafka-connect-oura-source/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          # Use runtime labels from docker_meta_backend as well as fixed labels
+          labels: |
+            ${{ steps.docker_meta.outputs.labels }}
+            maintainer=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
+            org.opencontainers.image.description=RADAR-base upload connector backend application
+            org.opencontainers.image.authors=Pauline Conde <[email protected]>, Yatharth Ranjan <[email protected]>
+            org.opencontainers.image.vendor=RADAR-base
+            org.opencontainers.image.licenses=Apache-2.0
+
+      - name: Inspect image
+        run: |
+          docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}

.gitignore

@@ -6,3 +6,6 @@ out/
 .gradle/
 docker/users
 docker/source-fitbit.properties
+docker/source-oura.properties
+bin/
+.DS_Store

build.gradle.kts

@@ -6,10 +6,17 @@ plugins {
     id("org.radarbase.radar-kotlin") version Versions.radarCommons apply false
 }
 
+repositories {
+    // Use jcenter for resolving dependencies.
+    // You can declare any Maven/Ivy/file repository here.
+    mavenCentral()
+}
+
 description = "Kafka connector for REST API sources"
 
 radarRootProject {
     projectVersion.set(Versions.project)
+    gradleVersion.set(Versions.wrapper)
 }
 
 subprojects {

buildSrc/build.gradle.kts

@@ -1,5 +1,5 @@
 plugins {
-    kotlin("jvm") version "1.8.10"
+    kotlin("jvm") version "1.9.10"
 }
 
 repositories {

buildSrc/src/main/kotlin/Versions.kt

@@ -1,28 +1,31 @@
+@Suppress("ConstPropertyName", "MemberVisibilityCanBePrivate")
 object Versions {
-    const val project = "0.4.1"
+    const val project = "0.5.0"
 
     const val java = 11
-    const val kotlin = "1.8.21"
+    const val kotlin = "1.9.10"
+    const val wrapper = "8.4"
 
-    const val radarCommons = "1.0.0"
-    const val confluent = "7.4.0"
+    const val radarCommons = "1.1.1"
+    const val confluent = "7.5.0"
     const val kafka = "$confluent-ce"
+    const val avro = "1.11.0"
 
     // From image
     const val jackson = "2.14.2"
 
     const val log4j2 = "2.20.0"
-    const val slf4j = "2.0.7"
+    const val slf4j = "2.0.9"
 
     const val okhttp = "4.11.0"
 
-    const val managementPortal = "2.0.0"
-
     const val firebaseAdmin = "9.1.0"
-    const val radarSchemas = "0.8.3"
+    const val radarSchemas = "0.8.6"
+    const val ktor = "2.3.5"
 
     const val junit = "5.9.3"
-    const val hamcrest = "2.2"
     const val wiremock = "2.27.2"
     const val mockito = "5.3.1"
+
+    const val kotlinVersion = "1.8.21"
 }

docker-compose.yml

@@ -1,5 +1,5 @@
 ---
-version: '2.4'
+version: "2.4"
 
 volumes:
   fitbit-logs: {}
@@ -143,7 +143,9 @@ services:
   # RADAR Fitbit connector                                                     #
   #---------------------------------------------------------------------------#
   radar-fitbit-connector:
-    build: .
+    build:
+      context: .
+      dockerfile: ./kafka-connect-fitbit-source/Dockerfile
     image: radarbase/radar-connect-fitbit-source
     restart: on-failure
     volumes:
@@ -178,3 +180,44 @@ services:
       KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
       KAFKA_BROKERS: 3
       CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"
+
+  #---------------------------------------------------------------------------#
+  # RADAR Oura connector                                                     #
+  #---------------------------------------------------------------------------#
+  radar-oura-connector:
+    build:
+      context: .
+      dockerfile: ./kafka-connect-oura-source/Dockerfile
+    image: radarbase/radar-connect-oura-source
+    restart: on-failure
+    volumes:
+      - ./docker/source-oura.properties:/etc/kafka-connect/source-oura.properties
+      - ./docker/users:/var/lib/kafka-connect-oura-source/users
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - kafka-1
+      - kafka-2
+      - kafka-3
+      - schema-registry-1
+    environment:
+      CONNECT_BOOTSTRAP_SERVERS: PLAINTEXT://kafka-1:9092,PLAINTEXT://kafka-2:9092,PLAINTEXT://kafka-3:9092
+      CONNECT_REST_PORT: 8083
+      CONNECT_GROUP_ID: "default"
+      CONNECT_CONFIG_STORAGE_TOPIC: "default.config"
+      CONNECT_OFFSET_STORAGE_TOPIC: "default.offsets"
+      CONNECT_STATUS_STORAGE_TOPIC: "default.status"
+      CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
+      CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
+      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry-1:8081"
+      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
+      CONNECT_OFFSET_STORAGE_FILE_FILENAME: "/var/lib/kafka-connect-oura-source/logs/connect.offsets"
+      CONNECT_REST_ADVERTISED_HOST_NAME: "radar-oura-connector"
+      CONNECT_ZOOKEEPER_CONNECT: zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181
+      CONNECTOR_PROPERTY_FILE_PREFIX: "source-oura"
+      KAFKA_HEAP_OPTS: "-Xms256m -Xmx768m"
+      KAFKA_BROKERS: 3
+      CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"