Add HTTP server and JSON API

.devcontainer/devcontainer.json

@@ -1,10 +1,11 @@
 {
-  "name": "FTL x86_64 Build Env",  
-  "image": "ghcr.io/pi-hole/ftl-build:x86_64", 
+  "name": "FTL x86_64 Build Env",
+  "image": "ghcr.io/pi-hole/ftl-build:x86_64-musl",
+  "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
   "extensions": [
     "jetmartin.bats",
     "ms-vscode.cpptools",
     "ms-vscode.cmake-tools",
     "eamodio.gitlens"
-  ],  
-}
+  ]
+}

.github/.codespellignore

@@ -2,3 +2,6 @@ ssudo
 tre
 ede
 nd
+doubleclick
+requestor
+requestors

.github/.codespellignore_lines

@@ -0,0 +1 @@
+			self.errors.append("Exception when GETing from FTL: " + str(e))

.github/Dockerfile

@@ -0,0 +1,33 @@
+ARG BUILDER="alpine"
+FROM ghcr.io/pi-hole/ftl-build:ftl-build-buildx-${BUILDER} AS builder
+
+WORKDIR /app
+
+COPY . /app
+
+ARG CI_ARCH="linux/amd64"
+ENV CI_ARCH ${CI_ARCH}
+ARG GIT_BRANCH="test"
+ENV GIT_BRANCH ${GIT_BRANCH}
+ARG GIT_TAG="test"
+ENV GIT_TAG ${GIT_TAG}
+
+# Build FTL
+# Remove possible old build files
+RUN rm -rf cmake && \
+# Build FTL
+    bash build.sh "-DSTATIC=${STATIC}" && \
+# Run binary architecture tests
+    bash test/arch_test.sh && \
+# Run full test suite
+    bash test/run.sh && \
+# Move FTL binary to root directory
+    cd / &&\
+    mv /app/pihole-FTL . && \
+# Create tarball of API docs
+    tar -C /app/src/api/docs/content/ -czvf /api-docs.tar.gz .
+
+# Create final image containing only the FTL binary and API docs
+FROM scratch AS result
+COPY --from=builder /pihole-FTL /pihole-FTL
+COPY --from=builder /api-docs.tar.gz /api-docs.tar.gz

.github/workflows/build.yml

@@ -50,81 +50,102 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
-
     needs: smoke-tests
-
-    container: ghcr.io/pi-hole/ftl-build:v1.26-${{ matrix.arch }}
-
     strategy:
       fail-fast: false
-
       matrix:
         include:
-          - arch: x86_64
-            bin_name: pihole-FTL-linux-x86_64
-          - arch: x86_64
-            arch_extra: _full
-            bin_name: pihole-FTL-linux-x86_64_full
-          - arch: x86_64-musl
-            bin_name: pihole-FTL-musl-linux-x86_64
-          - arch: x86_32
-            bin_name: pihole-FTL-linux-x86_32
-          - arch: armv4t
-            bin_name: pihole-FTL-armv4-linux-gnueabi
-          - arch: armv5te
-            bin_name: pihole-FTL-armv5-linux-gnueabi
-          - arch: armv6hf
-            bin_name:  pihole-FTL-armv6-linux-gnueabihf
-          - arch: armv7hf
-            bin_name: pihole-FTL-armv7-linux-gnueabihf
-          - arch: armv8a
-            bin_name: pihole-FTL-armv8-linux-gnueabihf
-          - arch: aarch64
-            bin_name: pihole-FTL-aarch64-linux-gnu
-          - arch: riscv64
-            bin_name: pihole-FTL-riscv64-linux-gnu
+          - platform: linux/amd64
+            bin_name: pihole-FTL-amd64
+            builder: alpine
+          - platform: linux/386
+            bin_name: pihole-FTL-386
+            builder: alpine
+          - platform: linux/arm/v5
+            bin_name: pihole-FTL-armv4
+            builder: debian
+          - platform: linux/arm/v6
+            bin_name: pihole-FTL-armv5
+            builder: debian
+          - platform: linux/arm/v6
+            bin_name: pihole-FTL-armv6
+            builder: alpine
+          - platform: linux/arm/v7
+            bin_name: pihole-FTL-armv7
+            builder: alpine
+          - platform: linux/arm64/v8
+            bin_name: pihole-FTL-arm64
+            builder: alpine
+          - platform: linux/riscv64
+            bin_name: pihole-FTL-riscv64
+            builder: alpine
 
     env:
-      CI_ARCH: ${{ matrix.arch }}${{ matrix.arch_extra }}
+      CI_ARCH: ${{ matrix.platform }}
+      GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
+      GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }}
 
     steps:
       -
         name: Checkout code
         uses: actions/[email protected]
       -
-        name: "Fix ownership of repository"
-        run: chown -R root .
-      -
-        name: "Fix ownership of repository"
-        run: chown -R root .
+        name: Set up Docker Buildx
+        uses: docker/[email protected]
       -
-        name: "Build"
-        env:
-          GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
-          GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }}
-        run: |
-          bash build.sh "-DSTATIC=${STATIC}"
+        name: Print directory contents
+        run: ls -l
       -
-        name: "Binary checks"
-        run: |
-          bash test/arch_test.sh
-      -
-        name: "Test x86_32/64 binaries"
-        if: matrix.arch == 'x86_64' || matrix.arch == 'x86_64-musl' || matrix.arch == 'x86_32'
+        name: Build and test FTL in ftl-build container (QEMU)
+        uses: docker/[email protected]
+        with:
+          # tags: ghcr.io/pi-hole/ftl-build:v2.0
+          platforms: ${{ matrix.platform }}
+          # Always load latest container image
+          pull: true
+          # Do not push anything
+          push: false
+          context: .
+          target: result
+          file: .github/Dockerfile
+          outputs: |
+            type=tar,dest=build.tar
+          build-args: |
+            "BUILDER=${{ matrix.builder }}"
+            "CI_ARCH=${{ matrix.platform }}"
+            "GIT_BRANCH=${{ needs.smoke-tests.outputs.GIT_BRANCH }}"
+            "GIT_TAG=${{ needs.smoke-tests.outputs.GIT_TAG }}"
+      -
+        name: List files in current directory
+        run: ls -l
+      -
+        name: Extract FTL binary from container
         run: |
-          bash test/run.sh
+          tar -xf build.tar pihole-FTL
       -
         name: "Generate checksum file"
         run: |
           mv pihole-FTL "${{ matrix.bin_name }}"
           sha1sum pihole-FTL-* > ${{ matrix.bin_name }}.sha1
       -
-        name: Upload artifacts to job for later processing
+        name: Store binary artifacts for later deployoment
         if: github.event_name != 'pull_request'
         uses: actions/[email protected]
         with:
-          name: tmp-binary-storage
+          name: tmp-storage
           path: '${{ matrix.bin_name }}*'
+      -
+        name: Extract documentation files from container
+        if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64'
+        run: |
+          tar -xf build.tar api-docs.tar.gz
+      -
+        name: Upload documentation artifacts for deployoment
+        if: github.event_name != 'pull_request' && matrix.platform == 'linux/amd64'
+        uses: actions/[email protected]
+        with:
+          name: tmp-storage
+          path: 'api-docs.tar.gz'
 
   deploy:
     if: github.event_name != 'pull_request'
@@ -135,11 +156,11 @@ jobs:
         name: Checkout code
         uses: actions/[email protected]
       -
-        name: Get Binaries built in previous jobs
+        name: Get Binaries and documentation built in previous jobs
         uses: actions/[email protected]
         id: download
         with:
-          name: tmp-binary-storage
+          name: tmp-storage
           path: ftl-builds/
       -
         name: Display structure of downloaded files
@@ -151,6 +172,16 @@ jobs:
         with:
           private-key: ${{ secrets.SSH_KEY }}
           known-hosts: ${{ secrets.KNOWN_HOSTS }}
+      -
+        name: Untar documentation files
+        working-directory: ${{steps.download.outputs.download-path}}
+        run: |
+          mkdir docs/
+          tar xzvf api-docs.tar.gz -C docs/
+      -
+        name: Display structure of files ready for upload
+        run: ls -R
+        working-directory: ${{steps.download.outputs.download-path}}
       -
         name: Transfer Builds to Pi-hole server for pihole checkout
         if: github.actor != 'dependabot[bot]'

.github/workflows/codespell.yml

@@ -16,4 +16,5 @@ jobs:
       uses: codespell-project/actions-codespell@master
       with:
         ignore_words_file: .github/.codespellignore
-        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs
+        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs,./src/webserver/civetweb,./src/zip/miniz,./src/api/docs/content/external
+        exclude_file: .github/.codespellignore_lines

.github/workflows/merge-conflict.yml

@@ -9,7 +9,7 @@ on:
     types: [synchronize]
 
 jobs:
-  main:
+  merge-conflict:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts

.github/workflows/openapi-validator.yml

@@ -0,0 +1,26 @@
+name: API validation
+
+on: [push]
+
+env:
+  CI: true
+
+jobs:
+  openapi-validator:
+    name: Node
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/[email protected]
+
+      - name: Set Node.js version
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+
+      - name: Install npm dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test

.gitignore

@@ -1,8 +1,9 @@
 # Generated binary
-pihole-FTL
+pihole-FTL*
 
 # Versioning files (generated by Makefile)
-version*
+version.h
+version~
 
 # CMake files generated during compilation
 /cmake/
@@ -12,6 +13,7 @@ version*
 # IDE files
 .idea/
 *.sw*
+/.vscode
 .vscode/
 /.vscode/
 /build/
@@ -20,6 +22,9 @@ version*
 tools/manuf.data
 tools/macvendor.db
 
+# Documentation files generated by cmake
+src/api/docs/hex
+
 # Test dependencies
 /node_modules/
 

CMakeLists.txt

@@ -9,6 +9,7 @@
 # Please see LICENSE file for your rights under this license.
 
 cmake_minimum_required(VERSION 2.8.12)
+
 project(PIHOLE_FTL C)
 
 set(DNSMASQ_VERSION pi-hole-v2.89-9461807)