From 646940c99541e2968e8c4bb052044e2239bf903d Mon Sep 17 00:00:00 2001 From: Public copy <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 21:43:58 +0000 Subject: [PATCH] automated commit Signed-off-by: Public copy <41898282+github-actions[bot]@users.noreply.github.com> --- images/apache-nifi/TESTING.md | 43 ++++++++++++++++++------ images/apache-nifi/config/main.tf | 2 +- images/apache-nifi/tests/check-nifi.sh | 45 ++++++++++++++++---------- reinstated-images.txt | 8 +++-- withdrawn-images.txt | 6 +++- 5 files changed, 73 insertions(+), 31 deletions(-) diff --git a/images/apache-nifi/TESTING.md b/images/apache-nifi/TESTING.md index a9753af593..b441fc7af2 100644 --- a/images/apache-nifi/TESTING.md +++ b/images/apache-nifi/TESTING.md @@ -6,8 +6,8 @@ To begin testing Chainguard's NiFi image, run it via Docker: docker run \ -it --rm \ -p : \ - -e NIFI_WEB_HTTP_HOST="0.0.0.0" \ - -e NIFI_WEB_HTTP_PORT="" \ + -e NIFI_WEB_HTTPS_HOST="0.0.0.0" \ + -e NIFI_WEB_HTTPS_PORT="" \ -e SINGLE_USER_CREDENTIALS_USERNAME="" \ -e SINGLE_USER_CREDENTIALS_PASSWORD="" \ --name \ @@ -22,7 +22,7 @@ NiFi will take a moment to start up. Once it has successfully started, you'll se NiFi has started. The UI is available at the following URLs ``` -At this point, you can access the Web UI at `localhost:`. +At this point, you can access the Web UI at `https://localhost:`. You will be greeted with a blank canvas, called a process group. @@ -30,10 +30,34 @@ You will be greeted with a blank canvas, called a process group. NiFi provides various API endpoints that we can use. For the purposes of this test, we'll create a processor and validate it was created successfully. +Nifi serves over https by default with a self-signed certificate. To make requests to the API, you'll need to include the `--insecure` flag in your curl requests. + +In production environments, you can use a non self-signed certificate or use a reverse proxy. + The first thing we'll do is acquire info about the root process group (the first canvas you see when accessing NiFi's Web UI): +Before you can make requests to the API, you'll need to generate a token. You can do this by sending a POST request to the following endpoint: + +The username and password below can be found in the Docker logs: + +Retrieve the username and password from startup logs: + +```bash +docker logs nifi | grep Generated +``` + +Generate a token: +```bash +ACCESS_TOKEN=$(curl --silent --insecure --request POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "username=${USERNAME}&password=${PASSWD}" \ + "https://localhost:${NIFI_PORT}/nifi-api/access/token") +``` + +Query the root process group: + ```bash -curl -s -X GET "http://localhost:/nifi-api/flow/process-groups/root" +curl --silent --insecure --request GET --header "Authorization: Bearer ${ACCESS_TOKEN}" "https://localhost:/nifi-api/flow/process-groups/root" ``` Take note of the ID. It will look like this: @@ -47,9 +71,10 @@ We'll need this for a creating a processor within the root process group. Now that we have the ID of the root process group, we can create a processor: ```bash -curl -s -X POST "http://localhost:/nifi-api/process-groups//processors" \ - -H "Content-Type: application/json" \ - -d '{ +curl --silent --insecure --request POST --header "Authorization: Bearer ${ACCESS_TOKEN}" \ + "https://localhost:/nifi-api/process-groups//processors" \ + --header "Content-Type: application/json" \ + --data '{ "revision": { "clientId": "test-client", "version": 0 @@ -65,12 +90,12 @@ curl -s -X POST "http://localhost:/nifi-api/process-groups//nifi-api/processors/" +curl --silent --insecure --request GET --header "Authorization: Bearer ${ACCESS_TOKEN}" "http://localhost:/nifi-api/processors/" ``` The processor's component name should be set to `GenerateFlowFile`. -Alternatively, you may access the Web UI over `localhost:`. The processor will be viewable on the canvas. +Alternatively, you may access the Web UI over `https://localhost:`. The processor will be viewable on the canvas. For more info on NiFi's API, see the [upstream API documentation](https://nifi.apache.org/docs/nifi-docs/rest-api/index.html). diff --git a/images/apache-nifi/config/main.tf b/images/apache-nifi/config/main.tf index 4d49f1297c..baa50a2c95 100644 --- a/images/apache-nifi/config/main.tf +++ b/images/apache-nifi/config/main.tf @@ -33,7 +33,7 @@ output "config" { accounts = module.accts.block environment = merge({ "LANG" : "en_US.UTF-8", - "JAVA_HOME" : "/usr/lib/jvm/java-11-openjdk" + "JAVA_HOME" : "/usr/lib/jvm/java-21-openjdk" "LANGUAGE" : "en_US:en" "LC_ALL" : "en_US.UTF-8" "NIFI_BASE_DIR" : "/usr/share/nifi" diff --git a/images/apache-nifi/tests/check-nifi.sh b/images/apache-nifi/tests/check-nifi.sh index 306bf802c3..34b57164b7 100755 --- a/images/apache-nifi/tests/check-nifi.sh +++ b/images/apache-nifi/tests/check-nifi.sh @@ -11,11 +11,12 @@ PASSWD="ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB" # Logs declare -a expected_logs=( - "Starting Apache NiFi" + "org.apache.nifi.runtime.Application Starting NiFi" "NiFi running with PID" - "Launched Apache NiFi with Process ID" - "NiFi has started" - "The UI is available at the following URLs" + "org.apache.nifi.py4j.Controller Listening for requests from Java side using Python Port" + "org.apache.nifi.web.server.JettyServer Started Server on" + "o.a.n.runtime.StandardManagementServer Started Management Server on" + "org.apache.nifi.runtime.Application Started Application" ) declare -a missing_logs=() @@ -23,8 +24,8 @@ declare -a missing_logs=() docker run \ -d --rm \ -p "${NIFI_PORT}":"${NIFI_PORT}" \ - -e NIFI_WEB_HTTP_HOST="0.0.0.0" \ - -e NIFI_WEB_HTTP_PORT="${NIFI_PORT}" \ + -e NIFI_WEB_HTTPS_HOST="0.0.0.0" \ + -e NIFI_WEB_HTTPS_PORT="${NIFI_PORT}" \ -e SINGLE_USER_CREDENTIALS_USERNAME="${USERNAME}" \ -e SINGLE_USER_CREDENTIALS_PASSWORD="${PASSWD}" \ --name "${CONTAINER_NAME}" \ @@ -72,10 +73,10 @@ TEST_validate_container_logs() { TEST_http_response() { # Retries local request_retries=15 - local retry_delay=5 + local retry_delay=10 for ((i=1; i<=${request_retries}; i++)); do - if [[ $(curl -sLo /dev/null -w "%{http_code}" "http://localhost:${NIFI_PORT}/nifi") -eq 200 ]]; then + if [[ $(curl --insecure --silent --location --output /dev/null -w "%{http_code}" "https://localhost:${NIFI_PORT}/nifi") -eq 200 ]]; then return 0 fi sleep ${retry_delay} @@ -87,22 +88,30 @@ TEST_http_response() { # Tests API by creating a processor in the root process group TEST_create_processor() { - apk add jq + # apk add jq + # Get API access token + local access_token=$(curl --silent --insecure --request POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "username=${USERNAME}&password=${PASSWD}" \ + "https://localhost:${NIFI_PORT}/nifi-api/access/token") # Fetch root process group - local pg=$(curl -s -X GET "http://localhost:${NIFI_PORT}/nifi-api/flow/process-groups/root") + local pg=$(curl --insecure --silent --request GET --header "Authorization: Bearer ${access_token}" \ + "https://localhost:${NIFI_PORT}/nifi-api/flow/process-groups/root") printf "Root process group:\n${pg}" # Extract root process group ID - local pg_id=$(echo "${pg}" | jq -r '.processGroupFlow.id') + local pg_id=$(echo "${pg}" | jq --raw-output '.processGroupFlow.id') echo "Root process group ID: ${pg_id}" # Create a processor local create_processor=$(\ - curl -s -w "%{http_code}" \ - -X POST "http://localhost:${NIFI_PORT}/nifi-api/process-groups/${pg_id}/processors" \ - -H "Content-Type: application/json" \ - -d '{ + curl --insecure --silent --write-out "%{http_code}" \ + --request POST \ + --header "Authorization: Bearer ${access_token}" \ + "https://localhost:${NIFI_PORT}/nifi-api/process-groups/${pg_id}/processors" \ + --header "Content-Type: application/json" \ + --data '{ "revision": { "clientId": "test-client", "version": 0 @@ -125,15 +134,17 @@ TEST_create_processor() { # Fetch processor ID local processor=$(echo "${create_processor}" | sed "s/${http_code}//") - local processor_id=$(echo "${processor}" | jq -r '.component.id') + local processor_id=$(echo "${processor}" | jq --raw-output '.component.id') echo "Processor ID: ${processor_id}" # Fetch processor info - local processor_info=$(curl -s -X GET "http://localhost:${NIFI_PORT}/nifi-api/processors/${processor_id}") + local processor_info=$(curl --insecure --silent --request GET --header "Authorization: Bearer ${access_token}" "https://localhost:${NIFI_PORT}/nifi-api/processors/${processor_id}") printf "Processor info:\n${processor_info}" } # Run tests TEST_http_response +# Sleep 5 seconds to ensure that all services start. In my testing this is more than enough time. +sleep 5 TEST_validate_container_logs TEST_create_processor diff --git a/reinstated-images.txt b/reinstated-images.txt index 1b3eb4d5e7..7b8b2f168b 100644 --- a/reinstated-images.txt +++ b/reinstated-images.txt @@ -1,3 +1,5 @@ -# These 2 are just for testing -cgr.dev/chainguard/curl:test1@sha256:8bf944b98bdef5ab6b4ef4bd6992d6a5ad5c008b51c119a78ffc385273a36eec -cgr.dev/chainguard/curl:test1-dev@sha256:98e4086d13835d7355e6c833472deed831e118becfd65fb806f951f2900efa4c +# Each line should be in the form $repo:$tag@$digest +# Note: If tag already exists, it will NOT be retagged and must be first deleted by adding to withdrawn-images.txt +# Example - add "test1" tag to curl repo pointing to sha257:8bf944...: +# cgr.dev/chainguard/curl:test1@sha256:8bf944b98bdef5ab6b4ef4bd6992d6a5ad5c008b51c119a78ffc385273a36eec +# diff --git a/withdrawn-images.txt b/withdrawn-images.txt index b0bb4720b2..de5a31a0e2 100644 --- a/withdrawn-images.txt +++ b/withdrawn-images.txt @@ -1185,4 +1185,8 @@ cgr.dev/chainguard/sdk:wolfi-20230104 cgr.dev/chainguard/jitsucom-bulker-bulker cgr.dev/chainguard/jitsucom-bulker-ingest cgr.dev/chainguard/jitsucom-bulker-syncctl -cgr.dev/chainguard/kaniko:test \ No newline at end of file +cgr.dev/chainguard/kaniko:test + +# Added for CI testing purposes +cgr.dev/chainguard/curl:test1 +cgr.dev/chainguard/curl:test1-dev