Merge pull request #26 from Probely/diagnostics

Add more diagnostic information

Dockerfile

@@ -54,6 +54,8 @@ COPY --from=go_builder /build/farconn/farconn /usr/local/bin
 COPY --from=go_builder /build/farcaster-go/bin/farcasterd /usr/local/bin
 COPY --from=rust_builder /moproxy/target/release/moproxy /usr/local/bin
 COPY --from=rust_builder /udp-over-tcp/target/release/udp2tcp /usr/local/bin
+ARG VERSION
+ENV FARCASTER_VERSION=${VERSION}
 RUN set -eux \
     && umask 077 \
     && apt-get update -y \

README.md

@@ -60,7 +60,7 @@ It is used, for example, to detect "Log4Shell"-type vulnerabilities.
 
 The agent runs on a Docker container. It should work on any system with a Docker installation.
 
-The agent needs an API token to connect to Probely's network.
+The agent needs a token to connect to Probely's network.
 
 > If you do not have an agent token, you can create one in the
 > [Scanning Agents](https://plus.probely.app/scanning-agents/) management area.

scripts/_lib.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 PROXY_REGEXP='^(http(s)?://)?(([0-9a-zA-Z_-]+:[0-9a-zA-Z_-]+)@)?([0-9a-zA-Z._-]+)(:([0-9]+))?$'
 WORKDIR=/run/farcaster
+LOGDIR=/logs
 HUB_IP_TTL=300
 WG_DEFAULT_PORT=51820
 INTERNAL_NETS="${INTERNAL_NETS:-10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16}"
@@ -229,6 +230,8 @@ start_udp_over_tcp_tunnel() {
 }
 
 get_first_nameserver() {
+	echo "1.1.1.1"
+	return 0
     ns=$(grep -m 1 '^nameserver' /etc/resolv.conf | awk '{print $2}')
 	if [ -z "${ns}" ]; then
 		ns="127.0.0.1"
@@ -331,17 +334,25 @@ start_proxy_maybe() {
 		echo "Could not set the proxy redirect rules" >&2
 		return 1
 	fi
+
+	log_level="info"
+	log_file="/dev/null"
+	if [ "$(debug_level)" -gt 0 ] && [ -d "${LOGDIR}" ]; then
+		log_level="trace"
+		log_file="${LOGDIR}/moproxy/moproxy.log"
+	fi
+
 	setpriv --reuid=proxy --regid=proxy --clear-groups --no-new-privs \
-		nohup /usr/local/bin/moproxy --host 0.0.0.0 --port "${listen_port}" --list "${config_path}" --allow-direct >/dev/null &
+		nohup /usr/local/bin/moproxy --log-level "${log_level}" --host 0.0.0.0 --port "${listen_port}" \
+		--list "${config_path}" --allow-direct >"${log_file}" &
 	sleep 3
 	kill -0 $!
 	return $?
 }
 
 start_userspace_agent() {
-	debug=$1
 	extra_args=""
-	if [ "$debug" -eq 1 ]; then
+	if [ "$(debug_level)" -gt 1 ]; then
 		extra_args="-d"
 	fi
 	CMD="/usr/local/bin/farcasterd ${extra_args}"
@@ -376,3 +387,25 @@ function print_log() {
 	echo
 	sleep 120
 }
+
+function print_diagnostics() {
+	echo
+	echo
+	echo "-----addresses-----"
+	ip addr show
+	echo
+	echo "-----routes-----"
+	ip route show
+	echo
+	echo "-----iptables-----"
+	${IPT_CMD} -t filter -n -L -v
+	${IPT_CMD} -t nat -n -L -v
+	echo
+	echo "-----moproxy config-----"
+	cat /run/moproxy/config.ini || echo "No moproxy config found"
+	echo
+}
+
+function debug_level() {
+	echo "${FARCASTER_DEBUG_LEVEL:-0}"
+}

scripts/entrypoint.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+echo "Starting Farcaster agent v${FARCASTER_VERSION}..."
+
 set -eu
 
 umask 007

scripts/run-hybrid.sh

@@ -4,7 +4,8 @@ set -eu
 
 . "${FARCASTER_PATH}"/bin/_lib.sh
 
-if [ "${FARCASTER_DEBUG:-0}" -ne 0 ]; then
+if [ "$(debug_level)" -gt 0 ]; then
+	echo "Debugging enabled"
 	set -x
 fi
 
@@ -32,10 +33,12 @@ echo "done"
 
 echo -ne "Starting Farcaster Agent\t...\n"
 
-set +x
+if [ "$(debug_level)" -gt 0 ]; then
+	print_diagnostics
+fi
 
 # Finally, start the userspace agent
-if ! start_userspace_agent "${FARCASTER_DEBUG:-0}"; then
+if ! start_userspace_agent; then
 	echo "Could not start the userspace agent!"
 	sleep 10
 	exit $?

tests/agent/docker-compose-diagnostics.yml

@@ -0,0 +1,54 @@
+services:
+  farcaster:
+    image: probely/farcaster-onprem-agent:v3
+    depends_on:
+      tcpdump:
+        condition: service_healthy
+    network_mode: service:tcpdump
+    environment:
+      - HTTP_PROXY
+      - HTTPS_PROXY
+      - RUN_MODE
+      - FARCASTER_DEBUG_LEVEL
+      - FARCASTER_AGENT_TOKEN
+      - FARCASTER_API_URL
+    cap_add:
+    - NET_ADMIN
+    volumes:
+    - shared-logs:/logs
+    restart: "no"
+    stop_grace_period: 1s
+
+  proxy:
+    image: ubuntu:noble
+    volumes:
+      - shared-logs:/logs
+    command:
+    - /bin/bash
+    - -c
+    - rm -rf /logs/moproxy
+      && mkdir /logs/moproxy
+      && chmod 1777 /logs/moproxy
+      && while true; do
+          test -f /logs/moproxy/moproxy.log && tail -F /logs/moproxy/moproxy.log || sleep 1;
+        done
+    restart: always
+    stop_grace_period: 1s
+
+  tcpdump:
+    image: alpine
+    healthcheck:
+      test: ["CMD", "pidof", "tcpdump"]
+      interval: 10s
+      timeout: 1s
+      retries: 3
+    command:
+    - /bin/ash
+    - -c
+    - apk add --no-cache tcpdump > /dev/null 2>&1
+      && tcpdump -s0 -n -i eth0
+    restart: always
+    stop_grace_period: 1s
+
+volumes:
+  shared-logs:

tests/agent/docker-compose.yml

@@ -1,5 +1,5 @@
 services:
-  main:
+  farcaster:
     build:
       dockerfile: Dockerfile
       context: ../../
@@ -10,22 +10,20 @@ services:
         condition: service_healthy
       network:
         condition: service_healthy
-    network_mode: service:network
+      tcpdump:
+        condition: service_healthy
+    network_mode: service:tcpdump
     environment:
       - HTTP_PROXY
       - HTTPS_PROXY
       - RUN_MODE
+      - FARCASTER_DEBUG_LEVEL
       - FARCASTER_AGENT_TOKEN
-      - FARCASTER_API_URL=${FARCASTER_API_URL:-https://api.stg.eu.probely.com}
-    cap_drop:
-    - ALL
+      - FARCASTER_API_URL
     cap_add:
-    - DAC_OVERRIDE
-    - NET_RAW
-    - SETUID
-    - SETGID
-    - KILL
     - ${NET_ADMIN}
+    volumes:
+    - shared-logs:/logs
     restart: "no"
     stop_grace_period: 1s
 
@@ -54,8 +52,12 @@ services:
       test: ["CMD", "test", "-f", "/tmp/healthy"]
       interval: 1m30s
       timeout: 1s
-      retries: 3 
+      retries: 3
       start_period: 10s
+    depends_on:
+      tcpdump:
+        condition: service_healthy
+    network_mode: service:tcpdump
     cap_add:
       - NET_ADMIN
     restart: "no"
@@ -72,3 +74,36 @@ services:
     restart: always
     stop_grace_period: 1s
 
+  tcp_proxy:
+    image: ubuntu:noble
+    volumes:
+      - shared-logs:/logs
+    command:
+    - /bin/bash
+    - -c
+    - rm -rf /logs/moproxy
+      && mkdir /logs/moproxy
+      && chmod 1777 /logs/moproxy
+      && while true; do
+          test -f /logs/moproxy/moproxy.log && tail -F /logs/moproxy/moproxy.log || sleep 1;
+        done
+    restart: always
+    stop_grace_period: 1s
+
+  tcpdump:
+    image: alpine
+    healthcheck:
+      test: ["CMD", "pidof", "tcpdump"]
+      interval: 10s
+      timeout: 1s
+      retries: 3
+    command:
+    - /bin/ash
+    - -c
+    - apk add --no-cache tcpdump > /dev/null 2>&1
+      && tcpdump -s0 -n -i eth0
+    restart: always
+    stop_grace_period: 1s
+
+volumes:
+  shared-logs: