From 36625a5dfd4a1e96e78bbddd6184cfb180d907a2 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 11 Apr 2024 07:09:21 +0100 Subject: [PATCH] fix: split RPC API and HTTP Gateway servers (#139) To better conform to Kubo's pattern, run the RPC endpoints on port 5001 and the HTTP Gateway on port 8080. Also simplifies the Docker file, there's no need to rebuild OpenSSL, the version that comes with `node:20-slim` is fine. --- .gitignore | 3 + Dockerfile | 46 +--- README.md | 12 +- debugging/README.md | 4 +- debugging/test-gateways.sh | 67 ++--- debugging/time-permutations.sh | 7 +- debugging/until-death.sh | 17 +- e2e-tests/gc.spec.ts | 4 +- e2e-tests/smoketest.spec.ts | 4 +- e2e-tests/version-response.spec.ts | 6 +- package-lock.json | 193 ++++++++------- playwright.config.ts | 4 +- src/constants.ts | 20 +- src/healthcheck.ts | 4 +- src/helia-http-gateway.ts | 170 +++++++++++++ src/helia-rpc-api.ts | 106 ++++++++ src/helia-server.ts | 376 ++++------------------------- src/index.ts | 200 ++++++++------- 18 files changed, 627 insertions(+), 616 deletions(-) create mode 100644 src/helia-http-gateway.ts create mode 100644 src/helia-rpc-api.ts diff --git a/.gitignore b/.gitignore index d58ec7b..7463338 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,9 @@ debugging/test-gateways.log permutation-logs *.csv *.log +blockstore +datastore +*.heapsnapshot # grafana/prometheus files config/grafana/alerting diff --git a/Dockerfile b/Dockerfile index f0c7dc6..f5e7cd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,9 @@ -# OpenSSL Build Stage -FROM --platform=$BUILDPLATFORM node:20-slim as openssl-builder - -RUN apt-get update && \ - apt-get install -y build-essential wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV OPEN_SSL_VERSION=1.1.1w - -# Download OpenSSL -RUN wget -P /tmp https://www.openssl.org/source/old/1.1.1/openssl-${OPEN_SSL_VERSION}.tar.gz - -# Extract OpenSSL and configure -RUN mkdir -p /opt/openssl && \ - tar -xzf /tmp/openssl-${OPEN_SSL_VERSION}.tar.gz -C /opt/openssl - -# Build and install OpenSSL -WORKDIR /opt/openssl/openssl-${OPEN_SSL_VERSION} - -# Configure OpenSSL -RUN ./config --prefix=/opt/openssl --openssldir=/opt/openssl/ssl - -# Build OpenSSL -RUN make - -# Test the build -RUN make test - -# Install OpenSSL -RUN make install - -# Cleanup unnecessary files to reduce image size -RUN cd /opt/openssl && \ - rm -rf /opt/openssl/openssl-${OPEN_SSL_VERSION} /tmp/openssl-${OPEN_SSL_VERSION}.tar.gz - # Application Build Stage FROM --platform=$BUILDPLATFORM node:20-slim as builder # Install dependencies required for building the app RUN apt-get update && \ - apt-get install -y tini && \ + apt-get install -y build-essential wget tini && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -64,12 +28,10 @@ COPY --from=builder /app ./ # copy tini from the builder stage COPY --from=builder /usr/bin/tini /usr/bin/tini -# copy OpenSSL libraries from the openssl-builder stage -COPY --from=openssl-builder /usr/lib/**/libcrypto* /usr/lib/ -COPY --from=openssl-builder /usr/lib/**/libssl* /usr/lib/ -COPY --from=openssl-builder /opt/openssl/lib /opt/openssl/lib -ENV LD_LIBRARY_PATH /opt/openssl/lib +# port for RPC API +EXPOSE 5001 +# port for HTTP Gateway EXPOSE 8080 HEALTHCHECK --interval=12s --timeout=12s --start-period=10s CMD node dist/src/healthcheck.js diff --git a/README.md b/README.md index 35ade48..a98a92d 100644 --- a/README.md +++ b/README.md @@ -114,25 +114,25 @@ Note that any of the following calls to docker can be replaced with something li #### Disable libp2p ```sh -$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia +$ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia ``` #### Disable bitswap ```sh -$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia +$ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia ``` #### Disable trustless gateways ```sh -$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia +$ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia ``` #### Customize trustless gateways ```sh -$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia +$ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia ``` diff --git a/debugging/README.md b/debugging/README.md index 272cd63..6f99c4d 100644 --- a/debugging/README.md +++ b/debugging/README.md @@ -35,9 +35,9 @@ USE_SUBDOMAINS=true USE_REDIRECTS=false docker build . --platform linux/$(arch) Then we need to start the container ```sh -docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:local-$(arch) +docker run -it -p 5001:5001 -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:local-$(arch) # or -docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" -e USE_REDIRECTS="false" -e USE_SUBDOMAINS="true" helia-http-gateway:local-$(arch) +docker run -it -p 5001:5001 -p 8080:8080 -e DEBUG="helia-http-gateway*" -e USE_REDIRECTS="false" -e USE_SUBDOMAINS="true" helia-http-gateway:local-$(arch) ``` ## Running tests against the container diff --git a/debugging/test-gateways.sh b/debugging/test-gateways.sh index 50a7ca0..04fe995 100755 --- a/debugging/test-gateways.sh +++ b/debugging/test-gateways.sh @@ -18,15 +18,16 @@ trap cleanup_gateway_test EXIT # Query all endpoints until failure # This script is intended to be run from the root of the helia-http-gateway repository -PORT=${PORT:-8080} -# If localhost:$PORT is not listening, then exit with non-zero error code -if ! nc -z localhost $PORT; then - echo "localhost:$PORT is not listening" +HTTP_PORT=${HTTP_PORT:-8080} +RPC_PORT=${RPC_PORT:-5001} +# If localhost:$HTTP_PORT is not listening, then exit with non-zero error code +if ! nc -z localhost $HTTP_PORT; then + echo "localhost:$HTTP_PORT is not listening" exit 1 fi ensure_gateway_running() { - npx wait-on "tcp:$PORT" -t 1000 || { + npx wait-on "tcp:$HTTP_PORT" -t 1000 || { EXIT_CODE=1 cleanup_gateway_test } @@ -40,58 +41,58 @@ test_website() { echo "Requesting $website" curl -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds (TTFB: %{time_starttransfer}, redirect: %{time_redirect})\n" -L $website echo "running GC" - curl -X POST -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds\n" http://localhost:$PORT/api/v0/repo/gc + curl -X POST -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds\n" http://localhost:$RPC_PORT/api/v0/repo/gc } -test_website http://localhost:$PORT/ipns/blog.ipfs.tech +test_website http://localhost:$HTTP_PORT/ipns/blog.ipfs.tech -test_website http://localhost:$PORT/ipns/blog.libp2p.io +test_website http://localhost:$HTTP_PORT/ipns/blog.libp2p.io -test_website http://localhost:$PORT/ipns/consensuslab.world +test_website http://localhost:$HTTP_PORT/ipns/consensuslab.world -test_website http://localhost:$PORT/ipns/docs.ipfs.tech +test_website http://localhost:$HTTP_PORT/ipns/docs.ipfs.tech -test_website http://localhost:$PORT/ipns/docs.libp2p.io +test_website http://localhost:$HTTP_PORT/ipns/docs.libp2p.io -# test_website http://localhost:$PORT/ipns/drand.love #drand.love is not publishing dnslink records +# test_website http://localhost:$HTTP_PORT/ipns/drand.love #drand.love is not publishing dnslink records -test_website http://localhost:$PORT/ipns/fil.org +test_website http://localhost:$HTTP_PORT/ipns/fil.org -test_website http://localhost:$PORT/ipns/filecoin.io +test_website http://localhost:$HTTP_PORT/ipns/filecoin.io -test_website http://localhost:$PORT/ipns/green.filecoin.io +test_website http://localhost:$HTTP_PORT/ipns/green.filecoin.io -test_website http://localhost:$PORT/ipns/ipfs.tech +test_website http://localhost:$HTTP_PORT/ipns/ipfs.tech -test_website http://localhost:$PORT/ipns/ipld.io +test_website http://localhost:$HTTP_PORT/ipns/ipld.io -test_website http://localhost:$PORT/ipns/libp2p.io +test_website http://localhost:$HTTP_PORT/ipns/libp2p.io -test_website http://localhost:$PORT/ipns/n0.computer +test_website http://localhost:$HTTP_PORT/ipns/n0.computer -test_website http://localhost:$PORT/ipns/probelab.io +test_website http://localhost:$HTTP_PORT/ipns/probelab.io -test_website http://localhost:$PORT/ipns/protocol.ai +test_website http://localhost:$HTTP_PORT/ipns/protocol.ai -test_website http://localhost:$PORT/ipns/research.protocol.ai +test_website http://localhost:$HTTP_PORT/ipns/research.protocol.ai -test_website http://localhost:$PORT/ipns/singularity.storage +test_website http://localhost:$HTTP_PORT/ipns/singularity.storage -test_website http://localhost:$PORT/ipns/specs.ipfs.tech +test_website http://localhost:$HTTP_PORT/ipns/specs.ipfs.tech -# test_website http://localhost:$PORT/ipns/strn.network -test_website http://localhost:$PORT/ipns/saturn.tech +# test_website http://localhost:$HTTP_PORT/ipns/strn.network +test_website http://localhost:$HTTP_PORT/ipns/saturn.tech -test_website http://localhost:$PORT/ipns/web3.storage +test_website http://localhost:$HTTP_PORT/ipns/web3.storage -test_website http://localhost:$PORT/ipfs/bafkreiezuss4xkt5gu256vjccx7vocoksxk77vwmdrpwoumfbbxcy2zowq # stock images 3 sec skateboarder video +test_website http://localhost:$HTTP_PORT/ipfs/bafkreiezuss4xkt5gu256vjccx7vocoksxk77vwmdrpwoumfbbxcy2zowq # stock images 3 sec skateboarder video -test_website http://localhost:$PORT/ipfs/bafybeidsp6fva53dexzjycntiucts57ftecajcn5omzfgjx57pqfy3kwbq # big buck bunny +test_website http://localhost:$HTTP_PORT/ipfs/bafybeidsp6fva53dexzjycntiucts57ftecajcn5omzfgjx57pqfy3kwbq # big buck bunny -test_website http://localhost:$PORT/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze # wikipedia +test_website http://localhost:$HTTP_PORT/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze # wikipedia -test_website http://localhost:$PORT/ipfs/bafybeifaiclxh6pc3bdtrrkpbvvqqxq6hz5r6htdzxaga4fikfpu2u56qi # uniswap interface +test_website http://localhost:$HTTP_PORT/ipfs/bafybeifaiclxh6pc3bdtrrkpbvvqqxq6hz5r6htdzxaga4fikfpu2u56qi # uniswap interface -test_website http://localhost:$PORT/ipfs/bafybeiae366charqmeewxags5b2jxtkhfmqyyagvqhrr5l7l7xfpp5ikpa # cid.ipfs.tech +test_website http://localhost:$HTTP_PORT/ipfs/bafybeiae366charqmeewxags5b2jxtkhfmqyyagvqhrr5l7l7xfpp5ikpa # cid.ipfs.tech -test_website http://localhost:$PORT/ipfs/bafybeiedlhslivmuj2iinnpd24ulx3fyd7cjenddbkeoxbf3snjiz3npda # docs.ipfs.tech +test_website http://localhost:$HTTP_PORT/ipfs/bafybeiedlhslivmuj2iinnpd24ulx3fyd7cjenddbkeoxbf3snjiz3npda # docs.ipfs.tech diff --git a/debugging/time-permutations.sh b/debugging/time-permutations.sh index e7ef141..b1e9f85 100755 --- a/debugging/time-permutations.sh +++ b/debugging/time-permutations.sh @@ -15,7 +15,8 @@ # export DEBUG="helia*,helia*:trace,libp2p*,libp2p*:trace" export DEBUG="*,*:trace" unset FASTIFY_DEBUG -export PORT=8080 +export HTTP_PORT=8080 +export RPC_PORT=5001 export HOST="0.0.0.0" export ECHO_HEADERS=false export METRICS=true @@ -47,7 +48,7 @@ echo "USE_SUBDOMAINS,USE_BITSWAP,USE_TRUSTLESS_GATEWAYS,USE_LIBP2P,USE_DELEGATED run_test() { - npx wait-on "tcp:$PORT" -t 10000 -r # wait for the port to be released + npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released config_id="USE_SUBDOMAINS=$USE_SUBDOMAINS,USE_BITSWAP=$USE_BITSWAP,USE_TRUSTLESS_GATEWAYS=$USE_TRUSTLESS_GATEWAYS,USE_LIBP2P=$USE_LIBP2P,USE_DELEGATED_ROUTING=$USE_DELEGATED_ROUTING" # if we cannot get any data, we should skip this run.. we need at least USE_BITSWAP enabled, plus either USE_LIBP2P or USE_DELEGATED_ROUTING @@ -133,7 +134,7 @@ cleanup_permutations() { echo "sent TERM signal to subshell" wait $subshell_pid # wait for the process to exit - npx wait-on "tcp:$PORT" -t 10000 -r # wait for the port to be released + npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released exit 1 } diff --git a/debugging/until-death.sh b/debugging/until-death.sh index fce1d56..ac5893c 100755 --- a/debugging/until-death.sh +++ b/debugging/until-death.sh @@ -8,7 +8,8 @@ fi # You have to pass `DEBUG=" " to disable debugging when using this script` export DEBUG=${DEBUG:-"helia-http-gateway,helia-http-gateway:server,helia-http-gateway:*:helia-fetch"} -export PORT=${PORT:-8080} +export HTTP_PORT=${HTTP_PORT:-8080} +export RPC_PORT=${RPC_PORT:-5001} EXIT_CODE=0 cleanup_until_death_called=false @@ -20,13 +21,12 @@ cleanup_until_death() { echo "cleanup_until_death called" cleanup_until_death_called=true if [ "$gateway_already_running" != true ]; then - lsof -i TCP:$PORT | grep LISTEN | awk '{print $2}' | xargs --no-run-if-empty kill -9 + lsof -i TCP:$HTTP_PORT | grep LISTEN | awk '{print $2}' | xargs --no-run-if-empty kill -9 echo "waiting for the gateway to exit" - npx wait-on "tcp:$PORT" -t 10000 -r # wait for the port to be released + npx wait-on "tcp:$HTTP_PORT" -t 10000 -r # wait for the port to be released fi - exit $EXIT_CODE } @@ -35,7 +35,8 @@ trap cleanup_until_death EXIT # Before starting, output all env vars that helia-http-gateway uses echo "DEBUG=$DEBUG" echo "FASTIFY_DEBUG=$FASTIFY_DEBUG" -echo "PORT=$PORT" +echo "HTTP_PORT=$HTTP_PORT" +echo "RPC_PORT=$RPC_PORT" echo "HOST=$HOST" echo "USE_SUBDOMAINS=$USE_SUBDOMAINS" echo "METRICS=$METRICS" @@ -51,7 +52,7 @@ echo "FILE_BLOCKSTORE_PATH=$FILE_BLOCKSTORE_PATH" echo "ALLOW_UNHANDLED_ERROR_RECOVERY=$ALLOW_UNHANDLED_ERROR_RECOVERY" gateway_already_running=false -if nc -z localhost $PORT; then +if nc -z localhost $HTTP_PORT; then echo "gateway is already running" gateway_already_running=true fi @@ -70,7 +71,7 @@ start_gateway() { (node --trace-warnings dist/src/index.js) & process_id=$! # echo "process id: $!" - npx wait-on "tcp:$PORT" -t 10000 || { + npx wait-on "tcp:$HTTP_PORT" -t 10000 || { EXIT_CODE=1 cleanup_until_death } @@ -78,7 +79,7 @@ start_gateway() { start_gateway ensure_gateway_running() { - npx wait-on "tcp:$PORT" -t 5000 || { + npx wait-on "tcp:$HTTP_PORT" -t 5000 || { EXIT_CODE=1 cleanup_until_death } diff --git a/e2e-tests/gc.spec.ts b/e2e-tests/gc.spec.ts index a9424b3..a7c01fe 100644 --- a/e2e-tests/gc.spec.ts +++ b/e2e-tests/gc.spec.ts @@ -1,8 +1,8 @@ import { test, expect } from '@playwright/test' -import { PORT } from '../src/constants.js' +import { RPC_PORT } from '../src/constants.js' test('POST /api/v0/repo/gc', async ({ page }) => { - const result = await page.request.post(`http://localhost:${PORT}/api/v0/repo/gc`) + const result = await page.request.post(`http://localhost:${RPC_PORT}/api/v0/repo/gc`) expect(result?.status()).toBe(200) const maybeContent = await result?.text() diff --git a/e2e-tests/smoketest.spec.ts b/e2e-tests/smoketest.spec.ts index 24617a5..4ffe227 100644 --- a/e2e-tests/smoketest.spec.ts +++ b/e2e-tests/smoketest.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test' -import { PORT } from '../src/constants.js' +import { HTTP_PORT } from '../src/constants.js' // test all the same pages listed at https://probelab.io/websites/ const pages = [ @@ -36,7 +36,7 @@ test.beforeEach(async ({ context }) => { }) pages.forEach((pagePath) => { - const url = `http://${pagePath}.ipns.localhost:${PORT}` + const url = `http://${pagePath}.ipns.localhost:${HTTP_PORT}` test(`helia-http-gateway can load path '${url}'`, async ({ page }) => { // only wait for 'commit' because we don't want to wait for all the assets to load, we just want to make sure that they *would* load (e.g. the html is valid) const heliaGatewayResponse = await page.goto(`${url}`, { waitUntil: 'commit' }) diff --git a/e2e-tests/version-response.spec.ts b/e2e-tests/version-response.spec.ts index d1356ca..f37d9d4 100644 --- a/e2e-tests/version-response.spec.ts +++ b/e2e-tests/version-response.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test' -import { PORT } from '../src/constants.js' +import { RPC_PORT } from '../src/constants.js' function validateResponse (content: string): void { expect(() => JSON.parse(content)).not.toThrow() @@ -10,7 +10,7 @@ function validateResponse (content: string): void { } test('GET /api/v0/version', async ({ page }) => { - const result = await page.goto(`http://localhost:${PORT}/api/v0/version`) + const result = await page.goto(`http://localhost:${RPC_PORT}/api/v0/version`) expect(result?.status()).toBe(200) const maybeContent = await result?.text() @@ -19,7 +19,7 @@ test('GET /api/v0/version', async ({ page }) => { }) test('POST /api/v0/version', async ({ page }) => { - const result = await page.request.post(`http://localhost:${PORT}/api/v0/version`) + const result = await page.request.post(`http://localhost:${RPC_PORT}/api/v0/version`) expect(result?.status()).toBe(200) const maybeContent = await result?.text() diff --git a/package-lock.json b/package-lock.json index 367bfb0..fe7d112 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3524,25 +3524,25 @@ } }, "node_modules/@helia/verified-fetch": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/@helia/verified-fetch/-/verified-fetch-1.3.7.tgz", - "integrity": "sha512-2mEM+X/dWeV3kq+2caGUxSS4gOAU5au3jCqwdQ8zQaSTdyZPzIZpJgTmkzaSAcnmJ8Ee51yFH/yxwAyf7BMNdQ==", - "dependencies": { - "@helia/block-brokers": "^2.0.2", - "@helia/car": "^3.1.0", - "@helia/http": "^1.0.2", - "@helia/interface": "^4.0.1", - "@helia/ipns": "^7.0.0", - "@helia/routers": "^1.0.1", - "@helia/unixfs": "^3.0.1", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/@helia/verified-fetch/-/verified-fetch-1.3.8.tgz", + "integrity": "sha512-rSo6meoOOtKotLgQ0nVofGFLQwhpcDwwGMTlvaSH4+0fWWmZ+xmqIddePL7BW8kDFTDlPdntrkOUB3xN+I3e4g==", + "dependencies": { + "@helia/block-brokers": "^2.0.3", + "@helia/car": "^3.1.2", + "@helia/http": "^1.0.3", + "@helia/interface": "^4.1.0", + "@helia/ipns": "^7.2.0", + "@helia/routers": "^1.0.2", + "@helia/unixfs": "^3.0.3", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", - "@libp2p/interface": "^1.1.4", - "@libp2p/kad-dht": "^12.0.8", - "@libp2p/peer-id": "^4.0.7", - "@multiformats/dns": "^1.0.2", - "cborg": "^4.0.9", + "@libp2p/interface": "^1.1.6", + "@libp2p/kad-dht": "^12.0.11", + "@libp2p/peer-id": "^4.0.9", + "@multiformats/dns": "^1.0.6", + "cborg": "^4.2.0", "hashlru": "^2.3.0", "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", @@ -3553,7 +3553,7 @@ "it-to-browser-readablestream": "^2.0.6", "multiformats": "^13.1.0", "progress-events": "^1.0.0", - "uint8arrays": "^5.0.2" + "uint8arrays": "^5.0.3" } }, "node_modules/@humanwhocodes/config-array": { @@ -4337,16 +4337,16 @@ } }, "node_modules/@libp2p/tcp": { - "version": "9.0.18", - "resolved": "https://registry.npmjs.org/@libp2p/tcp/-/tcp-9.0.18.tgz", - "integrity": "sha512-1Z3vKO1nSMo1sJOnLuHoS/yEXa9bdyT6/Eh/Q4lho1akNWds3PB06iwYHMhlYbh4XjuJVzAorvvHuVpzfb+DLA==", + "version": "9.0.19", + "resolved": "https://registry.npmjs.org/@libp2p/tcp/-/tcp-9.0.19.tgz", + "integrity": "sha512-/+YHJtFyqQAz41V+q9RuKlQK7KoHOmc/k+BnGJO6lR7LWosl68sXUAWksZaeluKZu3Q0LyqkIFzN34wCKX7tZA==", "dependencies": { "@libp2p/interface": "^1.1.6", "@libp2p/utils": "^5.2.8", "@multiformats/mafmt": "^12.1.6", "@multiformats/multiaddr": "^12.2.1", "@types/sinon": "^17.0.3", - "stream-to-it": "^0.2.4" + "stream-to-it": "^1.0.0" } }, "node_modules/@libp2p/tls": { @@ -4405,9 +4405,9 @@ } }, "node_modules/@libp2p/webrtc": { - "version": "4.0.24", - "resolved": "https://registry.npmjs.org/@libp2p/webrtc/-/webrtc-4.0.24.tgz", - "integrity": "sha512-HAotyz+nt9rJ0P92+S1xQa6bDRb7g2eBph6SP4r51JLZG9j9OjvN25waKXhyhhQ7Lq6NiaWE1rNpEaAR0oW3Hg==", + "version": "4.0.25", + "resolved": "https://registry.npmjs.org/@libp2p/webrtc/-/webrtc-4.0.25.tgz", + "integrity": "sha512-1DSOReguR5VXX0c7cThNQ/Djw6MMV5G42RdsR4Or+TCOojhh0KMG1N7iAO7kzrAvQ3VgVkMcJ8BHxF2/x1k+ig==", "dependencies": { "@chainsafe/libp2p-noise": "^15.0.0", "@libp2p/interface": "^1.1.6", @@ -4453,9 +4453,9 @@ } }, "node_modules/@libp2p/webtransport": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@libp2p/webtransport/-/webtransport-4.0.23.tgz", - "integrity": "sha512-VkEI0jMPoN/ookywDyrBAul9lb9E/ztbWu+9xQNlrvHgBEaCFplA8dXsWH9tMxFufQ8Zgw/CSWagmLKSsAO+3A==", + "version": "4.0.24", + "resolved": "https://registry.npmjs.org/@libp2p/webtransport/-/webtransport-4.0.24.tgz", + "integrity": "sha512-78bXy3lJ14vAwde0mlZhNBCuBsqoEKPv15eoZFMQPn4dKV1jD0n9vMhcML58AfM4tx8EPms0+g5gPJCI1W/QEQ==", "dependencies": { "@chainsafe/libp2p-noise": "^15.0.0", "@libp2p/interface": "^1.1.6", @@ -4673,9 +4673,9 @@ } }, "node_modules/@octokit/openapi-types": { - "version": "22.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.0.0.tgz", - "integrity": "sha512-kWzSxSIBjCtwrT8/O/A/nrSjmHvR5I9GGTHPyBU19VuEae+QZfaPnnfLwXgV56n51xHN3U2dYy8zh/kO9/39ig==", + "version": "22.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-22.0.1.tgz", + "integrity": "sha512-1yN5m1IMNXthoBDUXFF97N1gHop04B3H8ws7wtOr8GgRyDO1gKALjwMHARNBoMBiB/2vEe/vxstrApcJZzQbnQ==", "dev": true }, "node_modules/@octokit/plugin-paginate-rest": { @@ -4772,14 +4772,14 @@ } }, "node_modules/@octokit/request": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.3.1.tgz", - "integrity": "sha512-fin4cl5eHN5Ybmb/gtn7YZ+ycyUlcyqqkg5lfxeSChqj7sUt6TNaJPehREi+0PABKLREYL8pfaUhH3TicEWNoA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.0.tgz", + "integrity": "sha512-9Bb014e+m2TgBeEJGEbdplMVWwPmL1FPtggHQRkV+WVsMggPtEkLKPlcVYm/o8xKLkpJ7B+6N8WfQMtDLX2Dpw==", "dev": true, "dependencies": { "@octokit/endpoint": "^9.0.1", "@octokit/request-error": "^5.1.0", - "@octokit/types": "^13.0.0", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" }, "engines": { @@ -4801,12 +4801,12 @@ } }, "node_modules/@octokit/types": { - "version": "13.2.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.2.0.tgz", - "integrity": "sha512-K4rpfbIQLe4UimS/PWZAcImhZUC80lhe2f1NpAaaTulPJXv54QIAFFCQEEbdQdqTV/745QDmdvp8NI49LaI00A==", + "version": "13.4.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.4.0.tgz", + "integrity": "sha512-WlMegy3lPXYWASe3k9Jslc5a0anrYAYMWtsFrxBTdQjS70hvLH6C+PGvHbOsgy3RA3LouGJoU/vAt4KarecQLQ==", "dev": true, "dependencies": { - "@octokit/openapi-types": "^22.0.0" + "@octokit/openapi-types": "^22.0.1" } }, "node_modules/@opentelemetry/api": { @@ -7286,9 +7286,9 @@ } }, "node_modules/@types/node": { - "version": "20.12.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.6.tgz", - "integrity": "sha512-3KurE8taB8GCvZBPngVbp0lk5CKi8M9f9k1rsADh0Evdz5SzJ+Q+Hx9uHoFGsLnLnd1xmkDQr2hVhlA0Mn0lKQ==", + "version": "20.12.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz", + "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", "dependencies": { "undici-types": "~5.26.4" } @@ -8939,9 +8939,9 @@ } }, "node_modules/blockstore-core": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/blockstore-core/-/blockstore-core-4.4.0.tgz", - "integrity": "sha512-tjOJAJMPWlqahqCjn5awLJz2eZeJnrGOBA0OInBFK69/FfPZbSID2t7s5jFcBRhGaglca56BzG4t5XOV3MPxOQ==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/blockstore-core/-/blockstore-core-4.4.1.tgz", + "integrity": "sha512-peXfL9ZLx1cb84QALocMjhT8CsQ4JsreI/AitlN1inipSdC/G+jcYVJCqeCD5ecSTv/0LMpg8NlAPH/eBYZLjA==", "dependencies": { "@libp2p/logger": "^4.0.6", "err-code": "^3.0.1", @@ -9781,9 +9781,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001607", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001607.tgz", - "integrity": "sha512-WcvhVRjXLKFB/kmOFVwELtMxyhq3iM/MvmXcyCe2PNf166c39mptscOc/45TTS96n2gpNV2z7+NakArTWZCQ3w==", + "version": "1.0.30001608", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001608.tgz", + "integrity": "sha512-cjUJTQkk9fQlJR2s4HMuPMvTiRggl0rAVMtthQuyOlDWuqHXqN8azLq+pi8B2TjwKJ32diHjUqRIKeFX4z1FoA==", "funding": [ { "type": "opencollective", @@ -11705,9 +11705,9 @@ } }, "node_modules/datastore-level": { - "version": "10.1.7", - "resolved": "https://registry.npmjs.org/datastore-level/-/datastore-level-10.1.7.tgz", - "integrity": "sha512-/oy8Xd8EBID+pbXHAnpbmWRc3wlBjfYU+Z5QrLnpZV+zaDegARotFp7ulXwmRpAXfnymZ3HPC6CwoxBzBIXBCg==", + "version": "10.1.8", + "resolved": "https://registry.npmjs.org/datastore-level/-/datastore-level-10.1.8.tgz", + "integrity": "sha512-XUIUitzK0IfCLTcec1MANWvVb/7l+1a1RuElq4LhIuNoP0bJvEFRZvufXRYzcmWtNByKf8wizEzH/J6ZFwrFCQ==", "dependencies": { "datastore-core": "^9.0.0", "interface-datastore": "^8.0.0", @@ -13019,9 +13019,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.730", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.730.tgz", - "integrity": "sha512-oJRPo82XEqtQAobHpJIR3zW5YO3sSRRkPz2an4yxi1UvqhsGm54vR/wzTFV74a3soDOJ8CKW7ajOOX5ESzddwg==" + "version": "1.4.731", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.731.tgz", + "integrity": "sha512-+TqVfZjpRz2V/5SPpmJxq9qK620SC5SqCnxQIOi7i/U08ZDcTpKbT7Xjj9FU5CbXTMUb4fywbIr8C7cGv4hcjw==" }, "node_modules/electron-window": { "version": "0.8.1", @@ -14669,9 +14669,9 @@ "dev": true }, "node_modules/fast-json-stringify": { - "version": "5.14.0", - "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.14.0.tgz", - "integrity": "sha512-6m9a2JN9kDFMADmP9MHLbLPrFu5oSSfrwFLzpcqt/aFgcEi+SVhTJGsx/Wivlls8fQ3OnP8UDNfIY1d1qCC50w==", + "version": "5.14.1", + "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.14.1.tgz", + "integrity": "sha512-J1Grbf0oSXV3lKsBf3itz1AvRk43qVrx3Ac10sNvi3LZaz1by4oDdYKFrJycPhS8+Gb7y8rgV/Jqw1UZVjyNvw==", "dependencies": { "@fastify/merge-json-schemas": "^0.1.0", "ajv": "^8.10.0", @@ -19012,9 +19012,9 @@ } }, "node_modules/libp2p": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/libp2p/-/libp2p-1.3.2.tgz", - "integrity": "sha512-5qgiur5P1trzpMQcxlVEk1AIaL4+VnDlLwyWfFENwU3f4hcIvEPGJQdpjQpbdQZyN8H2OjcUxZsUI1tQf4jSjg==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/libp2p/-/libp2p-1.3.3.tgz", + "integrity": "sha512-EtSPDIOYHqa66S035XuEokf0U+89zvxxFJxdGPKqat8GWNmy3+LLJTZ0ZO2q4XPjKcACvyERP3uX5WK3Lg5pMw==", "dependencies": { "@libp2p/crypto": "^4.0.5", "@libp2p/interface": "^1.1.6", @@ -30112,6 +30112,23 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/read-package-up": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/read-package-up/-/read-package-up-11.0.0.tgz", + "integrity": "sha512-MbgfoNPANMdb4oRBNg5eqLbB2t2r+o5Ua1pNt8BqGp4I0FJZhuVSOj3PaBPni4azWuSzEdNn2evevzVmEk1ohQ==", + "dev": true, + "dependencies": { + "find-up-simple": "^1.0.0", + "read-pkg": "^9.0.0", + "type-fest": "^4.6.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/read-pkg": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-9.0.1.tgz", @@ -30904,9 +30921,9 @@ "dev": true }, "node_modules/semantic-release": { - "version": "23.0.7", - "resolved": "https://registry.npmjs.org/semantic-release/-/semantic-release-23.0.7.tgz", - "integrity": "sha512-PFxXQE57zrYiCbWKkdsVUF08s0SifEw3WhDhrN47ZEUWQiLl21FI9Dg/H8g7i/lPx0IkF6u7PjJbgxPceXKBeg==", + "version": "23.0.8", + "resolved": "https://registry.npmjs.org/semantic-release/-/semantic-release-23.0.8.tgz", + "integrity": "sha512-yZkuWcTTfh5h/DrR4Q4QvJSARJdb6wjwn/sN0qKMYEkvwaVFek8YWfrgtL8oWaRdl0fLte0Y1wWMzLbwoaII1g==", "dev": true, "dependencies": { "@semantic-release/commit-analyzer": "^12.0.0", @@ -30932,7 +30949,7 @@ "micromatch": "^4.0.2", "p-each-series": "^3.0.0", "p-reduce": "^3.0.0", - "read-pkg-up": "^11.0.0", + "read-package-up": "^11.0.0", "resolve-from": "^5.0.0", "semver": "^7.3.2", "semver-diff": "^4.0.0", @@ -30956,9 +30973,9 @@ } }, "node_modules/semantic-release/node_modules/@octokit/core": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-6.1.1.tgz", - "integrity": "sha512-uVypPdnZV7YoEa69Ky2kTSw3neFLGT0PZ54OwUMDph7w6TmhF0ZnoVcvb/kYnjDHCFo2mfoeRDYifLKhLNasUg==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-6.1.2.tgz", + "integrity": "sha512-hEb7Ma4cGJGEUNOAVmyfdB/3WirWMg5hDuNFVejGEDFqupeOysLc2sG6HJxY2etBp5YQu5Wtxwi020jS9xlUwg==", "dev": true, "dependencies": { "@octokit/auth-token": "^5.0.0", @@ -31000,19 +31017,13 @@ "node": ">= 18" } }, - "node_modules/semantic-release/node_modules/@octokit/openapi-types": { - "version": "20.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", - "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", - "dev": true - }, "node_modules/semantic-release/node_modules/@octokit/plugin-paginate-rest": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-11.0.0.tgz", - "integrity": "sha512-HLz6T9HWeNkX3SVqc7FUYlh+TAg3G7gCc1MGuNcov8mSrFU9dc4ABmRmgqR9TsY1doUx42vLN5UxxWlnqBX8xw==", + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-11.0.1.tgz", + "integrity": "sha512-6pPLXJOKXXhPkMlDU2pd6LCkiuAccTTJAsMlN7P/WbK3b1xbH/d90SGbdmz6yBadJJqM0xg80KRS5dYUHbo3oQ==", "dev": true, "dependencies": { - "@octokit/types": "^13.0.0" + "@octokit/types": "^13.3.0" }, "engines": { "node": ">= 18" @@ -31055,14 +31066,14 @@ } }, "node_modules/semantic-release/node_modules/@octokit/request": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.0.1.tgz", - "integrity": "sha512-kL+cAcbSl3dctYLuJmLfx6Iku2MXXy0jszhaEIjQNaCp4zjHXrhVAHeuaRdNvJjW9qjl3u1MJ72+OuBP0YW/pg==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.1.0.tgz", + "integrity": "sha512-1mDzqKSiryRKZM++MhO6WQBukWbikes6AN6UTxB5vpRnNUbPDkVfUhpSvZ3aXYEFnbcV8DZkikOnCr3pdgMD3Q==", "dev": true, "dependencies": { "@octokit/endpoint": "^10.0.0", "@octokit/request-error": "^6.0.1", - "@octokit/types": "^12.0.0", + "@octokit/types": "^13.1.0", "universal-user-agent": "^7.0.2" }, "engines": { @@ -31081,15 +31092,6 @@ "node": ">= 18" } }, - "node_modules/semantic-release/node_modules/@octokit/request/node_modules/@octokit/types": { - "version": "12.6.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", - "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", - "dev": true, - "dependencies": { - "@octokit/openapi-types": "^20.0.0" - } - }, "node_modules/semantic-release/node_modules/@semantic-release/commit-analyzer": { "version": "12.0.0", "resolved": "https://registry.npmjs.org/@semantic-release/commit-analyzer/-/commit-analyzer-12.0.0.tgz", @@ -32882,18 +32884,13 @@ } }, "node_modules/stream-to-it": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/stream-to-it/-/stream-to-it-0.2.4.tgz", - "integrity": "sha512-4vEbkSs83OahpmBybNJXlJd7d6/RxzkkSdT3I0mnGt79Xd2Kk+e1JqbvAvsQfCeKj3aKb0QIWkyK3/n0j506vQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-to-it/-/stream-to-it-1.0.1.tgz", + "integrity": "sha512-AqHYAYPHcmvMrcLNgncE/q0Aj/ajP6A4qGhxP6EVn7K3YTNs0bJpJyk57wc2Heb7MUL64jurvmnmui8D9kjZgA==", "dependencies": { - "get-iterator": "^1.0.2" + "it-stream-types": "^2.0.1" } }, - "node_modules/stream-to-it/node_modules/get-iterator": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/get-iterator/-/get-iterator-1.0.2.tgz", - "integrity": "sha512-v+dm9bNVfOYsY1OrhaCrmyOcYoSeVvbt+hHZ0Au+T+p1y+0Uyj9aMaGIeUTT6xdpRbWzDeYKvfOslPhggQMcsg==" - }, "node_modules/streaming-json-stringify": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/streaming-json-stringify/-/streaming-json-stringify-3.1.0.tgz", diff --git a/playwright.config.ts b/playwright.config.ts index c829324..22f66e0 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,6 +1,6 @@ import { join } from 'node:path' import { defineConfig, devices } from '@playwright/test' -import { PORT } from './src/constants.js' +import { HTTP_PORT } from './src/constants.js' /** * Run one of the variants of `npm run start` by setting the PLAYWRIGHT_START_CMD_MOD environment variable. @@ -60,7 +60,7 @@ export default defineConfig({ /* Run your local dev server before starting the tests */ webServer: { command: `npm run build && npm run start${PLAYWRIGHT_START_CMD_MOD}`, - port: PORT, + port: HTTP_PORT, // Tiros does not re-use the existing server. reuseExistingServer: process.env.CI == null, env: { diff --git a/src/constants.ts b/src/constants.ts index c6f14f8..2633464 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,12 @@ -export const PORT = Number(process.env.PORT ?? 8080) +/** + * Where we listen for gateway requests + */ +export const HTTP_PORT = Number(process.env.HTTP_PORT ?? 8080) + +/** + * Where we listen for RPC API requests + */ +export const RPC_PORT = Number(process.env.RPC_PORT ?? 5001) export const HOST = process.env.HOST ?? '0.0.0.0' @@ -56,6 +64,16 @@ export const USE_DELEGATED_ROUTING = process.env.USE_DELEGATED_ROUTING !== 'fals */ export const DELEGATED_ROUTING_V1_HOST = process.env.DELEGATED_ROUTING_V1_HOST ?? 'https://delegated-ipfs.dev' +/** + * How long to wait for GC to complete + */ +export const GC_TIMEOUT_MS = 20000 + +/** + * How long to wait for the healthcheck retrieval of an identity CID to complete + */ +export const HEALTHCHECK_TIMEOUT_MS = 1000 + /** * You can set `RECOVERABLE_ERRORS` to a comma delimited list of errors to recover from. * If you want to recover from all errors, set `RECOVERABLE_ERRORS` to 'all'. diff --git a/src/healthcheck.ts b/src/healthcheck.ts index d92e9f4..18b7575 100644 --- a/src/healthcheck.ts +++ b/src/healthcheck.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -import { HOST, PORT } from './constants.js' +import { HOST, RPC_PORT } from './constants.js' import { logger } from './logger.js' const log = logger.forComponent('healthcheck') @@ -7,7 +7,7 @@ const log = logger.forComponent('healthcheck') /** * This healthcheck script is used to check if the server is running and healthy. */ -const rootReq = await fetch(`http://${HOST}:${PORT}/api/v0/http-gateway-healthcheck`, { method: 'GET' }) +const rootReq = await fetch(`http://${HOST}:${RPC_PORT}/api/v0/http-gateway-healthcheck`, { method: 'GET' }) const status = rootReq.status log(`Healthcheck status: ${status}`) diff --git a/src/helia-http-gateway.ts b/src/helia-http-gateway.ts new file mode 100644 index 0000000..1d3d01d --- /dev/null +++ b/src/helia-http-gateway.ts @@ -0,0 +1,170 @@ +import { USE_SUBDOMAINS } from './constants.js' +import { dnsLinkLabelEncoder, isInlinedDnsLink } from './dns-link-labels.js' +import { getFullUrlFromFastifyRequest, getRequestAwareSignal } from './helia-server.js' +import { getIpnsAddressDetails } from './ipns-address-utils.js' +import type { VerifiedFetch } from '@helia/verified-fetch' +import type { ComponentLogger } from '@libp2p/interface' +import type { FastifyReply, FastifyRequest, RouteOptions } from 'fastify' +import type { Helia } from 'helia' + +interface EntryParams { + ns: string + address: string + '*': string +} + +export interface HeliaHTTPGatewayOptions { + logger: ComponentLogger + helia: Helia + fetch: VerifiedFetch +} + +export function httpGateway (opts: HeliaHTTPGatewayOptions): RouteOptions[] { + const log = opts.logger.forComponent('http-gateway') + + /** + * Redirects to the subdomain gateway. + */ + async function handleEntry (request: FastifyRequest, reply: FastifyReply): Promise { + const { params } = request + log('fetch request %s', request.url) + const { ns: namespace, '*': relativePath, address } = params as EntryParams + + log('handling entry: ', { address, namespace, relativePath }) + if (!USE_SUBDOMAINS) { + log('subdomains are disabled, fetching without subdomain') + return fetch(request, reply) + } else { + log('subdomains are enabled, redirecting to subdomain') + } + + const { peerId, cid } = getIpnsAddressDetails(address) + if (peerId != null) { + return fetch(request, reply) + } + const cidv1Address = cid?.toString() + + const query = request.query as Record + log.trace('query: ', query) + // eslint-disable-next-line no-warning-comments + // TODO: enable support for query params + if (query != null) { + // http://localhost:8090/ipfs/bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly/dir?format=raw + // eslint-disable-next-line no-warning-comments + // TODO: temporary ipfs gateway spec? + // if (query.uri != null) { + // // Test = http://localhost:8080/ipns/?uri=ipns%3A%2F%2Fdnslink-subdomain-gw-test.example.org + // log('got URI query parameter: ', query.uri) + // const url = new URL(query.uri) + // address = url.hostname + // } + // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`) + } + let encodedDnsLink = address + + if (!isInlinedDnsLink(address)) { + encodedDnsLink = dnsLinkLabelEncoder(address) + } + + const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` + log('redirecting to final URL:', finalUrl) + await reply + .headers({ + Location: finalUrl + }) + .code(301) + .send() + } + + /** + * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia. + */ + async function fetch (request: FastifyRequest, reply: FastifyReply): Promise { + const url = getFullUrlFromFastifyRequest(request, log) + log('fetching url "%s" with @helia/verified-fetch', url) + + const signal = getRequestAwareSignal(request, log, { + url + }) + + const resp = await opts.fetch(url, { signal, redirect: 'manual' }) + await convertVerifiedFetchResponseToFastifyReply(resp, reply) + } + + async function convertVerifiedFetchResponseToFastifyReply (verifiedFetchResponse: Response, reply: FastifyReply): Promise { + if (!verifiedFetchResponse.ok) { + log('verified-fetch response not ok: ', verifiedFetchResponse.status) + await reply.code(verifiedFetchResponse.status).send(verifiedFetchResponse.statusText) + return + } + const contentType = verifiedFetchResponse.headers.get('content-type') + if (contentType == null) { + log('verified-fetch response has no content-type') + await reply.code(200).send(verifiedFetchResponse.body) + return + } + if (verifiedFetchResponse.body == null) { + // this should never happen + log('verified-fetch response has no body') + await reply.code(501).send('empty') + return + } + + const headers: Record = {} + for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { + headers[headerName] = headerValue + } + // Fastify really does not like streams despite what the documentation and github issues say. + const reader = verifiedFetchResponse.body.getReader() + reply.raw.writeHead(verifiedFetchResponse.status, headers) + try { + let done = false + while (!done) { + const { done: _done, value } = await reader.read() + if (value != null) { + reply.raw.write(Buffer.from(value)) + } + done = _done + } + } catch (err) { + log.error('error reading response:', err) + } finally { + reply.raw.end() + } + } + + return [ + { + // without this non-wildcard postfixed path, the '/*' route will match first. + url: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid + method: 'GET', + handler: async (request, reply): Promise => handleEntry(request, reply) + }, + { + url: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath + method: 'GET', + handler: async (request, reply): Promise => handleEntry(request, reply) + }, + { + url: '/*', + method: 'GET', + handler: async (request, reply): Promise => { + try { + await fetch(request, reply) + } catch { + await reply.code(200).send('try /ipfs/ or /ipns/') + } + } + }, + { + url: '/', + method: 'GET', + handler: async (request, reply): Promise => { + if (USE_SUBDOMAINS && request.hostname.split('.').length > 1) { + return fetch(request, reply) + } + await reply.code(200).send('try /ipfs/ or /ipns/') + } + } + ] +} diff --git a/src/helia-rpc-api.ts b/src/helia-rpc-api.ts new file mode 100644 index 0000000..4821e19 --- /dev/null +++ b/src/helia-rpc-api.ts @@ -0,0 +1,106 @@ +import { GC_TIMEOUT_MS, HEALTHCHECK_TIMEOUT_MS } from './constants.js' +import { getRequestAwareSignal } from './helia-server.js' +import type { VerifiedFetch } from '@helia/verified-fetch' +import type { ComponentLogger } from '@libp2p/interface' +import type { FastifyReply, FastifyRequest, RouteOptions } from 'fastify' +import type { Helia } from 'helia' + +const HELIA_RELEASE_INFO_API = (version: string): string => `https://api.github.com/repos/ipfs/helia/git/ref/tags/helia-v${version}` + +export interface HeliaRPCAPIOptions { + logger: ComponentLogger + helia: Helia + fetch: VerifiedFetch +} + +export function rpcApi (opts: HeliaRPCAPIOptions): RouteOptions[] { + const log = opts.logger.forComponent('rpc-api') + let heliaVersionInfo: { Version: string, Commit: string } | undefined + + /** + * Get the helia version + */ + async function heliaVersion (request: FastifyRequest, reply: FastifyReply): Promise { + try { + if (heliaVersionInfo === undefined) { + log('fetching Helia version info') + const { default: packageJson } = await import('../../node_modules/helia/package.json', { + assert: { type: 'json' } + }) + const { version: heliaVersionString } = packageJson + log('helia version string:', heliaVersionString) + + // handling the next versioning strategy + const [heliaNextVersion, heliaNextCommit] = heliaVersionString.split('-') + if (heliaNextCommit != null) { + heliaVersionInfo = { + Version: heliaNextVersion, + Commit: heliaNextCommit + } + } else { + // if this is not a next version, we will fetch the commit from github. + const ghResp = await (await fetch(HELIA_RELEASE_INFO_API(heliaVersionString))).json() + heliaVersionInfo = { + Version: heliaVersionString, + Commit: ghResp.object.sha.slice(0, 7) + } + } + } + + log('helia version info:', heliaVersionInfo) + await reply.code(200).header('Content-Type', 'application/json; charset=utf-8').send(heliaVersionInfo) + } catch (error) { + await reply.code(500).send(error) + } + } + + /** + * GC the node + */ + async function gc (request: FastifyRequest, reply: FastifyReply): Promise { + log('running `gc` on Helia node') + const signal = getRequestAwareSignal(request, log, { + timeout: GC_TIMEOUT_MS + }) + await opts.helia.gc({ signal }) + await reply.code(200).send('OK') + } + + async function healthCheck (request: FastifyRequest, reply: FastifyReply): Promise { + const signal = getRequestAwareSignal(request, log, { + timeout: HEALTHCHECK_TIMEOUT_MS + }) + try { + // echo "hello world" | npx kubo add --cid-version 1 -Q --inline + // inline CID is bafkqaddimvwgy3zao5xxe3debi + await opts.fetch('ipfs://bafkqaddimvwgy3zao5xxe3debi', { signal, redirect: 'follow' }) + await reply.code(200).send('OK') + } catch (error) { + await reply.code(500).send(error) + } + } + + return [ + { + url: '/api/v0/version', + method: ['POST', 'GET'], + handler: async (request, reply): Promise => heliaVersion(request, reply) + }, { + url: '/api/v0/repo/gc', + method: ['POST', 'GET'], + handler: async (request, reply): Promise => gc(request, reply) + }, + { + url: '/api/v0/http-gateway-healthcheck', + method: 'GET', + handler: async (request, reply): Promise => healthCheck(request, reply) + }, + { + url: '/*', + method: 'GET', + handler: async (request, reply): Promise => { + await reply.code(400).send('API + Method not supported') + } + } + ] +} diff --git a/src/helia-server.ts b/src/helia-server.ts index 5f6a3d3..65869c6 100644 --- a/src/helia-server.ts +++ b/src/helia-server.ts @@ -1,349 +1,63 @@ import { setMaxListeners } from 'node:events' -import { createVerifiedFetch, type VerifiedFetch } from '@helia/verified-fetch' -import { type FastifyReply, type FastifyRequest, type RouteGenericInterface } from 'fastify' -import { USE_SUBDOMAINS } from './constants.js' -import { contentTypeParser } from './content-type-parser.js' -import { dnsLinkLabelEncoder, isInlinedDnsLink } from './dns-link-labels.js' -import { getCustomHelia } from './get-custom-helia.js' -import { getIpnsAddressDetails } from './ipns-address-utils.js' -import type { ComponentLogger, Logger } from '@libp2p/interface' -import type { Helia } from 'helia' +import type { Logger } from '@libp2p/interface' +import type { FastifyRequest } from 'fastify' -const HELIA_RELEASE_INFO_API = (version: string): string => `https://api.github.com/repos/ipfs/helia/git/ref/tags/helia-v${version}` - -export interface RouteEntry { - path: string - type: 'GET' | 'POST' | 'OPTIONS' - handler(request: FastifyRequest, reply: FastifyReply): Promise -} +export function getFullUrlFromFastifyRequest (request: FastifyRequest, log: Logger): string { + let query = '' + if (request.query != null) { + log('request.query:', request.query) + const pairs: string[] = [] + Object.keys(request.query).forEach((key: string) => { + const value = (request.query as Record)[key] + pairs.push(`${key}=${value}`) + }) + if (pairs.length > 0) { + query += '?' + pairs.join('&') + } + } -interface RouteHandler { - request: FastifyRequest - reply: FastifyReply + return `${request.protocol}://${request.hostname}${request.url}${query}` } -interface EntryParams { - ns: string - address: string - '*': string +export interface GetRequestAwareSignalOpts { + timeout?: number + url?: string } -export class HeliaServer { - private heliaFetch!: VerifiedFetch - private heliaVersionInfo!: { Version: string, Commit: string } - private readonly log: Logger - public isReady: Promise - public routes: RouteEntry[] - private heliaNode!: Helia - - constructor (logger: ComponentLogger) { - this.log = logger.forComponent('server') - this.isReady = this.init() - .then(() => { - this.log('initialized') - }) - .catch((error) => { - this.log.error('error initializing:', error) - throw error - }) - this.routes = [] - } - - /** - * Initialize the HeliaServer instance - */ - async init (): Promise { - this.heliaNode = await getCustomHelia() - this.heliaFetch = await createVerifiedFetch(this.heliaNode, { contentTypeParser }) - - this.log('heliaServer Started!') - this.routes = [ - { - // without this non-wildcard postfixed path, the '/*' route will match first. - path: '/:ns(ipfs|ipns)/:address', // ipns/dnsLink or ipfs/cid - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, - { - path: '/:ns(ipfs|ipns)/:address/*', // ipns/dnsLink/relativePath or ipfs/cid/relativePath - type: 'GET', - handler: async (request, reply): Promise => this.handleEntry({ request, reply }) - }, - { - path: '/api/v0/version', - type: 'POST', - handler: async (request, reply): Promise => this.heliaVersion({ request, reply }) - }, { - path: '/api/v0/version', - type: 'GET', - handler: async (request, reply): Promise => this.heliaVersion({ request, reply }) - }, { - path: '/api/v0/repo/gc', - type: 'POST', - handler: async (request, reply): Promise => this.gc({ request, reply }) - }, - { - path: '/api/v0/repo/gc', - type: 'GET', - handler: async (request, reply): Promise => this.gc({ request, reply }) - }, - { - path: '/api/v0/http-gateway-healthcheck', - type: 'GET', - handler: async (request, reply): Promise => { - const signal = AbortSignal.timeout(1000) - setMaxListeners(Infinity, signal) - try { - // echo "hello world" | npx kubo add --cid-version 1 -Q --inline - // inline CID is bafkqaddimvwgy3zao5xxe3debi - await this.heliaFetch('ipfs://bafkqaddimvwgy3zao5xxe3debi', { signal, redirect: 'follow' }) - await reply.code(200).send('OK') - } catch (error) { - await reply.code(500).send(error) - } - } - }, - { - path: '/*', - type: 'GET', - handler: async (request, reply): Promise => { - try { - await this.fetch({ request, reply }) - } catch { - await reply.code(200).send('try /ipfs/ or /ipns/') - } - } - }, - { - path: '/', - type: 'GET', - handler: async (request, reply): Promise => { - if (request.url.includes('/api/v0')) { - await reply.code(400).send('API + Method not supported') - return - } - if (USE_SUBDOMAINS && request.hostname.split('.').length > 1) { - return this.fetch({ request, reply }) - } - await reply.code(200).send('try /ipfs/ or /ipns/') - } - } - ] - } - - /** - * Redirects to the subdomain gateway. - */ - private async handleEntry ({ request, reply }: RouteHandler): Promise { - const { params } = request - this.log('fetch request %s', request.url) - const { ns: namespace, '*': relativePath, address } = params as EntryParams - - this.log('handling entry: ', { address, namespace, relativePath }) - if (!USE_SUBDOMAINS) { - this.log('subdomains are disabled, fetching without subdomain') - return this.fetch({ request, reply }) +export function getRequestAwareSignal (request: FastifyRequest, log: Logger, options: GetRequestAwareSignalOpts = {}): AbortSignal { + const url = options.url ?? getFullUrlFromFastifyRequest(request, log) + + const opController = new AbortController() + setMaxListeners(Infinity, opController.signal) + const cleanupFn = (): void => { + if (request.raw.readableAborted) { + log.trace('request aborted by client for url "%s"', url) + } else if (request.raw.destroyed) { + log.trace('request destroyed for url "%s"', url) + } else if (request.raw.complete) { + log.trace('request closed or ended in completed state for url "%s"', url) } else { - this.log('subdomains are enabled, redirecting to subdomain') - } - - const { peerId, cid } = getIpnsAddressDetails(address) - if (peerId != null) { - return this.fetch({ request, reply }) - } - const cidv1Address = cid?.toString() - - const query = request.query as Record - this.log.trace('query: ', query) - // eslint-disable-next-line no-warning-comments - // TODO: enable support for query params - if (query != null) { - // http://localhost:8090/ipfs/bafybeie72edlprgtlwwctzljf6gkn2wnlrddqjbkxo3jomh4n7omwblxly/dir?format=raw - // eslint-disable-next-line no-warning-comments - // TODO: temporary ipfs gateway spec? - // if (query.uri != null) { - // // Test = http://localhost:8080/ipns/?uri=ipns%3A%2F%2Fdnslink-subdomain-gw-test.example.org - // this.log('got URI query parameter: ', query.uri) - // const url = new URL(query.uri) - // address = url.hostname - // } - // finalUrl += encodeURIComponent(`?${new URLSearchParams(request.query).toString()}`) - } - let encodedDnsLink = address - if (!isInlinedDnsLink(address)) { - encodedDnsLink = dnsLinkLabelEncoder(address) - } - - const finalUrl = `${request.protocol}://${cidv1Address ?? encodedDnsLink}.${namespace}.${request.hostname}/${relativePath ?? ''}` - this.log('redirecting to final URL:', finalUrl) - await reply - .headers({ - Location: finalUrl - }) - .code(301) - .send() - } - - #getFullUrlFromFastifyRequest (request: FastifyRequest): string { - let query = '' - if (request.query != null) { - this.log('request.query:', request.query) - const pairs: string[] = [] - Object.keys(request.query).forEach((key: string) => { - const value = (request.query as Record)[key] - pairs.push(`${key}=${value}`) - }) - if (pairs.length > 0) { - query += '?' + pairs.join('&') - } + log.trace('request closed or ended gracefully for url "%s"', url) } - return `${request.protocol}://${request.hostname}${request.url}${query}` - } - - #convertVerifiedFetchResponseToFastifyReply = async (verifiedFetchResponse: Response, reply: FastifyReply): Promise => { - if (!verifiedFetchResponse.ok) { - this.log('verified-fetch response not ok: ', verifiedFetchResponse.status) - await reply.code(verifiedFetchResponse.status).send(verifiedFetchResponse.statusText) - return - } - const contentType = verifiedFetchResponse.headers.get('content-type') - if (contentType == null) { - this.log('verified-fetch response has no content-type') - await reply.code(200).send(verifiedFetchResponse.body) - return - } - if (verifiedFetchResponse.body == null) { - // this should never happen - this.log('verified-fetch response has no body') - await reply.code(501).send('empty') - return - } - - const headers: Record = {} - for (const [headerName, headerValue] of verifiedFetchResponse.headers.entries()) { - headers[headerName] = headerValue - } - // Fastify really does not like streams despite what the documentation and github issues say. - const reader = verifiedFetchResponse.body.getReader() - reply.raw.writeHead(verifiedFetchResponse.status, headers) - try { - let done = false - while (!done) { - const { done: _done, value } = await reader.read() - if (value != null) { - reply.raw.write(Buffer.from(value)) - } - done = _done - } - } catch (err) { - this.log.error('error reading response:', err) - } finally { - reply.raw.end() - } - } - - #getRequestAwareSignal (request: FastifyRequest, url = this.#getFullUrlFromFastifyRequest(request), timeout?: number): AbortSignal { - const opController = new AbortController() - setMaxListeners(Infinity, opController.signal) - const cleanupFn = (): void => { - if (request.raw.readableAborted) { - this.log.trace('request aborted by client for url "%s"', url) - } else if (request.raw.destroyed) { - this.log.trace('request destroyed for url "%s"', url) - } else if (request.raw.complete) { - this.log.trace('request closed or ended in completed state for url "%s"', url) - } else { - this.log.trace('request closed or ended gracefully for url "%s"', url) - } - // we want to stop all further processing because the request is closed - opController.abort() - } - /** - * The 'close' event is emitted when the stream and any of its underlying resources (a file descriptor, for example) have been closed. The event indicates that no more events will be emitted, and no further computation will occur. - * A Readable stream will always emit the 'close' event if it is created with the emitClose option. - * - * @see https://nodejs.org/api/stream.html#event-close_1 - */ - request.raw.on('close', cleanupFn) - - if (timeout != null) { - setTimeout(() => { - this.log.trace('request timed out for url "%s"', url) - opController.abort() - }, timeout) - } - return opController.signal + // we want to stop all further processing because the request is closed + opController.abort() } /** - * Fetches a content for a subdomain, which basically queries delegated routing API and then fetches the path from helia. + * The 'close' event is emitted when the stream and any of its underlying resources (a file descriptor, for example) have been closed. The event indicates that no more events will be emitted, and no further computation will occur. + * A Readable stream will always emit the 'close' event if it is created with the emitClose option. + * + * @see https://nodejs.org/api/stream.html#event-close_1 */ - async fetch ({ request, reply }: RouteHandler): Promise { - const url = this.#getFullUrlFromFastifyRequest(request) - this.log('fetching url "%s" with @helia/verified-fetch', url) - - const signal = this.#getRequestAwareSignal(request, url) + request.raw.on('close', cleanupFn) - await this.isReady - const resp = await this.heliaFetch(url, { signal, redirect: 'manual' }) - await this.#convertVerifiedFetchResponseToFastifyReply(resp, reply) - } - - /** - * Get the helia version - */ - async heliaVersion ({ reply }: RouteHandler): Promise { - await this.isReady - - try { - if (this.heliaVersionInfo === undefined) { - this.log('fetching Helia version info') - const { default: packageJson } = await import('../../node_modules/helia/package.json', { - assert: { type: 'json' } - }) - const { version: heliaVersionString } = packageJson - this.log('helia version string:', heliaVersionString) - - // handling the next versioning strategy - const [heliaNextVersion, heliaNextCommit] = heliaVersionString.split('-') - if (heliaNextCommit != null) { - this.heliaVersionInfo = { - Version: heliaNextVersion, - Commit: heliaNextCommit - } - } else { - // if this is not a next version, we will fetch the commit from github. - const ghResp = await (await fetch(HELIA_RELEASE_INFO_API(heliaVersionString))).json() - this.heliaVersionInfo = { - Version: heliaVersionString, - Commit: ghResp.object.sha.slice(0, 7) - } - } - } - - this.log('helia version info:', this.heliaVersionInfo) - await reply.code(200).header('Content-Type', 'application/json; charset=utf-8').send(this.heliaVersionInfo) - } catch (error) { - await reply.code(500).send(error) - } - } - - /** - * GC the node - */ - async gc ({ reply, request }: RouteHandler): Promise { - await this.isReady - this.log('running `gc` on Helia node') - const signal = this.#getRequestAwareSignal(request, undefined, 20000) - await this.heliaNode?.gc({ signal }) - await reply.code(200).send('OK') + if (options.timeout != null) { + setTimeout(() => { + log.trace('request timed out for url "%s"', url) + opController.abort() + }, options.timeout) } - /** - * Stop the server - */ - async stop (): Promise { - await this.heliaFetch.stop() - } + return opController.signal } diff --git a/src/index.ts b/src/index.ts index 6964e55..20b8974 100644 --- a/src/index.ts +++ b/src/index.ts @@ -92,25 +92,25 @@ * #### Disable libp2p * * ```sh - * $ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia + * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia * ``` * * #### Disable bitswap * * ```sh - * $ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia + * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia * ``` * * #### Disable trustless gateways * * ```sh - * $ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia + * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia * ``` * * #### Customize trustless gateways * * ```sh - * $ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia + * $ docker run -it -p $RPC_PORT:5001 -p $HTTP_PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia * ``` * * * @@ -158,90 +158,50 @@ import compress from '@fastify/compress' import cors from '@fastify/cors' -import Fastify from 'fastify' +import { createVerifiedFetch } from '@helia/verified-fetch' +import fastify, { type FastifyInstance, type RouteOptions } from 'fastify' import metricsPlugin from 'fastify-metrics' -import { HOST, PORT, METRICS, ECHO_HEADERS, FASTIFY_DEBUG, RECOVERABLE_ERRORS, ALLOW_UNHANDLED_ERROR_RECOVERY } from './constants.js' -import { HeliaServer, type RouteEntry } from './helia-server.js' +import { HOST, HTTP_PORT, RPC_PORT, METRICS, ECHO_HEADERS, FASTIFY_DEBUG, RECOVERABLE_ERRORS, ALLOW_UNHANDLED_ERROR_RECOVERY } from './constants.js' +import { contentTypeParser } from './content-type-parser.js' +import { getCustomHelia } from './get-custom-helia.js' +import { httpGateway } from './helia-http-gateway.js' +import { rpcApi } from './helia-rpc-api.js' import { logger } from './logger.js' const log = logger.forComponent('index') -const heliaServer = new HeliaServer(logger) -await heliaServer.isReady +const helia = await getCustomHelia() +const fetch = await createVerifiedFetch(helia, { contentTypeParser }) -// Add the prometheus middleware -const app = Fastify({ - logger: { - enabled: FASTIFY_DEBUG !== '', - msgPrefix: 'helia-http-gateway:fastify ', - level: 'info', - transport: { - target: 'pino-pretty' - } - } -}) - -if (METRICS === 'true') { - await app.register(metricsPlugin.default, { endpoint: '/metrics' }) -} - -await app.register(cors, { - /** - * @see https://github.com/ipfs/gateway-conformance/issues/186 - * @see https://github.com/ipfs/gateway-conformance/blob/d855ec4fb9dac4a5aaecf3776037b005cc74c566/tests/path_gateway_cors_test.go#L16-L56 - */ - allowedHeaders: ['Content-Type', 'Range', 'User-Agent', 'X-Requested-With'], - origin: '*', - exposedHeaders: [ - 'Content-Range', - 'Content-Length', - 'X-Ipfs-Path', - 'X-Ipfs-Roots', - 'X-Chunked-Output', - 'X-Stream-Output' - ], - methods: ['GET', 'HEAD', 'OPTIONS'], - strictPreflight: false, - preflightContinue: true -}) -await app.register(compress, { - global: true -}) -heliaServer.routes.forEach(({ path, type, handler }: RouteEntry) => { - app.route({ - method: type, - url: path, - handler +const [rpcApiServer, httpGatewayServer] = await Promise.all([ + createServer('rpc-api', rpcApi({ + logger, + helia, + fetch + }), { + metrics: false + }), + createServer('http-gateway', httpGateway({ + logger, + helia, + fetch + }), { + metrics: METRICS === 'true' }) -}) +]) -if ([ECHO_HEADERS].includes(true)) { - app.addHook('onRequest', async (request, reply) => { - if (ECHO_HEADERS) { - log('fastify hook onRequest: echoing headers:') - Object.keys(request.headers).forEach((headerName) => { - log('\t %s: %s', headerName, request.headers[headerName]) - }) - } - }) - - app.addHook('onSend', async (request, reply, payload) => { - if (ECHO_HEADERS) { - log('fastify hook onSend: echoing headers:') - const responseHeaders = reply.getHeaders() - Object.keys(responseHeaders).forEach((headerName) => { - log('\t %s: %s', headerName, responseHeaders[headerName]) - }) - } - return payload - }) -} +await Promise.all([ + rpcApiServer.listen({ port: RPC_PORT, host: HOST }), + httpGatewayServer.listen({ port: HTTP_PORT, host: HOST }) +]) -await app.listen({ port: PORT, host: HOST }) +console.info(`API server listening on /ip4/${HOST}/tcp/${RPC_PORT}`) // eslint-disable-line no-console +console.info(`Gateway (readonly) server listening on /ip4/${HOST}/tcp/${HTTP_PORT}`) // eslint-disable-line no-console const stopWebServer = async (): Promise => { try { - await app.close() + await rpcApiServer.close() + await httpGatewayServer.close() } catch (error) { log.error(error) process.exit(1) @@ -258,9 +218,11 @@ async function closeGracefully (signal: number | string): Promise { } shutdownRequested = true - await Promise.all([heliaServer.stop().then(() => { - log('Stopped Helia.') - }), stopWebServer()]) + await stopWebServer() + await fetch.stop() + await helia.stop() + + log('Stopped Helia.') process.kill(process.pid, signal) } @@ -281,3 +243,79 @@ const uncaughtHandler = (error: any): void => { process.on('uncaughtException', uncaughtHandler) process.on('unhandledRejection', uncaughtHandler) + +interface ServerOptions { + metrics: boolean +} + +async function createServer (name: string, routes: RouteOptions[], options: ServerOptions): Promise { + const app = fastify({ + logger: { + enabled: FASTIFY_DEBUG !== '', + msgPrefix: `helia-http-gateway:fastify:${name} `, + level: 'info', + transport: { + target: 'pino-pretty' + } + } + }) + + if (options.metrics) { + // Add the prometheus middleware + await app.register(metricsPlugin.default, { endpoint: '/metrics' }) + } + + await app.register(cors, { + /** + * @see https://github.com/ipfs/gateway-conformance/issues/186 + * @see https://github.com/ipfs/gateway-conformance/blob/d855ec4fb9dac4a5aaecf3776037b005cc74c566/tests/path_gateway_cors_test.go#L16-L56 + */ + allowedHeaders: ['Content-Type', 'Range', 'User-Agent', 'X-Requested-With'], + origin: '*', + exposedHeaders: [ + 'Content-Range', + 'Content-Length', + 'X-Ipfs-Path', + 'X-Ipfs-Roots', + 'X-Chunked-Output', + 'X-Stream-Output' + ], + methods: ['GET', 'HEAD', 'OPTIONS'], + strictPreflight: false, + preflightContinue: true + }) + + // enable compression + await app.register(compress, { + global: true + }) + + // set up routes + routes.forEach(route => { + app.route(route) + }) + + if ([ECHO_HEADERS].includes(true)) { + app.addHook('onRequest', async (request, reply) => { + if (ECHO_HEADERS) { + log('fastify hook onRequest: echoing headers:') + Object.keys(request.headers).forEach((headerName) => { + log('\t %s: %s', headerName, request.headers[headerName]) + }) + } + }) + + app.addHook('onSend', async (request, reply, payload) => { + if (ECHO_HEADERS) { + log('fastify hook onSend: echoing headers:') + const responseHeaders = reply.getHeaders() + Object.keys(responseHeaders).forEach((headerName) => { + log('\t %s: %s', headerName, responseHeaders[headerName]) + }) + } + return payload + }) + } + + return app +}