diff --git a/.github/workflows/startos-iso.yaml b/.github/workflows/startos-iso.yaml index 184c2b0c7..7d4a43685 100644 --- a/.github/workflows/startos-iso.yaml +++ b/.github/workflows/startos-iso.yaml @@ -82,8 +82,8 @@ jobs: - name: Set up docker QEMU uses: docker/setup-qemu-action@v2 - - name: Set up system QEMU - run: sudo apt-get update && sudo apt-get install -y qemu-user-static + - name: Set up system dependencies + run: sudo apt-get update && sudo apt-get install -y qemu-user-static systemd-container - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -140,6 +140,10 @@ jobs: }')[matrix.platform] }} steps: + - name: Free space + run: rm -rf /opt/hostedtoolcache* + if: ${{ github.event.inputs.runner != 'fast' }} + - uses: actions/checkout@v3 with: submodules: recursive diff --git a/Makefile b/Makefile index c988b11b8..e1ba4ed53 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ PATCH_DB_CLIENT_SRC := $(shell git ls-files --recurse-submodules patch-db/client GZIP_BIN := $(shell which pigz || which gzip) TAR_BIN := $(shell which gtar || which tar) COMPILED_TARGETS := $(BINS) system-images/compat/docker-images/$(ARCH).tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar container-runtime/rootfs.$(ARCH).squashfs -ALL_TARGETS := $(STARTD_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) $(COMPILED_TARGETS) $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-musl/release/pi-beep; fi) $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; fi') $(PLATFORM_FILE) sdk/lib/test +ALL_TARGETS := $(STARTD_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE) $(COMPILED_TARGETS) $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo cargo-deps/aarch64-unknown-linux-musl/release/pi-beep; fi) $(shell /bin/bash -c 'if [[ "${ENVIRONMENT}" =~ (^|-)unstable($$|-) ]]; then echo cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console; fi') $(PLATFORM_FILE) ifeq ($(REMOTE),) mkdir = mkdir -p $1 @@ -103,7 +103,7 @@ deb: results/$(BASENAME).deb debian/control: build/lib/depends build/lib/conflicts ./debuild/control.sh -results/$(BASENAME).deb: dpkg-build.sh $(DEBIAN_SRC) $(VERSION_FILE) $(PLATFORM_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) +results/$(BASENAME).deb: dpkg-build.sh $(DEBIAN_SRC) $(ALL_TARGETS) PLATFORM=$(PLATFORM) ./dpkg-build.sh $(IMAGE_TYPE): results/$(BASENAME).$(IMAGE_TYPE) @@ -173,7 +173,7 @@ emulate-reflash: $(ALL_TARGETS) upload-ota: results/$(BASENAME).squashfs TARGET=$(TARGET) KEY=$(KEY) ./upload-ota.sh -container-runtime/alpine.$(ARCH).squashfs: +container-runtime/debian.$(ARCH).squashfs: ARCH=$(ARCH) ./container-runtime/download-base-image.sh container-runtime/node_modules: container-runtime/package.json container-runtime/package-lock.json sdk/dist @@ -197,7 +197,7 @@ container-runtime/dist/node_modules container-runtime/dist/package.json containe ./container-runtime/install-dist-deps.sh touch container-runtime/dist/node_modules -container-runtime/rootfs.$(ARCH).squashfs: container-runtime/alpine.$(ARCH).squashfs container-runtime/containerRuntime.rc container-runtime/update-image.sh container-runtime/dist/index.js container-runtime/dist/node_modules core/target/$(ARCH)-unknown-linux-musl/release/containerbox | sudo +container-runtime/rootfs.$(ARCH).squashfs: container-runtime/debian.$(ARCH).squashfs container-runtime/container-runtime.service container-runtime/update-image.sh container-runtime/deb-install.sh container-runtime/dist/index.js container-runtime/dist/node_modules core/target/$(ARCH)-unknown-linux-musl/release/containerbox | sudo ARCH=$(ARCH) ./container-runtime/update-image.sh build/lib/depends build/lib/conflicts: build/dpkg-deps/* diff --git a/container-runtime/Dockerfile b/container-runtime/Dockerfile deleted file mode 100644 index f936ee11b..000000000 --- a/container-runtime/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM node:18-alpine - -ADD ./startInit.js /usr/local/lib/startInit.js -ADD ./entrypoint.sh /usr/local/bin/entrypoint.sh \ No newline at end of file diff --git a/container-runtime/container-runtime.service b/container-runtime/container-runtime.service new file mode 100644 index 000000000..b9d5ec5ae --- /dev/null +++ b/container-runtime/container-runtime.service @@ -0,0 +1,9 @@ +[Unit] +Description=StartOS Container Runtime + +[Service] +Type=simple +ExecStart=/usr/bin/node --experimental-detect-module --unhandled-rejections=warn /usr/lib/startos/init/index.js + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/container-runtime/containerRuntime.rc b/container-runtime/containerRuntime.rc deleted file mode 100644 index 203b99659..000000000 --- a/container-runtime/containerRuntime.rc +++ /dev/null @@ -1,10 +0,0 @@ -#!/sbin/openrc-run - -name=containerRuntime -#cfgfile="/etc/containerRuntime/containerRuntime.conf" -command="/usr/bin/node" -command_args="--experimental-detect-module --unhandled-rejections=warn /usr/lib/startos/init/index.js" -pidfile="/run/containerRuntime.pid" -command_background="yes" -output_log="/var/log/containerRuntime.log" -error_log="/var/log/containerRuntime.err" diff --git a/container-runtime/deb-install.sh b/container-runtime/deb-install.sh new file mode 100644 index 000000000..9a64dbbdd --- /dev/null +++ b/container-runtime/deb-install.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +mkdir -p /run/systemd/resolve +echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf + +apt-get update +apt-get install -y curl rsync + +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +source ~/.bashrc +nvm install 20 + +ln -s $(which node) /usr/bin/node + +systemctl enable container-runtime.service + +rm -rf /run/systemd \ No newline at end of file diff --git a/container-runtime/download-base-image.sh b/container-runtime/download-base-image.sh index 23a140ea5..f7dc4c844 100755 --- a/container-runtime/download-base-image.sh +++ b/container-runtime/download-base-image.sh @@ -4,8 +4,8 @@ cd "$(dirname "${BASH_SOURCE[0]}")" set -e -DISTRO=alpine -VERSION=3.19 +DISTRO=debian +VERSION=bookworm ARCH=${ARCH:-$(uname -m)} FLAVOR=default @@ -16,4 +16,4 @@ elif [ "$_ARCH" = "aarch64" ]; then _ARCH=arm64 fi -curl https://images.linuxcontainers.org/$(curl --silent https://images.linuxcontainers.org/meta/1.0/index-system | grep "^$DISTRO;$VERSION;$_ARCH;$FLAVOR;" | head -n1 | sed 's/^.*;//g')/rootfs.squashfs --output alpine.${ARCH}.squashfs \ No newline at end of file +curl https://images.linuxcontainers.org/$(curl --silent https://images.linuxcontainers.org/meta/1.0/index-system | grep "^$DISTRO;$VERSION;$_ARCH;$FLAVOR;" | head -n1 | sed 's/^.*;//g')/rootfs.squashfs --output debian.${ARCH}.squashfs \ No newline at end of file diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index f2bcc5eba..c28615fc9 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -40,6 +40,7 @@ export class MainLoop { ...system.manifest.main.args, ] + await this.setupInterfaces(effects) await effects.setMainStatus({ status: "running" }) const jsMain = (this.system.moduleCode as any)?.jsMain const dockerProcedureContainer = await DockerProcedureContainer.of( @@ -69,6 +70,52 @@ export class MainLoop { } } + private async setupInterfaces(effects: HostSystemStartOs) { + for (const interfaceId in this.system.manifest.interfaces) { + const iface = this.system.manifest.interfaces[interfaceId] + const internalPorts = new Set() + for (const port of Object.values( + iface["tor-config"]?.["port-mapping"] || {}, + )) { + internalPorts.add(parseInt(port)) + } + for (const port of Object.values(iface["lan-config"] || {})) { + internalPorts.add(port.internal) + } + for (const internalPort of internalPorts) { + const torConf = Object.entries( + iface["tor-config"]?.["port-mapping"] || {}, + ) + .map(([external, internal]) => ({ + internal: parseInt(internal), + external: parseInt(external), + })) + .find((conf) => conf.internal == internalPort) + const lanConf = Object.entries(iface["lan-config"] || {}) + .map(([external, conf]) => ({ + external: parseInt(external), + ...conf, + })) + .find((conf) => conf.internal == internalPort) + await effects.bind({ + kind: "multi", + id: interfaceId, + internalPort, + preferredExternalPort: torConf?.external || internalPort, + scheme: "http", + secure: null, + addSsl: lanConf?.ssl + ? { + scheme: "https", + preferredExternalPort: lanConf.external, + alpn: { specified: ["http/1.1"] }, + } + : null, + }) + } + } + } + public async clean(options?: { timeout?: number }) { const { mainEvent, healthLoops } = this const main = await mainEvent diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts index 9b70f884b..85cd1bac3 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts @@ -68,13 +68,24 @@ export const matchManifest = object( volumes: dictionary([string, matchVolume]), interfaces: dictionary([ string, - object({ - name: string, - "tor-config": object({}), - "lan-config": object({}), - ui: boolean, - protocols: array(string), - }), + object( + { + name: string, + "tor-config": object({ + "port-mapping": dictionary([string, string]), + }), + "lan-config": dictionary([ + string, + object({ + ssl: boolean, + internal: number, + }), + ]), + ui: boolean, + protocols: array(string), + }, + ["lan-config", "tor-config"], + ), ]), backup: object({ create: matchProcedure, diff --git a/container-runtime/update-image.sh b/container-runtime/update-image.sh index bef525b3b..1c571408a 100755 --- a/container-runtime/update-image.sh +++ b/container-runtime/update-image.sh @@ -4,12 +4,11 @@ cd "$(dirname "${BASH_SOURCE[0]}")" set -e - - -if mountpoint tmp/combined; then sudo umount tmp/combined; fi +if mountpoint tmp/combined; then sudo umount -R tmp/combined; fi if mountpoint tmp/lower; then sudo umount tmp/lower; fi +sudo rm -rf tmp mkdir -p tmp/lower tmp/upper tmp/work tmp/combined -sudo mount alpine.${ARCH}.squashfs tmp/lower +sudo mount debian.${ARCH}.squashfs tmp/lower sudo mount -t overlay -olowerdir=tmp/lower,upperdir=tmp/upper,workdir=tmp/work overlay tmp/combined QEMU= @@ -18,21 +17,21 @@ if [ "$ARCH" != "$(uname -m)" ]; then sudo cp $(which qemu-$ARCH-static) tmp/combined${QEMU} fi -echo "nameserver 8.8.8.8" | sudo tee tmp/combined/etc/resolv.conf # TODO - delegate to host resolver? -sudo chroot tmp/combined $QEMU /sbin/apk add nodejs rsync sudo mkdir -p tmp/combined/usr/lib/startos/ sudo rsync -a --copy-unsafe-links dist/ tmp/combined/usr/lib/startos/init/ -sudo cp containerRuntime.rc tmp/combined/etc/init.d/containerRuntime +sudo chown -R 0:0 tmp/combined/usr/lib/startos/ +sudo cp container-runtime.service tmp/combined/lib/systemd/system/container-runtime.service +sudo chown 0:0 tmp/combined/lib/systemd/system/container-runtime.service sudo cp ../core/target/$ARCH-unknown-linux-musl/release/containerbox tmp/combined/usr/bin/start-cli -sudo chmod +x tmp/combined/etc/init.d/containerRuntime -sudo chroot tmp/combined $QEMU /sbin/rc-update add containerRuntime default +sudo chown 0:0 tmp/combined/usr/bin/start-cli +echo container-runtime | sha256sum | head -c 32 | cat - <(echo) | sudo tee tmp/combined/etc/machine-id +cat deb-install.sh | sudo systemd-nspawn --console=pipe -D tmp/combined $QEMU /bin/bash +sudo truncate -s 0 tmp/combined/etc/machine-id if [ -n "$QEMU" ]; then sudo rm tmp/combined${QEMU} fi -sudo truncate -s 0 tmp/combined/etc/resolv.conf -sudo chown -R 0:0 tmp/combined rm -f rootfs.${ARCH}.squashfs mkdir -p ../build/lib/container-runtime sudo mksquashfs tmp/combined rootfs.${ARCH}.squashfs diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 0effccd5e..6d0f87c53 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -326,7 +326,7 @@ async fn get_service_port_forward( data: GetServicePortForwardParams, ) -> Result { let internal_port = data.internal_port as u16; - + let context = context.deref()?; let net_service = context.persistent_container.net_service.lock().await; net_service.get_ext_port(data.host_id, internal_port) @@ -446,8 +446,18 @@ struct BindParams { #[serde(flatten)] options: BindOptions, } -async fn bind(_: AnyContext, BindParams { .. }: BindParams) -> Result { - todo!() +async fn bind( + context: EffectContext, + BindParams { + kind, + id, + internal_port, + options, + }: BindParams, +) -> Result<(), Error> { + let ctx = context.deref()?; + let mut svc = ctx.persistent_container.net_service.lock().await; + svc.bind(kind, id, internal_port, options).await } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] diff --git a/dpkg-build.sh b/dpkg-build.sh index e8ffdb0ac..783c205b3 100755 --- a/dpkg-build.sh +++ b/dpkg-build.sh @@ -17,6 +17,7 @@ fi rm -rf dpkg-workdir/$BASENAME mkdir -p dpkg-workdir/$BASENAME +make make install DESTDIR=dpkg-workdir/$BASENAME DEPENDS=$(cat dpkg-workdir/$BASENAME/usr/lib/startos/depends | tr $'\n' ',' | sed 's/,,\+/,/g' | sed 's/,$//') diff --git a/patch-db b/patch-db index 3dc11afd4..e4a3f7b57 160000 --- a/patch-db +++ b/patch-db @@ -1 +1 @@ -Subproject commit 3dc11afd46d93094ac52ae1fef311a91c4561e8c +Subproject commit e4a3f7b577df56c611b21d5ad03eb459e80fb919