From fd1acd3a311b549b72021bc31919c294ea4c4a77 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Mon, 22 Apr 2024 19:46:42 -0600 Subject: [PATCH 01/16] complete get_primary_url fn --- core/startos/src/net/host/address.rs | 2 +- core/startos/src/net/host/mod.rs | 14 +++++++++ .../src/service/service_effect_handler.rs | 29 +++++++++++++++++-- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/core/startos/src/net/host/address.rs b/core/startos/src/net/host/address.rs index cb3b485f6..75abf350e 100644 --- a/core/startos/src/net/host/address.rs +++ b/core/startos/src/net/host/address.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use torut::onion::OnionAddressV3; use ts_rs::TS; -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)] #[serde(rename_all = "camelCase")] #[serde(tag = "kind")] #[ts(export)] diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index 2d50df15a..3066e43cc 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -86,3 +86,17 @@ impl Model { }) // TODO: handle host kind change } } + +impl HostInfo { + pub fn get_host_primary(&self, host_id: &HostId) -> Option { + match self.0.get(&host_id) { + Some(h) => { + match h.primary { + Some(ha) => Some(ha), + None => None, + } + } + None => None, + } + } +} diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index b1411450c..73b634c01 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -25,6 +25,7 @@ use crate::db::model::package::{ use crate::disk::mount::filesystem::idmapped::IdMapped; use crate::disk::mount::filesystem::loop_dev::LoopDev; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; +use crate::net::host::address::HostAddress; use crate::net::host::binding::BindOptions; use crate::net::host::HostKind; use crate::prelude::*; @@ -230,6 +231,7 @@ struct GetPrimaryUrlParams { package_id: Option, service_interface_id: String, callback: Callback, + host_id: HostId, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] @@ -343,8 +345,31 @@ async fn export_service_interface( async fn get_primary_url( context: EffectContext, data: GetPrimaryUrlParams, -) -> Result { - todo!() +) -> Result { + let context = context.deref()?; + let package_id = context.id.clone(); + + let db_model = context + .ctx + .db + .peek() + .await; + +let pkg_data_model = db_model + .as_public() + .as_package_data() + .as_idx(&package_id) + .or_not_found(&package_id)?; + + let host = pkg_data_model + .de()? + .hosts + .get_host_primary(&data.host_id); + + match host { + Some(host_address) => Ok(host_address), + None => Err(Error::new(eyre!("Primary Url not found for {}", data.host_id), crate::ErrorKind::NotFound)), + } } async fn list_service_interfaces( context: EffectContext, From 53e74e2ea802fc0a775651c70edfa71fcd305563 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Thu, 25 Apr 2024 11:41:31 -0600 Subject: [PATCH 02/16] complete clear_network_interfaces fn --- .../src/service/service_effect_handler.rs | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 73b634c01..8399b6e4c 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -333,8 +333,24 @@ async fn get_service_port_forward( let net_service = context.persistent_container.net_service.lock().await; net_service.get_ext_port(data.host_id, internal_port) } -async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result { - todo!() +async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result<(), Error> { + let context = context.deref()?; + let package_id = context.id.clone(); + + context + .ctx + .db + .mutate(|db| { + let model = db + .as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package_id) + .or_not_found(&package_id)? + .as_service_interfaces_mut(); + let mut new_map = BTreeMap::new(); + model.ser(&mut new_map) + }) + .await } async fn export_service_interface( context: EffectContext, From 55d9e6d9a1ae59a1e7609c07a1785b86ec89e7d2 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Thu, 25 Apr 2024 11:42:06 -0600 Subject: [PATCH 03/16] formatting --- .../src/service/service_effect_handler.rs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 8399b6e4c..e4f6d3aeb 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -365,26 +365,22 @@ async fn get_primary_url( let context = context.deref()?; let package_id = context.id.clone(); - let db_model = context - .ctx - .db - .peek() - .await; + let db_model = context.ctx.db.peek().await; -let pkg_data_model = db_model - .as_public() - .as_package_data() - .as_idx(&package_id) - .or_not_found(&package_id)?; + let pkg_data_model = db_model + .as_public() + .as_package_data() + .as_idx(&package_id) + .or_not_found(&package_id)?; + + let host = pkg_data_model.de()?.hosts.get_host_primary(&data.host_id); - let host = pkg_data_model - .de()? - .hosts - .get_host_primary(&data.host_id); - match host { Some(host_address) => Ok(host_address), - None => Err(Error::new(eyre!("Primary Url not found for {}", data.host_id), crate::ErrorKind::NotFound)), + None => Err(Error::new( + eyre!("Primary Url not found for {}", data.host_id), + crate::ErrorKind::NotFound, + )), } } async fn list_service_interfaces( From 79f1d14693780256cf619bcb9263f6e8dd29a89e Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Thu, 25 Apr 2024 16:20:22 -0600 Subject: [PATCH 04/16] complete remove_address fn --- .../src/service/service_effect_handler.rs | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index e4f6d3aeb..dc6f7b5a8 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -11,7 +11,9 @@ use clap::Parser; use emver::VersionRange; use imbl::OrdMap; use imbl_value::{json, InternedString}; -use models::{ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, VolumeId}; +use models::{ + ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, ServiceInterfaceId, VolumeId, +}; use patch_db::json_ptr::JsonPointer; use rpc_toolkit::{from_fn, from_fn_async, AnyContext, Context, Empty, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; @@ -245,7 +247,7 @@ struct ListServiceInterfacesParams { #[ts(export)] #[serde(rename_all = "camelCase")] struct RemoveAddressParams { - id: String, + id: ServiceInterfaceId, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] @@ -389,8 +391,24 @@ async fn list_service_interfaces( ) -> Result { todo!() } -async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result { - todo!() +async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result<(), Error> { + let context = context.deref()?; + let package_id = context.id.clone(); + + context + .ctx + .db + .mutate(|db| { + let model = db + .as_public_mut() + .as_package_data_mut() + .as_idx_mut(&package_id) + .or_not_found(&package_id)? + .as_service_interfaces_mut(); + model.remove(&data.id) + }) + .await?; + Ok(()) } async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> { let context = context.deref()?; From de38992409f21ab2e34f0fda612b9e3bc2fd1fd7 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Thu, 25 Apr 2024 16:46:39 -0600 Subject: [PATCH 05/16] get_system_smtp wip --- core/startos/src/db/model/public.rs | 1 + core/startos/src/service/service_effect_handler.rs | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index a51303cf4..b8f6d3d2a 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -134,6 +134,7 @@ pub struct ServerInfo { #[serde(default)] pub zram: bool, pub governor: Option, + pub smtp: String } #[derive(Debug, Deserialize, Serialize, HasModel, TS)] diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index dc6f7b5a8..6a0e009a0 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -317,8 +317,17 @@ struct MountParams { async fn get_system_smtp( context: EffectContext, data: GetSystemSmtpParams, -) -> Result { - todo!() +) -> Result { + let context = context.deref()?; + context + .ctx + .db + .peek() + .await + .into_public() + .into_server_info() + .into_smtp() + .de() } async fn get_container_ip(context: EffectContext, _: Empty) -> Result { let context = context.deref()?; From b5b2ca26512ac58a617786b2cc19986eb83f5c1a Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Mon, 29 Apr 2024 11:03:06 -0600 Subject: [PATCH 06/16] complete get_system_smtp and set_system_smtp --- core/startos/src/db/model/public.rs | 3 +- .../src/service/service_effect_handler.rs | 28 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/core/startos/src/db/model/public.rs b/core/startos/src/db/model/public.rs index b8f6d3d2a..e9557d625 100644 --- a/core/startos/src/db/model/public.rs +++ b/core/startos/src/db/model/public.rs @@ -75,6 +75,7 @@ impl Public { ntp_synced: false, zram: true, governor: None, + smtp: None, }, package_data: AllPackageData::default(), ui: serde_json::from_str(include_str!(concat!( @@ -134,7 +135,7 @@ pub struct ServerInfo { #[serde(default)] pub zram: bool, pub governor: Option, - pub smtp: String + pub smtp: Option } #[derive(Debug, Deserialize, Serialize, HasModel, TS)] diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 6a0e009a0..768d97503 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -314,12 +314,28 @@ struct MountParams { location: String, target: MountTarget, } +async fn set_system_smtp( + context: EffectContext, + smtp: String, +) -> Result<(), Error> { + let context = context.deref()?; + context + .ctx + .db + .mutate(|db| { + let model = db.as_public_mut() + .as_server_info_mut() + .as_smtp_mut(); + model.ser(&mut Some(smtp)) + }) + .await +} async fn get_system_smtp( context: EffectContext, data: GetSystemSmtpParams, ) -> Result { let context = context.deref()?; - context + let res = context .ctx .db .peek() @@ -327,7 +343,15 @@ async fn get_system_smtp( .into_public() .into_server_info() .into_smtp() - .de() + .de()?; + + match res { + Some(smtp) => Ok(smtp), + None => Err(Error::new( + eyre!("SMTP not found"), + crate::ErrorKind::NotFound, + )), + } } async fn get_container_ip(context: EffectContext, _: Empty) -> Result { let context = context.deref()?; From 6e62465c37432753d3dac06b5e751cdf4ed2fea4 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Mon, 29 Apr 2024 11:10:24 -0600 Subject: [PATCH 07/16] add SetSystemSmtpParams struct --- core/startos/src/service/service_effect_handler.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 768d97503..af5691958 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -187,6 +187,12 @@ struct GetSystemSmtpParams { #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] +struct SetSystemSmtpParams { + smtp: String, +} +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] +#[ts(export)] +#[serde(rename_all = "camelCase")] struct GetServicePortForwardParams { #[ts(type = "string | null")] package_id: Option, @@ -316,7 +322,7 @@ struct MountParams { } async fn set_system_smtp( context: EffectContext, - smtp: String, + data: SetSystemSmtpParams, ) -> Result<(), Error> { let context = context.deref()?; context @@ -326,7 +332,7 @@ async fn set_system_smtp( let model = db.as_public_mut() .as_server_info_mut() .as_smtp_mut(); - model.ser(&mut Some(smtp)) + model.ser(&mut Some(data.smtp)) }) .await } From 149b2048e02d69223b2cef1a5ef52d6f22f1e173 Mon Sep 17 00:00:00 2001 From: Shadowy Super Coder Date: Mon, 29 Apr 2024 11:11:34 -0600 Subject: [PATCH 08/16] add set_system_smtp subcommand --- core/startos/src/service/service_effect_handler.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index af5691958..bac744227 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -151,6 +151,7 @@ pub fn service_effect_handler() -> ParentHandler { .no_display() .with_remote_cli::(), ) + .subcommand("setSystemSmtp", from_fn_async(set_system_smtp).no_cli()) .subcommand("getSystemSmtp", from_fn_async(get_system_smtp).no_cli()) .subcommand("getContainerIp", from_fn_async(get_container_ip).no_cli()) .subcommand( From 97edc0d8420a4b3ee64fd8000c99d210ce6501ef Mon Sep 17 00:00:00 2001 From: Dominion5254 Date: Tue, 30 Apr 2024 11:39:25 -0600 Subject: [PATCH 09/16] Remove 'Copy' implementation from `HostAddress` Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> --- core/startos/src/net/host/address.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/startos/src/net/host/address.rs b/core/startos/src/net/host/address.rs index 75abf350e..d9e2f4206 100644 --- a/core/startos/src/net/host/address.rs +++ b/core/startos/src/net/host/address.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use torut::onion::OnionAddressV3; use ts_rs::TS; -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, TS)] #[serde(rename_all = "camelCase")] #[serde(tag = "kind")] #[ts(export)] From d35e3187da08eab5430512f5232f710c0086fcf0 Mon Sep 17 00:00:00 2001 From: Dominion5254 Date: Tue, 30 Apr 2024 11:44:40 -0600 Subject: [PATCH 10/16] Refactor `get_host_primary` fn and clone resulting `HostAddress` Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> --- core/startos/src/net/host/mod.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index 3066e43cc..1948cc9e7 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -89,14 +89,6 @@ impl Model { impl HostInfo { pub fn get_host_primary(&self, host_id: &HostId) -> Option { - match self.0.get(&host_id) { - Some(h) => { - match h.primary { - Some(ha) => Some(ha), - None => None, - } - } - None => None, - } + self.0.get(&host_id).and_then(|h| h.primary.clone()) } } From 1ff61e9c985628192fb26c65297e72704322ddb3 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 6 May 2024 15:46:56 -0600 Subject: [PATCH 11/16] misc fixes and debug info --- build/dpkg-deps/depends | 2 ++ build/lib/scripts/add-apt-sources | 3 --- build/lib/scripts/chroot-and-upgrade | 2 ++ core/models/src/id/mod.rs | 7 ++++++- core/startos/src/net/net_controller.rs | 5 ++++- core/startos/src/service/service_effect_handler.rs | 2 +- core/startos/startd.service | 2 ++ debian/postinst | 4 ---- image-recipe/build.sh | 6 ------ sdk/lib/osBindings/ExportServiceInterfaceParams.ts | 7 ++++++- sdk/lib/osBindings/GetPrimaryUrlParams.ts | 2 ++ sdk/lib/osBindings/RemoveAddressParams.ts | 3 ++- sdk/lib/osBindings/ServerInfo.ts | 1 + sdk/lib/osBindings/SetSystemSmtpParams.ts | 3 +++ sdk/lib/osBindings/index.ts | 1 + web/projects/ui/src/app/services/api/mock-patch.ts | 1 + 16 files changed, 33 insertions(+), 18 deletions(-) create mode 100644 sdk/lib/osBindings/SetSystemSmtpParams.ts diff --git a/build/dpkg-deps/depends b/build/dpkg-deps/depends index 38d9b1a58..7519cff1b 100644 --- a/build/dpkg-deps/depends +++ b/build/dpkg-deps/depends @@ -18,6 +18,7 @@ grub-common htop httpdirfs iotop +iptables iw jq libyajl2 @@ -34,6 +35,7 @@ network-manager nvme-cli nyx openssh-server +podman postgresql psmisc qemu-guest-agent diff --git a/build/lib/scripts/add-apt-sources b/build/lib/scripts/add-apt-sources index 638d8dad6..9d4f54a28 100755 --- a/build/lib/scripts/add-apt-sources +++ b/build/lib/scripts/add-apt-sources @@ -4,6 +4,3 @@ set -e curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor -o- > /usr/share/keyrings/tor-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org bullseye main" > /etc/apt/sources.list.d/tor.list - -curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o- > /usr/share/keyrings/docker-archive-keyring.gpg -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" > /etc/apt/sources.list.d/docker.list diff --git a/build/lib/scripts/chroot-and-upgrade b/build/lib/scripts/chroot-and-upgrade index ef3208806..7adaaaccb 100755 --- a/build/lib/scripts/chroot-and-upgrade +++ b/build/lib/scripts/chroot-and-upgrade @@ -33,6 +33,7 @@ set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters if [ -z "$NO_SYNC" ]; then echo 'Syncing...' umount -R /media/startos/next 2> /dev/null + umount -R /media/startos/upper 2> /dev/null rm -rf /media/startos/upper /media/startos/next mkdir /media/startos/upper mount -t tmpfs tmpfs /media/startos/upper @@ -105,4 +106,5 @@ if [ "$CHROOT_RES" -eq 0 ]; then fi umount -R /media/startos/next +umount -R /media/startos/upper rm -rf /media/startos/upper /media/startos/next \ No newline at end of file diff --git a/core/models/src/id/mod.rs b/core/models/src/id/mod.rs index 11644c71d..10882d4f1 100644 --- a/core/models/src/id/mod.rs +++ b/core/models/src/id/mod.rs @@ -28,8 +28,13 @@ lazy_static::lazy_static! { pub static ref SYSTEM_ID: Id = Id(InternedString::intern("x_system")); } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] pub struct Id(InternedString); +impl std::fmt::Debug for Id { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} impl TryFrom for Id { type Error = InvalidId; fn try_from(value: InternedString) -> Result { diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index aae98ae96..c77b5d83b 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -162,7 +162,7 @@ impl NetController { } } -#[derive(Default)] +#[derive(Default, Debug)] struct HostBinds { lan: BTreeMap, Arc<()>)>, tor: BTreeMap, Vec>)>, @@ -193,6 +193,7 @@ impl NetService { internal_port: u16, options: BindOptions, ) -> Result<(), Error> { + dbg!("bind", &kind, &id, internal_port, &options); let id_ref = &id; let pkg_id = &self.id; let host = self @@ -219,6 +220,8 @@ impl NetService { } async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> { + dbg!(&host); + dbg!(&self.binds); let ctrl = self.net_controller()?; let binds = { if !self.binds.contains_key(&id) { diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 7965597e3..b270849ec 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -694,7 +694,7 @@ fn chroot( cmd.env(k, v); } } - nix::unistd::setsid().with_kind(ErrorKind::Lxc)?; // TODO: error code + nix::unistd::setsid().ok(); // https://stackoverflow.com/questions/25701333/os-setsid-operation-not-permitted std::os::unix::fs::chroot(path)?; if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { cmd.uid(uid); diff --git a/core/startos/startd.service b/core/startos/startd.service index 56cf92e22..8a45b764d 100644 --- a/core/startos/startd.service +++ b/core/startos/startd.service @@ -1,5 +1,7 @@ [Unit] Description=StartOS Daemon +After=lxc-net.service +Wants=lxc-net.service [Service] Type=simple diff --git a/debian/postinst b/debian/postinst index 4e8350376..238bd9457 100755 --- a/debian/postinst +++ b/debian/postinst @@ -81,10 +81,6 @@ sed -i '/^\s*#\?\s*issue_discards\s*=\s*/c\issue_discards = 1' /etc/lvm/lvm.conf mkdir -p /etc/nginx/ssl -# fix to suppress docker warning, fixed in 21.xx release of docker cli: https://github.com/docker/cli/pull/2934 -mkdir -p /root/.docker -touch /root/.docker/config.json - cat << EOF > /etc/tor/torrc SocksPort 0.0.0.0:9050 SocksPolicy accept 127.0.0.1 diff --git a/image-recipe/build.sh b/image-recipe/build.sh index 9f62000ce..5ec500ce3 100755 --- a/image-recipe/build.sh +++ b/image-recipe/build.sh @@ -166,12 +166,6 @@ fi curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc > config/archives/tor.key echo "deb [arch=${IB_TARGET_ARCH} signed-by=/etc/apt/trusted.gpg.d/tor.key.gpg] https://deb.torproject.org/torproject.org ${IB_SUITE} main" > config/archives/tor.list -curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o config/archives/docker.key -echo "deb [arch=${IB_TARGET_ARCH} signed-by=/etc/apt/trusted.gpg.d/docker.key.gpg] https://download.docker.com/linux/debian ${IB_SUITE} stable" > config/archives/docker.list - -curl -fsSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/Debian_Testing/Release.key | gpg --dearmor -o config/archives/podman.key -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/podman.key.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/unstable/Debian_Testing/ /" > config/archives/podman.list - # Dependencies ## Base dependencies diff --git a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts index 152a800bb..847a6090a 100644 --- a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts +++ b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts @@ -1,9 +1,12 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AddressInfo } from "./AddressInfo" +import type { ExportedHostnameInfo } from "./ExportedHostnameInfo" +import type { HostKind } from "./HostKind" +import type { ServiceInterfaceId } from "./ServiceInterfaceId" import type { ServiceInterfaceType } from "./ServiceInterfaceType" export type ExportServiceInterfaceParams = { - id: string + id: ServiceInterfaceId name: string description: string hasPrimary: boolean @@ -11,4 +14,6 @@ export type ExportServiceInterfaceParams = { masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType + hostKind: HostKind + hostnames: Array } diff --git a/sdk/lib/osBindings/GetPrimaryUrlParams.ts b/sdk/lib/osBindings/GetPrimaryUrlParams.ts index d9394f4ce..1a68ecc7b 100644 --- a/sdk/lib/osBindings/GetPrimaryUrlParams.ts +++ b/sdk/lib/osBindings/GetPrimaryUrlParams.ts @@ -1,8 +1,10 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Callback } from "./Callback" +import type { HostId } from "./HostId" export type GetPrimaryUrlParams = { packageId: string | null serviceInterfaceId: string callback: Callback + hostId: HostId } diff --git a/sdk/lib/osBindings/RemoveAddressParams.ts b/sdk/lib/osBindings/RemoveAddressParams.ts index bdc781837..14099ebbc 100644 --- a/sdk/lib/osBindings/RemoveAddressParams.ts +++ b/sdk/lib/osBindings/RemoveAddressParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ServiceInterfaceId } from "./ServiceInterfaceId" -export type RemoveAddressParams = { id: string } +export type RemoveAddressParams = { id: ServiceInterfaceId } diff --git a/sdk/lib/osBindings/ServerInfo.ts b/sdk/lib/osBindings/ServerInfo.ts index 284eba336..935e3a99f 100644 --- a/sdk/lib/osBindings/ServerInfo.ts +++ b/sdk/lib/osBindings/ServerInfo.ts @@ -28,4 +28,5 @@ export type ServerInfo = { ntpSynced: boolean zram: boolean governor: Governor | null + smtp: string | null } diff --git a/sdk/lib/osBindings/SetSystemSmtpParams.ts b/sdk/lib/osBindings/SetSystemSmtpParams.ts new file mode 100644 index 000000000..49c66e86c --- /dev/null +++ b/sdk/lib/osBindings/SetSystemSmtpParams.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetSystemSmtpParams = { smtp: string } diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index d14a9fb87..4fcb9a617 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -113,6 +113,7 @@ export { SetDependenciesParams } from "./SetDependenciesParams" export { SetHealth } from "./SetHealth" export { SetMainStatus } from "./SetMainStatus" export { SetStoreParams } from "./SetStoreParams" +export { SetSystemSmtpParams } from "./SetSystemSmtpParams" export { SignAssetParams } from "./SignAssetParams" export { SignatureInfo } from "./SignatureInfo" export { Signature } from "./Signature" diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index 9f17c22eb..a4ed61278 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -74,6 +74,7 @@ export const mockPatchData: DataModel = { platform: 'x86_64-nonfree', zram: true, governor: 'performance', + smtp: 'todo', wifi: { interface: 'wlan0', ssids: [], From cea24e96c2a07e2530c0b74a15942caec19eeda6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 6 May 2024 18:18:41 -0600 Subject: [PATCH 12/16] seed hosts with a tor address --- Makefile | 2 +- .../DockerProcedureContainer.ts | 2 +- .../Systems/SystemForEmbassy/MainLoop.ts | 2 +- .../SystemForEmbassy/polyfillEffects.ts | 8 +- core/startos/src/db/prelude.rs | 4 +- core/startos/src/init.rs | 6 ++ core/startos/src/net/host/mod.rs | 73 +++++++++++++++---- core/startos/src/net/net_controller.rs | 23 ++---- core/startos/src/registry/os/asset/add.rs | 10 ++- core/startos/src/registry/os/version/mod.rs | 2 +- .../s9pk/merkle_archive/directory_contents.rs | 4 +- core/startos/src/service/cli.rs | 2 +- .../src/service/service_effect_handler.rs | 4 +- core/startos/startd.service | 2 - sdk/lib/StartSdk.ts | 15 ++-- sdk/lib/health/HealthCheck.ts | 17 +++-- sdk/lib/mainFn/Daemons.ts | 10 +-- sdk/lib/util/Overlay.ts | 13 +++- 18 files changed, 128 insertions(+), 71 deletions(-) diff --git a/Makefile b/Makefile index dbe10d677..1dffd3629 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ container-runtime/node_modules: container-runtime/package.json container-runtime npm --prefix container-runtime ci touch container-runtime/node_modules -sdk/lib/osBindings: core/startos/bindings +sdk/lib/osBindings: $(shell core/startos/bindings) mkdir -p sdk/lib/osBindings ls core/startos/bindings/*.ts | sed 's/core\/startos\/bindings\/\([^.]*\)\.ts/export { \1 } from ".\/\1";/g' > core/startos/bindings/index.ts npm --prefix sdk exec -- prettier --config ./sdk/package.json -w ./core/startos/bindings/*.ts diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index b4a5f5829..c06395a17 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -17,7 +17,7 @@ export class DockerProcedureContainer { data: DockerProcedure, volumes: { [id: VolumeId]: Volume }, ) { - const overlay = await Overlay.of(effects, data.image) + const overlay = await Overlay.of(effects, { id: data.image }) if (data.mounts) { const mounts = data.mounts diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 7e18c69f1..23be7071f 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -53,7 +53,7 @@ export class MainLoop { } const daemon = await daemons.runDaemon()( this.effects, - this.system.manifest.main.image, + { id: this.system.manifest.main.image }, currentCommand, { overlay: dockerProcedureContainer.overlay, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 66a303e7b..30e572d97 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -96,7 +96,7 @@ export class PolyfillEffects implements oet.Effects { return startSdk .runCommand( this.effects, - this.manifest.main.image, + { id: this.manifest.main.image }, [command, ...(args || [])], {}, ) @@ -118,7 +118,7 @@ export class PolyfillEffects implements oet.Effects { const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => daemons.runDaemon()( this.effects, - this.manifest.main.image, + { id: this.manifest.main.image }, [input.command, ...(input.args || [])], { overlay: dockerProcedureContainer.overlay, @@ -143,7 +143,7 @@ export class PolyfillEffects implements oet.Effects { await startSdk .runCommand( this.effects, - this.manifest.main.image, + { id: this.manifest.main.image }, ["chown", "--recursive", input.uid, `/drive/${input.path}`], { mounts: [ @@ -178,7 +178,7 @@ export class PolyfillEffects implements oet.Effects { await startSdk .runCommand( this.effects, - this.manifest.main.image, + { id: this.manifest.main.image }, ["chmod", "--recursive", input.mode, `/drive/${input.path}`], { mounts: [ diff --git a/core/startos/src/db/prelude.rs b/core/startos/src/db/prelude.rs index d40c3c17c..419b356ef 100644 --- a/core/startos/src/db/prelude.rs +++ b/core/startos/src/db/prelude.rs @@ -220,7 +220,7 @@ where } pub fn upsert(&mut self, key: &T::Key, value: F) -> Result<&mut Model, Error> where - F: FnOnce() -> T::Value, + F: FnOnce() -> Result, { use serde::ser::Error; match &mut self.value { @@ -233,7 +233,7 @@ where s.as_ref().index_or_insert(v) }); if !exists { - res.ser(&value())?; + res.ser(&value()?)?; } Ok(res) } diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 361288c10..694d4e3a3 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -348,6 +348,12 @@ pub async fn init(cfg: &ServerConfig) -> Result { }) .await?; + Command::new("systemctl") + .arg("start") + .arg("lxc-net.service") + .invoke(ErrorKind::Lxc) + .await?; + crate::version::init(&db).await?; db.mutate(|d| { diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index 1948cc9e7..2d8599ba9 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -1,10 +1,12 @@ use std::collections::{BTreeMap, BTreeSet}; use imbl_value::InternedString; -use models::HostId; +use models::{HostId, PackageId}; use serde::{Deserialize, Serialize}; +use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use ts_rs::TS; +use crate::db::model::DatabaseModel; use crate::net::forward::AvailablePorts; use crate::net::host::address::HostAddress; use crate::net::host::binding::{BindInfo, BindOptions}; @@ -64,26 +66,67 @@ impl Map for HostInfo { } } -impl Model { +pub fn host_for<'a>( + db: &'a mut DatabaseModel, + package_id: &PackageId, + host_id: &HostId, + host_kind: HostKind, +) -> Result<&'a mut Model, Error> { + fn host_info<'a>( + db: &'a mut DatabaseModel, + package_id: &PackageId, + ) -> Result<&'a mut Model, Error> { + Ok::<_, Error>( + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(package_id) + .or_not_found(package_id)? + .as_hosts_mut(), + ) + } + let tor_key = if host_info(db, package_id)?.as_idx(host_id).is_none() { + Some( + db.as_private_mut() + .as_key_store_mut() + .as_onion_mut() + .new_key()?, + ) + } else { + None + }; + host_info(db, package_id)?.upsert(host_id, || { + let mut h = Host::new(host_kind); + h.addresses.insert(HostAddress::Onion { + address: tor_key + .or_not_found("generated tor key")? + .public() + .get_onion_address(), + }); + Ok(h) + }) +} + +impl Model { + pub fn set_kind(&mut self, kind: HostKind) -> Result<(), Error> { + match (self.as_kind().de()?, kind) { + (HostKind::Multi, HostKind::Multi) => Ok(()), + } + } pub fn add_binding( &mut self, available_ports: &mut AvailablePorts, - kind: HostKind, - id: &HostId, internal_port: u16, options: BindOptions, ) -> Result<(), Error> { - self.upsert(id, || Host::new(kind))? - .as_bindings_mut() - .mutate(|b| { - let info = if let Some(info) = b.remove(&internal_port) { - info.update(available_ports, options)? - } else { - BindInfo::new(available_ports, options)? - }; - b.insert(internal_port, info); - Ok(()) - }) // TODO: handle host kind change + self.as_bindings_mut().mutate(|b| { + let info = if let Some(info) = b.remove(&internal_port) { + info.update(available_ports, options)? + } else { + BindInfo::new(available_ports, options)? + }; + b.insert(internal_port, info); + Ok(()) + }) } } diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index c77b5d83b..205cc4e36 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -16,7 +16,7 @@ use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; use crate::net::host::binding::{AddSslOptions, BindOptions}; -use crate::net::host::{Host, HostKind}; +use crate::net::host::{host_for, Host, HostKind}; use crate::net::tor::TorController; use crate::net::vhost::{AlpnInfo, VHostController}; use crate::prelude::*; @@ -194,25 +194,16 @@ impl NetService { options: BindOptions, ) -> Result<(), Error> { dbg!("bind", &kind, &id, internal_port, &options); - let id_ref = &id; let pkg_id = &self.id; let host = self .net_controller()? .db - .mutate(|d| { - let mut ports = d.as_private().as_available_ports().de()?; - let hosts = d - .as_public_mut() - .as_package_data_mut() - .as_idx_mut(pkg_id) - .or_not_found(pkg_id)? - .as_hosts_mut(); - hosts.add_binding(&mut ports, kind, &id, internal_port, options)?; - let host = hosts - .as_idx(&id) - .or_not_found(lazy_format!("Host {id_ref} for {pkg_id}"))? - .de()?; - d.as_private_mut().as_available_ports_mut().ser(&ports)?; + .mutate(|db| { + let mut ports = db.as_private().as_available_ports().de()?; + let host = host_for(db, pkg_id, &id, kind)?; + host.add_binding(&mut ports, internal_port, options)?; + let host = host.de()?; + db.as_private_mut().as_available_ports_mut().ser(&ports)?; Ok(host) }) .await?; diff --git a/core/startos/src/registry/os/asset/add.rs b/core/startos/src/registry/os/asset/add.rs index 6e259e314..d2c20e711 100644 --- a/core/startos/src/registry/os/asset/add.rs +++ b/core/startos/src/registry/os/asset/add.rs @@ -9,7 +9,7 @@ use futures::{FutureExt, TryStreamExt}; use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; -use rpc_toolkit::{from_fn_async, CallRemote, Context, HandlerArgs, HandlerExt, ParentHandler}; +use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha512}; use ts_rs::TS; @@ -106,9 +106,11 @@ async fn add_asset( .as_idx_mut(&version) .or_not_found(&version)?, ) - .upsert(&platform, || RegistryAsset { - url, - signature_info: SignatureInfo::new(SIG_CONTEXT), + .upsert(&platform, || { + Ok(RegistryAsset { + url, + signature_info: SignatureInfo::new(SIG_CONTEXT), + }) })? .as_signature_info_mut() .mutate(|s| s.add_sig(&signature))?; diff --git a/core/startos/src/registry/os/version/mod.rs b/core/startos/src/registry/os/version/mod.rs index 414994875..5af407c5c 100644 --- a/core/startos/src/registry/os/version/mod.rs +++ b/core/startos/src/registry/os/version/mod.rs @@ -81,7 +81,7 @@ pub async fn add_version( db.as_index_mut() .as_os_mut() .as_versions_mut() - .upsert(&version, || OsVersionInfo::default())? + .upsert(&version, || Ok(OsVersionInfo::default()))? .mutate(|i| { i.headline = headline; i.release_notes = release_notes; diff --git a/core/startos/src/s9pk/merkle_archive/directory_contents.rs b/core/startos/src/s9pk/merkle_archive/directory_contents.rs index 19821cf32..ebc46a60b 100644 --- a/core/startos/src/s9pk/merkle_archive/directory_contents.rs +++ b/core/startos/src/s9pk/merkle_archive/directory_contents.rs @@ -275,9 +275,7 @@ impl DirectoryContents { _ => std::cmp::Ordering::Equal, }) { varint::serialize_varstring(&**name, w).await?; - if let Some(pos) = entry.serialize_header(queue.add(entry).await?, w).await? { - eprintln!("DEBUG ====> {name} @ {pos}"); - } + entry.serialize_header(queue.add(entry).await?, w).await?; } Ok(()) diff --git a/core/startos/src/service/cli.rs b/core/startos/src/service/cli.rs index 82c63d6a7..87491932b 100644 --- a/core/startos/src/service/cli.rs +++ b/core/startos/src/service/cli.rs @@ -29,7 +29,7 @@ impl ContainerCliContext { Self(Arc::new(ContainerCliSeed { socket: cfg .socket - .unwrap_or_else(|| Path::new("/").join(HOST_RPC_SERVER_SOCKET)), + .unwrap_or_else(|| Path::new("/media/startos/rpc").join(HOST_RPC_SERVER_SOCKET)), runtime: OnceCell::new(), })) } diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index ebcaafde2..0bd31dc9b 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -822,7 +822,7 @@ async fn set_store( let model = db .as_private_mut() .as_package_stores_mut() - .upsert(&package_id, || json!({}))?; + .upsert(&package_id, || Ok(json!({})))?; let mut model_value = model.de()?; if model_value.is_null() { model_value = json!({}); @@ -1140,7 +1140,7 @@ pub async fn create_overlayed_image( .s9pk .as_archive() .contents() - .get_path(dbg!(&path)) + .get_path(&path) .and_then(|e| e.as_file()) { let guid = new_guid(); diff --git a/core/startos/startd.service b/core/startos/startd.service index 8a45b764d..56cf92e22 100644 --- a/core/startos/startd.service +++ b/core/startos/startd.service @@ -1,7 +1,5 @@ [Unit] Description=StartOS Daemon -After=lxc-net.service -Wants=lxc-net.service [Service] Type=simple diff --git a/sdk/lib/StartSdk.ts b/sdk/lib/StartSdk.ts index 4d7514d88..f2eabe6be 100644 --- a/sdk/lib/StartSdk.ts +++ b/sdk/lib/StartSdk.ts @@ -28,7 +28,7 @@ import { DependencyConfig, Update } from "./dependencies/DependencyConfig" import { BackupSet, Backups } from "./backup/Backups" import { smtpConfig } from "./config/configConstants" import { Daemons } from "./mainFn/Daemons" -import { healthCheck } from "./health/HealthCheck" +import { healthCheck, HealthCheckParams } from "./health/HealthCheck" import { checkPortListening } from "./health/checkFns/checkPortListening" import { checkWebUrl, runHealthScript } from "./health/checkFns" import { List } from "./config/builder/list" @@ -78,6 +78,7 @@ import { Checker, EmVer } from "./emverLite/mod" import { ExposedStorePaths } from "./store/setupExposeStore" import { PathBuilder, extractJsonPath, pathBuilder } from "./store/PathBuilder" import { checkAllDependencies } from "./dependencies/dependencies" +import { health } from "." // prettier-ignore type AnyNeverCond = @@ -186,13 +187,13 @@ export class StartSdk { nullIfEmpty, runCommand: async ( effects: Effects, - imageId: Manifest["images"][number], + image: { id: Manifest["images"][number]; sharedRun?: boolean }, command: ValidIfNoStupidEscape | [string, ...string[]], options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] }, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => { - return runCommand(effects, imageId, command, options) + return runCommand(effects, image, command, options) }, createAction: < @@ -264,7 +265,9 @@ export class StartSdk { ) }, HealthCheck: { - of: healthCheck, + of(o: HealthCheckParams) { + return healthCheck(o) + }, }, Dependency: { of(data: Dependency["data"]) { @@ -740,14 +743,14 @@ export class StartSdk { export async function runCommand( effects: Effects, - imageId: Manifest["images"][number], + image: { id: Manifest["images"][number]; sharedRun?: boolean }, command: string | [string, ...string[]], options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] }, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> { const commands = splitCommand(command) - const overlay = await Overlay.of(effects, imageId) + const overlay = await Overlay.of(effects, image) try { for (let mount of options.mounts || []) { await overlay.mount(mount.options, mount.path) diff --git a/sdk/lib/health/HealthCheck.ts b/sdk/lib/health/HealthCheck.ts index d7bbc2f44..ca0e3ebb9 100644 --- a/sdk/lib/health/HealthCheck.ts +++ b/sdk/lib/health/HealthCheck.ts @@ -1,5 +1,5 @@ import { InterfaceReceipt } from "../interfaces/interfaceReceipt" -import { Daemon, Effects } from "../types" +import { Daemon, Effects, SDKManifest } from "../types" import { CheckResult } from "./checkFns/CheckResult" import { HealthReceipt } from "./HealthReceipt" import { Trigger } from "../trigger" @@ -9,16 +9,23 @@ import { once } from "../util/once" import { Overlay } from "../util/Overlay" import { object, unknown } from "ts-matches" -export function healthCheck(o: { +export type HealthCheckParams = { effects: Effects name: string - imageId: string + image: { + id: Manifest["images"][number] + sharedRun?: boolean + } trigger?: Trigger fn(overlay: Overlay): Promise | CheckResult onFirstSuccess?: () => unknown | Promise -}) { +} + +export function healthCheck( + o: HealthCheckParams, +) { new Promise(async () => { - const overlay = await Overlay.of(o.effects, o.imageId) + const overlay = await Overlay.of(o.effects, o.image) try { let currentValue: TriggerInput = { hadSuccess: false, diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index 01fd7f2c7..efa1a2484 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -23,7 +23,7 @@ type Daemon< > = { id: "" extends Id ? never : Id command: ValidIfNoStupidEscape | [string, ...string[]] - imageId: Manifest["images"][number] + image: { id: Manifest["images"][number]; sharedRun?: boolean } mounts: Mounts env?: Record ready: { @@ -40,7 +40,7 @@ export const runDaemon = () => async ( effects: Effects, - imageId: Manifest["images"][number], + image: { id: Manifest["images"][number]; sharedRun?: boolean }, command: ValidIfNoStupidEscape | [string, ...string[]], options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] @@ -48,7 +48,7 @@ export const runDaemon = }, ): Promise => { const commands = splitCommand(command) - const overlay = options.overlay || (await Overlay.of(effects, imageId)) + const overlay = options.overlay || (await Overlay.of(effects, image)) for (let mount of options.mounts || []) { await overlay.mount(mount.options, mount.path) } @@ -183,9 +183,9 @@ export class Daemons { daemon.requires?.map((id) => daemonsStarted[id]) ?? [], ) daemonsStarted[daemon.id] = requiredPromise.then(async () => { - const { command, imageId } = daemon + const { command, image } = daemon - const child = runDaemon()(effects, imageId, command, { + const child = runDaemon()(effects, image, command, { env: daemon.env, mounts: daemon.mounts.build(), }) diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/Overlay.ts index 0e2dec58e..794b78732 100644 --- a/sdk/lib/util/Overlay.ts +++ b/sdk/lib/util/Overlay.ts @@ -12,10 +12,19 @@ export class Overlay { readonly rootfs: string, readonly guid: string, ) {} - static async of(effects: T.Effects, imageId: string) { + static async of( + effects: T.Effects, + image: { id: string; sharedRun?: boolean }, + ) { + const { id: imageId, sharedRun } = image const [rootfs, guid] = await effects.createOverlayedImage({ imageId }) - for (const dirPart of ["dev", "sys", "proc", "run"] as const) { + const shared = ["dev", "sys", "proc"] + if (!!sharedRun) { + shared.push("run") + } + + for (const dirPart of shared) { await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true }) await execFile("mount", [ "--rbind", From 3c51be7afcec2fffabf5686a78c49fa470df4498 Mon Sep 17 00:00:00 2001 From: J H Date: Wed, 8 May 2024 15:58:44 -0600 Subject: [PATCH 13/16] fix: Making the daemons keep up the status. --- .../Systems/SystemForEmbassy/MainLoop.ts | 15 +- .../SystemForEmbassy/polyfillEffects.ts | 2 +- container-runtime/src/Models/Effects.ts | 4 +- .../src/service/service_effect_handler.rs | 15 +- sdk/lib/mainFn/CommandController.ts | 103 +++++++++ sdk/lib/mainFn/Daemon.ts | 70 ++++++ sdk/lib/mainFn/Daemons.ts | 211 ++++++------------ sdk/lib/mainFn/HealthDaemon.ts | 152 +++++++++++++ sdk/lib/osBindings/SetMainStatus.ts | 2 +- sdk/lib/test/startosTypeValidation.test.ts | 7 +- sdk/lib/types.ts | 5 +- sdk/package.json | 2 +- 12 files changed, 424 insertions(+), 164 deletions(-) create mode 100644 sdk/lib/mainFn/CommandController.ts create mode 100644 sdk/lib/mainFn/Daemon.ts create mode 100644 sdk/lib/mainFn/HealthDaemon.ts diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 23be7071f..c89d6064a 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -3,6 +3,7 @@ import { DockerProcedureContainer } from "./DockerProcedureContainer" import { SystemForEmbassy } from "." import { HostSystemStartOs } from "../../HostSystemStartOs" import { Daemons, T, daemons } from "@start9labs/start-sdk" +import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon" const EMBASSY_HEALTH_INTERVAL = 15 * 1000 const EMBASSY_PROPERTIES_LOOP = 30 * 1000 @@ -21,8 +22,7 @@ export class MainLoop { private mainEvent: | Promise<{ - daemon: T.DaemonReturned - wait: Promise + daemon: Daemon }> | undefined constructor( @@ -51,7 +51,7 @@ export class MainLoop { if (jsMain) { throw new Error("Unreachable") } - const daemon = await daemons.runDaemon()( + const daemon = await Daemon.of()( this.effects, { id: this.system.manifest.main.image }, currentCommand, @@ -59,14 +59,9 @@ export class MainLoop { overlay: dockerProcedureContainer.overlay, }, ) + daemon.start() return { daemon, - wait: daemon.wait().finally(() => { - this.clean() - effects - .setMainStatus({ status: "stopped" }) - .catch((e) => console.error("Could not set the status to stopped")) - }), } } @@ -121,7 +116,7 @@ export class MainLoop { const main = await mainEvent delete this.mainEvent delete this.healthLoops - if (mainEvent) await main?.daemon.term() + if (mainEvent) await main?.daemon.stop() if (healthLoops) healthLoops.forEach((x) => clearInterval(x.interval)) } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 30e572d97..12c0a58d5 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -116,7 +116,7 @@ export class PolyfillEffects implements oet.Effects { this.manifest.volumes, ) const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => - daemons.runDaemon()( + daemons.runCommand()( this.effects, { id: this.manifest.main.image }, [input.command, ...(input.args || [])], diff --git a/container-runtime/src/Models/Effects.ts b/container-runtime/src/Models/Effects.ts index 757d51238..bacf8894a 100644 --- a/container-runtime/src/Models/Effects.ts +++ b/container-runtime/src/Models/Effects.ts @@ -1,5 +1,3 @@ import { types as T } from "@start9labs/start-sdk" -export type Effects = T.Effects & { - setMainStatus(o: { status: "running" | "stopped" }): Promise -} +export type Effects = T.Effects diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 0bd31dc9b..95f212a6d 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -1011,21 +1011,23 @@ async fn set_configured(context: EffectContext, params: SetConfigured) -> Result #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] -enum Status { +enum SetMainStatusStatus { Running, Stopped, + Starting, } -impl FromStr for Status { +impl FromStr for SetMainStatusStatus { type Err = color_eyre::eyre::Report; fn from_str(s: &str) -> Result { match s { "running" => Ok(Self::Running), "stopped" => Ok(Self::Stopped), + "starting" => Ok(Self::Starting), _ => Err(eyre!("unknown status {s}")), } } } -impl ValueParserFactory for Status { +impl ValueParserFactory for SetMainStatusStatus { type Parser = FromStrParser; fn value_parser() -> Self::Parser { FromStrParser::new() @@ -1037,14 +1039,15 @@ impl ValueParserFactory for Status { #[command(rename_all = "camelCase")] #[ts(export)] struct SetMainStatus { - status: Status, + status: SetMainStatusStatus, } async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Result { dbg!(format!("Status for main will be is {params:?}")); let context = context.deref()?; match params.status { - Status::Running => context.started(), - Status::Stopped => context.stopped(), + SetMainStatusStatus::Running => context.started(), + SetMainStatusStatus::Stopped => context.stopped(), + SetMainStatusStatus::Starting => context.stopped(), } Ok(Value::Null) } diff --git a/sdk/lib/mainFn/CommandController.ts b/sdk/lib/mainFn/CommandController.ts new file mode 100644 index 000000000..b894600a2 --- /dev/null +++ b/sdk/lib/mainFn/CommandController.ts @@ -0,0 +1,103 @@ +import { NO_TIMEOUT, SIGTERM } from "../StartSdk" +import { SDKManifest } from "../manifest/ManifestTypes" +import { Effects, ValidIfNoStupidEscape } from "../types" +import { MountOptions, Overlay } from "../util/Overlay" +import { splitCommand } from "../util/splitCommand" +import { cpExecFile } from "./Daemons" + +export class CommandController { + private constructor( + readonly runningAnswer: Promise, + readonly overlay: Overlay, + readonly pid: number | undefined, + ) {} + static of() { + return async ( + effects: Effects, + imageId: { + id: Manifest["images"][number] + sharedRun?: boolean + }, + command: ValidIfNoStupidEscape | [string, ...string[]], + options: { + mounts?: { path: string; options: MountOptions }[] + overlay?: Overlay + env?: + | { + [variable: string]: string + } + | undefined + cwd?: string | undefined + user?: string | undefined + onStdout?: (x: Buffer) => void + onStderr?: (x: Buffer) => void + }, + ) => { + const commands = splitCommand(command) + const overlay = options.overlay || (await Overlay.of(effects, imageId)) + for (let mount of options.mounts || []) { + await overlay.mount(mount.options, mount.path) + } + const childProcess = await overlay.spawn(commands, { + env: options.env, + }) + const answer = new Promise((resolve, reject) => { + childProcess.stdout.on( + "data", + options.onStdout ?? + ((data: any) => { + console.log(data.toString()) + }), + ) + childProcess.stderr.on( + "data", + options.onStderr ?? + ((data: any) => { + console.error(data.toString()) + }), + ) + + childProcess.on("exit", (code: any) => { + if (code === 0) { + return resolve(null) + } + return reject(new Error(`${commands[0]} exited with code ${code}`)) + }) + }) + + const pid = childProcess.pid + + return new CommandController(answer, overlay, pid) + } + } + async wait() { + try { + return await this.runningAnswer + } finally { + await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch((_) => {}) + } + } + async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) { + try { + await cpExecFile("pkill", [`-${signal}`, "-s", String(this.pid)]) + + if (timeout > NO_TIMEOUT) { + const didTimeout = await Promise.race([ + new Promise((resolve) => setTimeout(resolve, timeout)).then( + () => true, + ), + this.runningAnswer.then(() => false), + ]) + if (didTimeout) { + await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch( + (_: any) => {}, + ) + } + } else { + await this.runningAnswer + } + } finally { + await this.overlay.destroy() + } + } +} diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts new file mode 100644 index 000000000..7cbb422b9 --- /dev/null +++ b/sdk/lib/mainFn/Daemon.ts @@ -0,0 +1,70 @@ +import { SDKManifest } from "../manifest/ManifestTypes" +import { Effects, ValidIfNoStupidEscape } from "../types" +import { MountOptions, Overlay } from "../util/Overlay" +import { CommandController } from "./CommandController" + +/** + * This is a wrapper around CommandController that has a state of off, where the command shouldn't be running + * and the others state of running, where it will keep a living running command + */ + +export class Daemon { + private commandController: CommandController | null = null + private shouldBeRunning = false + private constructor(private startCommand: () => Promise) {} + static of() { + return async ( + effects: Effects, + imageId: { + id: Manifest["images"][number] + sharedRun?: boolean + }, + command: ValidIfNoStupidEscape | [string, ...string[]], + options: { + mounts?: { path: string; options: MountOptions }[] + overlay?: Overlay + env?: + | { + [variable: string]: string + } + | undefined + cwd?: string | undefined + user?: string | undefined + onStdout?: (x: Buffer) => void + onStderr?: (x: Buffer) => void + }, + ) => { + const startCommand = () => + CommandController.of()(effects, imageId, command, options) + return new Daemon(startCommand) + } + } + + async start() { + if (this.commandController) { + return + } + this.shouldBeRunning = true + new Promise(async () => { + while (this.shouldBeRunning) { + this.commandController = await this.startCommand() + await this.commandController.wait() + } + }).catch((err) => { + console.error(err) + }) + } + async term(termOptions?: { + signal?: NodeJS.Signals | undefined + timeout?: number | undefined + }) { + return this.stop(termOptions) + } + async stop(termOptions?: { + signal?: NodeJS.Signals | undefined + timeout?: number | undefined + }) { + this.shouldBeRunning = false + await this.commandController?.term(termOptions) + } +} diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index efa1a2484..89bf2ebec 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -13,98 +13,38 @@ import { splitCommand } from "../util/splitCommand" import { promisify } from "node:util" import * as CP from "node:child_process" +export { Daemon } from "./Daemon" +export { CommandController } from "./CommandController" +import { HealthDaemon } from "./HealthDaemon" +import { Daemon } from "./Daemon" +import { CommandController } from "./CommandController" + const cpExec = promisify(CP.exec) -const cpExecFile = promisify(CP.execFile) -type Daemon< +export const cpExecFile = promisify(CP.execFile) +export type Ready = { + display: string | null + fn: () => Promise | CheckResult + trigger?: Trigger +} + +type DaemonsParams< Manifest extends SDKManifest, Ids extends string, Command extends string, Id extends string, > = { - id: "" extends Id ? never : Id command: ValidIfNoStupidEscape | [string, ...string[]] image: { id: Manifest["images"][number]; sharedRun?: boolean } - mounts: Mounts + mounts: { path: string; options: MountOptions }[] env?: Record - ready: { - display: string | null - fn: () => Promise | CheckResult - trigger?: Trigger - } + ready: Ready requires: Exclude[] } type ErrorDuplicateId = `The id '${Id}' is already used` -export const runDaemon = - () => - async ( - effects: Effects, - image: { id: Manifest["images"][number]; sharedRun?: boolean }, - command: ValidIfNoStupidEscape | [string, ...string[]], - options: CommandOptions & { - mounts?: { path: string; options: MountOptions }[] - overlay?: Overlay - }, - ): Promise => { - const commands = splitCommand(command) - const overlay = options.overlay || (await Overlay.of(effects, image)) - for (let mount of options.mounts || []) { - await overlay.mount(mount.options, mount.path) - } - const childProcess = await overlay.spawn(commands, { - env: options.env, - }) - const answer = new Promise((resolve, reject) => { - childProcess.stdout.on("data", (data: any) => { - console.log(data.toString()) - }) - childProcess.stderr.on("data", (data: any) => { - console.error(data.toString()) - }) - - childProcess.on("exit", (code: any) => { - if (code === 0) { - return resolve(null) - } - return reject(new Error(`${commands[0]} exited with code ${code}`)) - }) - }) - - const pid = childProcess.pid - return { - async wait() { - try { - return await answer - } finally { - await cpExecFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}) - } - }, - async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) { - try { - await cpExecFile("pkill", [`-${signal}`, "-s", String(pid)]) - - if (timeout > NO_TIMEOUT) { - const didTimeout = await Promise.race([ - new Promise((resolve) => setTimeout(resolve, timeout)).then( - () => true, - ), - answer.then(() => false), - ]) - if (didTimeout) { - await cpExecFile("pkill", [`-9`, "-s", String(pid)]).catch( - (_) => {}, - ) - } - } else { - await answer - } - } finally { - await overlay.destroy() - } - }, - } - } +export const runCommand = () => + CommandController.of() /** * A class for defining and controlling the service daemons @@ -133,7 +73,9 @@ export class Daemons { private constructor( readonly effects: Effects, readonly started: (onTerm: () => PromiseLike) => PromiseLike, - readonly daemons?: Daemon[], + readonly daemons: Promise[], + readonly ids: Ids[], + readonly healthDaemons: HealthDaemon[], ) {} /** * Returns an empty new Daemons class with the provided config. @@ -150,7 +92,13 @@ export class Daemons { started: (onTerm: () => PromiseLike) => PromiseLike healthReceipts: HealthReceipt[] }) { - return new Daemons(config.effects, config.started) + return new Daemons( + config.effects, + config.started, + [], + [], + [], + ) } /** * Returns the complete list of daemons, including the one defined here @@ -165,73 +113,56 @@ export class Daemons { ErrorDuplicateId extends Id ? never : Id extends Ids ? ErrorDuplicateId : Id, - newDaemon: Omit, "id">, + options: DaemonsParams, ) { - const daemons = ((this?.daemons ?? []) as any[]).concat({ - ...newDaemon, + const daemonIndex = this.daemons.length + const daemon = Daemon.of()( + this.effects, + options.image, + options.command, + options, + ) + const healthDaemon = new HealthDaemon( + daemon, + daemonIndex, + options.requires + .map((x) => this.ids.indexOf(id as any)) + .filter((x) => x >= 0) + .map((id) => this.healthDaemons[id]), id, - }) - return new Daemons(this.effects, this.started, daemons) + this.ids, + options.ready, + this.effects, + ) + const daemons = this.daemons.concat(daemon) + const ids = [...this.ids, id] as (Ids | Id)[] + const healthDaemons = [...this.healthDaemons, healthDaemon] + return new Daemons( + this.effects, + this.started, + daemons, + ids, + healthDaemons, + ) } async build() { - const daemonsStarted = {} as Record> - const { effects } = this - const daemons = this.daemons ?? [] - for (const daemon of daemons) { - const requiredPromise = Promise.all( - daemon.requires?.map((id) => daemonsStarted[id]) ?? [], - ) - daemonsStarted[daemon.id] = requiredPromise.then(async () => { - const { command, image } = daemon - - const child = runDaemon()(effects, image, command, { - env: daemon.env, - mounts: daemon.mounts.build(), - }) - let currentInput: TriggerInput = {} - const getCurrentInput = () => currentInput - const trigger = (daemon.ready.trigger ?? defaultTrigger)( - getCurrentInput, - ) - return new Promise(async (resolve) => { - for ( - let res = await trigger.next(); - !res.done; - res = await trigger.next() - ) { - const response = await Promise.resolve(daemon.ready.fn()).catch( - (err) => - ({ - status: "failure", - message: "message" in err ? err.message : String(err), - }) as CheckResult, - ) - currentInput.lastResult = response.status || null - if (!currentInput.hadSuccess && response.status === "success") { - currentInput.hadSuccess = true - resolve(child) - } - } - resolve(child) - }) - }) - } + this.updateMainHealth() + this.healthDaemons.forEach((x) => + x.addWatcher(() => this.updateMainHealth()), + ) return { - async term(options?: { signal?: Signals; timeout?: number }) { - await Promise.all( - Object.values>(daemonsStarted).map((x) => - x.then((x) => x.term(options)), - ), - ) - }, - async wait() { - await Promise.all( - Object.values>(daemonsStarted).map((x) => - x.then((x) => x.wait()), - ), - ) + term: async (options?: { signal?: Signals; timeout?: number }) => { + await Promise.all(this.healthDaemons.map((x) => x.term(options))) }, } } + + private updateMainHealth() { + if (this.healthDaemons.every((x) => x.health.status === "success")) { + this.effects.setMainStatus({ status: "running" }) + } else { + this.effects.setMainStatus({ status: "starting" }) + } + } } diff --git a/sdk/lib/mainFn/HealthDaemon.ts b/sdk/lib/mainFn/HealthDaemon.ts new file mode 100644 index 000000000..84e9e34d7 --- /dev/null +++ b/sdk/lib/mainFn/HealthDaemon.ts @@ -0,0 +1,152 @@ +import { CheckResult } from "../health/checkFns" +import { defaultTrigger } from "../trigger/defaultTrigger" +import { Ready } from "./Daemons" +import { Daemon } from "./Daemon" +import { Effects } from "../types" + +const oncePromise = () => { + let resolve: (value: T) => void + const promise = new Promise((res) => { + resolve = res + }) + return { resolve: resolve!, promise } +} + +/** + * Wanted a structure that deals with controlling daemons by their health status + * States: + * -- Waiting for dependencies to be success + * -- Running: Daemon is running and the status is in the health + * + */ +export class HealthDaemon { + #health: CheckResult = { status: "starting", message: null } + #healthWatchers: Array<() => unknown> = [] + #running = false + #hadSuccess = false + constructor( + readonly daemon: Promise, + readonly daemonIndex: number, + readonly dependencies: HealthDaemon[], + readonly id: string, + readonly ids: string[], + readonly ready: Ready, + readonly effects: Effects, + ) { + this.updateStatus() + this.dependencies.forEach((d) => d.addWatcher(() => this.updateStatus())) + } + + /** Run after we want to do cleanup */ + async term(termOptions?: { + signal?: NodeJS.Signals | undefined + timeout?: number | undefined + }) { + this.#healthWatchers = [] + this.#running = false + this.#healthCheckCleanup?.() + + await this.daemon.then((d) => d.stop(termOptions)) + } + + /** Want to add another notifier that the health might have changed */ + addWatcher(watcher: () => unknown) { + this.#healthWatchers.push(watcher) + } + + get health() { + return Object.freeze(this.#health) + } + + private async changeRunning(newStatus: boolean) { + if (this.#running === newStatus) return + + this.#running = newStatus + + if (newStatus) { + ;(await this.daemon).start() + this.setupHealthCheck() + } else { + ;(await this.daemon).stop() + this.turnOffHealthCheck() + + this.setHealth({ status: "starting", message: null }) + } + } + + #healthCheckCleanup: (() => void) | null = null + private turnOffHealthCheck() { + this.#healthCheckCleanup?.() + } + private async setupHealthCheck() { + if (this.#healthCheckCleanup) return + const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({ + hadSuccess: this.#hadSuccess, + lastResult: this.#health.status, + })) + + const { promise: status, resolve: setStatus } = oncePromise<{ + done: true + }>() + new Promise(async () => { + for ( + let res = await Promise.race([status, trigger.next()]); + !res.done; + res = await Promise.race([status, trigger.next()]) + ) { + const response: CheckResult = await Promise.resolve( + this.ready.fn(), + ).catch((err) => { + console.error(err) + return { + status: "failure", + message: "message" in err ? err.message : String(err), + } + }) + this.setHealth(response) + if (response.status === "success") { + this.#hadSuccess = true + } + } + }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`)) + + this.#healthCheckCleanup = () => { + setStatus({ done: true }) + this.#healthCheckCleanup = null + } + } + + private setHealth(health: CheckResult) { + this.#health = health + this.#healthWatchers.forEach((watcher) => watcher()) + const display = this.ready.display + const status = health.status + if (!display) { + return + } + if ( + status === "success" || + status === "disabled" || + status === "starting" + ) { + this.effects.setHealth({ + result: status, + message: health.message, + id: display, + name: display, + }) + } else { + this.effects.setHealth({ + result: health.status, + message: health.message || "", + id: display, + name: display, + }) + } + } + + private async updateStatus() { + const healths = this.dependencies.map((d) => d.#health) + this.changeRunning(healths.every((x) => x.status === "success")) + } +} diff --git a/sdk/lib/osBindings/SetMainStatus.ts b/sdk/lib/osBindings/SetMainStatus.ts index 32a839984..aa4aff0b2 100644 --- a/sdk/lib/osBindings/SetMainStatus.ts +++ b/sdk/lib/osBindings/SetMainStatus.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Status } from "./Status" -export type SetMainStatus = { status: Status } +export type SetMainStatus = { status: "running" | "stopped" | "starting" } diff --git a/sdk/lib/test/startosTypeValidation.test.ts b/sdk/lib/test/startosTypeValidation.test.ts index fd11ab5b6..4f6d50f53 100644 --- a/sdk/lib/test/startosTypeValidation.test.ts +++ b/sdk/lib/test/startosTypeValidation.test.ts @@ -1,5 +1,9 @@ import { Effects } from "../types" -import { CheckDependenciesParam, ExecuteAction } from ".././osBindings" +import { + CheckDependenciesParam, + ExecuteAction, + SetMainStatus, +} from ".././osBindings" import { CreateOverlayedImageParams } from ".././osBindings" import { DestroyOverlayedImageParams } from ".././osBindings" import { BindParams } from ".././osBindings" @@ -66,6 +70,7 @@ describe("startosTypeValidation ", () => { mount: {} as MountParams, checkDependencies: {} as CheckDependenciesParam, getDependencies: undefined, + setMainStatus: {} as SetMainStatus, }) typeEquality[0]>( testInput as ExecuteAction, diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index 96e256766..7db42e5a9 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -4,6 +4,7 @@ import { DependencyRequirement, SetHealth, HealthCheckResult, + SetMainStatus, } from "./osBindings" import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk" @@ -163,7 +164,7 @@ export type CommandType = | [string, ...string[]] export type DaemonReturned = { - wait(): Promise + wait(): Promise term(options?: { signal?: Signals; timeout?: number }): Promise } @@ -380,6 +381,8 @@ export type Effects = { }): Promise } + setMainStatus(o: SetMainStatus): Promise + getSystemSmtp(input: { callback: (config: unknown, previousConfig: unknown) => void }): Promise diff --git a/sdk/package.json b/sdk/package.json index 409c5bb02..db387c036 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./cjs/lib/index.js", "types": "./cjs/lib/index.d.ts", From 69a5186c99617e0490377c52191162615beada84 Mon Sep 17 00:00:00 2001 From: J H Date: Thu, 9 May 2024 17:51:49 -0600 Subject: [PATCH 14/16] wipFix: Making a service start --- .../src/service/service_effect_handler.rs | 22 +++++++++++++++++-- sdk/lib/mainFn/Daemon.ts | 2 +- sdk/lib/mainFn/Daemons.ts | 1 + 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 95f212a6d..44d17da30 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -581,10 +581,28 @@ struct GetHostInfoParams { callback: Callback, } async fn get_host_info( - _: EffectContext, + ctx: EffectContext, GetHostInfoParams { .. }: GetHostInfoParams, ) -> Result { - todo!() + let ctx = ctx.deref()?; + Ok(json!({ + "id": "fakeId1", + "kind": "multi", + "hostnames": [{ + "kind": "ip", + "networkInterfaceId": "fakeNetworkInterfaceId1", + "public": true, + "hostname":{ + "kind": "domain", + "domain": format!("{}", ctx.id), + "subdomain": (), + "port": (), + "sslPort": () + } + } + + ] + })) } async fn clear_bindings(context: EffectContext, _: Empty) -> Result { diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index 7cbb422b9..77d27dae0 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -48,7 +48,7 @@ export class Daemon { new Promise(async () => { while (this.shouldBeRunning) { this.commandController = await this.startCommand() - await this.commandController.wait() + await this.commandController.wait().catch((err) => console.error(err)) } }).catch((err) => { console.error(err) diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index 89bf2ebec..5f765dea4 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -154,6 +154,7 @@ export class Daemons { return { term: async (options?: { signal?: Signals; timeout?: number }) => { await Promise.all(this.healthDaemons.map((x) => x.term(options))) + this.effects.setMainStatus({ status: "stopped" }) }, } } From 1227999374e86bf087a47f673b7c191d2b2d4cd1 Mon Sep 17 00:00:00 2001 From: J H Date: Fri, 10 May 2024 09:54:52 -0600 Subject: [PATCH 15/16] fix: Both the start + stop of the service. --- .../src/Adapters/Systems/SystemForEmbassy/MainLoop.ts | 3 ++- .../src/Adapters/Systems/SystemForEmbassy/index.ts | 4 +++- container-runtime/src/Models/Duration.ts | 2 +- sdk/lib/mainFn/Daemon.ts | 5 ++++- sdk/lib/mainFn/Daemons.ts | 7 +++++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index c89d6064a..917bfff83 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -116,7 +116,8 @@ export class MainLoop { const main = await mainEvent delete this.mainEvent delete this.healthLoops - if (mainEvent) await main?.daemon.stop() + await main?.daemon.stop().catch((e) => console.error(e)) + this.effects.setMainStatus({ status: "stopped" }) if (healthLoops) healthLoops.forEach((x) => clearInterval(x.interval)) } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 8988c5446..ada893b2b 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -385,13 +385,15 @@ export class SystemForEmbassy implements System { timeoutMs: number | null, ): Promise { const { currentRunning } = this + this.currentRunning?.clean() delete this.currentRunning if (currentRunning) { await currentRunning.clean({ timeout: this.manifest.main["sigterm-timeout"], }) } - return duration(this.manifest.main["sigterm-timeout"], "s") + const durationValue = duration(this.manifest.main["sigterm-timeout"], "s") + return durationValue } private async createBackup( effects: HostSystemStartOs, diff --git a/container-runtime/src/Models/Duration.ts b/container-runtime/src/Models/Duration.ts index 8c701a703..75154c782 100644 --- a/container-runtime/src/Models/Duration.ts +++ b/container-runtime/src/Models/Duration.ts @@ -2,5 +2,5 @@ export type TimeUnit = "d" | "h" | "s" | "ms" export type Duration = `${number}${TimeUnit}` export function duration(timeValue: number, timeUnit: TimeUnit = "s") { - return `${timeValue}${timeUnit}` as Duration + return `${timeValue > 0 ? timeValue : 0}${timeUnit}` as Duration } diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index 77d27dae0..d4f08dc80 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -65,6 +65,9 @@ export class Daemon { timeout?: number | undefined }) { this.shouldBeRunning = false - await this.commandController?.term(termOptions) + await this.commandController + ?.term(termOptions) + .catch((e) => console.error(e)) + this.commandController = null } } diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index 5f765dea4..5771136c9 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -153,8 +153,11 @@ export class Daemons { ) return { term: async (options?: { signal?: Signals; timeout?: number }) => { - await Promise.all(this.healthDaemons.map((x) => x.term(options))) - this.effects.setMainStatus({ status: "stopped" }) + try { + await Promise.all(this.healthDaemons.map((x) => x.term(options))) + } finally { + this.effects.setMainStatus({ status: "stopped" }) + } }, } } From 2bcc53cb5d46e71e65b17ddf8ec170f0559a24c0 Mon Sep 17 00:00:00 2001 From: J H Date: Fri, 10 May 2024 14:27:54 -0600 Subject: [PATCH 16/16] fix: Weird edge case of failure and kids --- sdk/lib/mainFn/CommandController.ts | 7 ++++++- sdk/lib/mainFn/Daemon.ts | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/lib/mainFn/CommandController.ts b/sdk/lib/mainFn/CommandController.ts index b894600a2..e2e11dbfc 100644 --- a/sdk/lib/mainFn/CommandController.ts +++ b/sdk/lib/mainFn/CommandController.ts @@ -75,11 +75,16 @@ export class CommandController { return await this.runningAnswer } finally { await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch((_) => {}) + await this.overlay.destroy().catch((_) => {}) } } async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) { try { - await cpExecFile("pkill", [`-${signal}`, "-s", String(this.pid)]) + await cpExecFile("pkill", [ + `-${signal.replace("SIG", "")}`, + "-s", + String(this.pid), + ]) if (timeout > NO_TIMEOUT) { const didTimeout = await Promise.race([ diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index d4f08dc80..2ec438801 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -3,6 +3,8 @@ import { Effects, ValidIfNoStupidEscape } from "../types" import { MountOptions, Overlay } from "../util/Overlay" import { CommandController } from "./CommandController" +const TIMEOUT_INCREMENT_MS = 1000 +const MAX_TIMEOUT_MS = 30000 /** * This is a wrapper around CommandController that has a state of off, where the command shouldn't be running * and the others state of running, where it will keep a living running command @@ -45,10 +47,14 @@ export class Daemon { return } this.shouldBeRunning = true + let timeoutCounter = 0 new Promise(async () => { while (this.shouldBeRunning) { this.commandController = await this.startCommand() await this.commandController.wait().catch((err) => console.error(err)) + await new Promise((resolve) => setTimeout(resolve, timeoutCounter)) + timeoutCounter += TIMEOUT_INCREMENT_MS + timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter) } }).catch((err) => { console.error(err)