From b4dee35573c252c24b387bf08be9bd4b757a32ea Mon Sep 17 00:00:00 2001 From: J H Date: Mon, 29 Apr 2024 14:57:35 -0600 Subject: [PATCH 1/8] feat: Implementation on the backup for the service. --- .../DockerProcedureContainer.ts | 5 +- .../Systems/SystemForEmbassy/MainLoop.ts | 9 ++ .../Systems/SystemForEmbassy/index.ts | 49 +---------- .../Systems/SystemForEmbassy/matchManifest.ts | 1 + core/startos/src/service/mod.rs | 11 ++- .../src/service/persistent_container.rs | 32 +++++++- .../src/service/service_effect_handler.rs | 12 +-- core/startos/src/service/transition/backup.rs | 82 +++++++++++++++++++ core/startos/src/service/transition/mod.rs | 14 ++-- .../startos/src/service/transition/restart.rs | 32 ++++++-- .../ExportServiceInterfaceParams.ts | 7 +- 11 files changed, 184 insertions(+), 70 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index c8388d151..b92da4818 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -68,7 +68,10 @@ export class DockerProcedureContainer { }, }) } else if (volumeMount.type === "backup") { - throw new Error("TODO") + await overlay.mount( + { type: "volume", id: mount, subpath: null, readonly: false }, + mounts[mount], + ) } } } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 0b4f92002..7e18c69f1 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -145,6 +145,15 @@ export class MainLoop { ...actionProcedure.args, JSON.stringify(timeChanged), ]) + if (executed.exitCode === 0) { + await effects.setHealth({ + id: healthId, + name: value.name, + result: "success", + message: actionProcedure["success-message"], + }) + return + } if (executed.exitCode === 59) { await effects.setHealth({ id: healthId, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 4cd3d1bfe..f1d2f8b8e 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -399,11 +399,10 @@ export class SystemForEmbassy implements System { ): Promise { const backup = this.manifest.backup.create if (backup.type === "docker") { - const container = await DockerProcedureContainer.of( - effects, - backup, - this.manifest.volumes, - ) + const container = await DockerProcedureContainer.of(effects, backup, { + ...this.manifest.volumes, + BACKUP: { type: "backup", readonly: false }, + }) await container.execFail([backup.entrypoint, ...backup.args], timeoutMs) } else { const moduleCode = await this.moduleCode @@ -664,46 +663,6 @@ export class SystemForEmbassy implements System { } throw new Error(`Unknown type in the fetch properties: ${setConfigValue}`) } - private async health( - effects: HostSystemStartOs, - healthId: string, - timeSinceStarted: unknown, - timeoutMs: number | null, - ): Promise { - const healthProcedure = this.manifest["health-checks"][healthId] - if (!healthProcedure) return - if (healthProcedure.type === "docker") { - const container = await DockerProcedureContainer.of( - effects, - healthProcedure, - this.manifest.volumes, - ) - return JSON.parse( - ( - await container.execFail( - [ - healthProcedure.entrypoint, - ...healthProcedure.args, - JSON.stringify(timeSinceStarted), - ], - timeoutMs, - ) - ).stdout.toString(), - ) - } else if (healthProcedure.type === "script") { - const moduleCode = await this.moduleCode - const method = moduleCode.health?.[healthId] - if (!method) throw new Error("Expecting that the method health exists") - await method( - new PolyfillEffects(effects, this.manifest), - Number(timeSinceStarted), - ).then((x) => { - if ("result" in x) return x.result - if ("error" in x) throw new Error("Error getting config: " + x.error) - throw new Error("Error getting config: " + x["error-code"][1]) - }) - } - } private async action( effects: HostSystemStartOs, actionId: string, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts index 85cd1bac3..d4758669b 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts @@ -57,6 +57,7 @@ export const matchManifest = object( matchProcedure, object({ name: string, + ["success-message"]: string, }), ), ]), diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index a4e3a4858..fe8baeceb 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -348,9 +348,14 @@ impl Service { Ok(()) } - pub async fn backup(&self, _guard: impl GenericMountGuard) -> Result { - // TODO - Err(Error::new(eyre!("not yet implemented"), ErrorKind::Unknown)) + #[instrument(skip_all)] + pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> { + self.actor + .send(transition::backup::Backup { + path: guard.path().to_path_buf(), + }) + .await?; + Ok(()) } pub fn container_id(&self) -> Result { diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index e69498e5f..8b1dd459e 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -1,7 +1,7 @@ -use std::collections::BTreeMap; use std::path::Path; use std::sync::{Arc, Weak}; use std::time::Duration; +use std::{collections::BTreeMap, path::PathBuf}; use futures::future::ready; use futures::{Future, FutureExt}; @@ -222,6 +222,36 @@ impl PersistentContainer { }) } + #[instrument(skip_all)] + pub async fn mount_backup(&self, backup_path: impl AsRef) -> Result { + let backup_path: PathBuf = backup_path.as_ref().to_path_buf(); + let mountpoint = self + .lxc_container + .get() + .ok_or_else(|| { + Error::new( + eyre!("PersistentContainer has been destroyed"), + ErrorKind::Incoherent, + ) + })? + .rootfs_dir() + .join("media/startos/volumes/BACKUP"); + tokio::fs::create_dir_all(&mountpoint).await?; + let bind = Bind::new(&backup_path); + let mount_guard = MountGuard::mount(&bind, &mountpoint, MountType::ReadWrite).await; + Command::new("chown") + .arg("100000:100000") + .arg(mountpoint.as_os_str()) + .invoke(ErrorKind::Filesystem) + .await?; + Command::new("chown") + .arg("100000:100000") + .arg(backup_path.as_os_str()) + .invoke(ErrorKind::Filesystem) + .await?; + mount_guard + } + #[instrument(skip_all)] pub async fn init(&self, seed: Weak) -> Result<(), Error> { let socket_server_context = EffectContext::new(seed); diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 0ad45aad1..aa3560fe5 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -1069,11 +1069,6 @@ pub async fn create_overlayed_image( .rootfs_dir(); let mountpoint = rootfs_dir.join("media/startos/overlays").join(&*guid); tokio::fs::create_dir_all(&mountpoint).await?; - Command::new("chown") - .arg("100000:100000") - .arg(&mountpoint) - .invoke(ErrorKind::Filesystem) - .await?; let container_mountpoint = Path::new("/").join( mountpoint .strip_prefix(rootfs_dir) @@ -1082,9 +1077,14 @@ pub async fn create_overlayed_image( tracing::info!("Mounting overlay {guid} for {image_id}"); let guard = OverlayGuard::mount( &IdMapped::new(LoopDev::from(&**image), 0, 100000, 65536), - mountpoint, + &mountpoint, ) .await?; + Command::new("chown") + .arg("100000:100000") + .arg(&mountpoint) + .invoke(ErrorKind::Filesystem) + .await?; tracing::info!("Mounted overlay {guid} for {image_id}"); ctx.persistent_container .overlays diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index 8b1378917..f52f7bb65 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -1 +1,83 @@ +use std::path::PathBuf; +use futures::FutureExt; +use models::ProcedureName; + +use super::TempDesiredRestore; +use crate::prelude::*; +use crate::service::config::GetConfig; +use crate::service::dependencies::DependencyConfig; +use crate::service::transition::{TransitionKind, TransitionState}; +use crate::service::ServiceActor; +use crate::util::actor::background::BackgroundJobQueue; +use crate::util::actor::{ConflictBuilder, Handler}; +use crate::util::future::RemoteCancellable; + +pub(in crate::service) struct Backup { + pub path: PathBuf, +} +impl Handler for ServiceActor { + type Response = (); + fn conflicts_with(_: &Backup) -> ConflictBuilder { + ConflictBuilder::everything() + .except::() + .except::() + } + async fn handle(&mut self, backup: Backup, jobs: &BackgroundJobQueue) -> Self::Response { + // So Need a handle to just a single field in the state + let temp: TempDesiredRestore = TempDesiredRestore::new(&self.0.persistent_container.state); + let mut current = self.0.persistent_container.state.subscribe(); + let path = backup.path.clone(); + let seed = self.0.clone(); + + let transition = RemoteCancellable::new(async move { + temp.stop(); + current + .wait_for(|s| s.running_status.is_none()) + .await + .with_kind(ErrorKind::Unknown)?; + + let backup_guard = seed.persistent_container.mount_backup(path).await?; + seed.persistent_container + .execute(ProcedureName::CreateBackup, Value::Null, None) + .await?; + drop(backup_guard); + + if temp.restore().is_start() { + current + .wait_for(|s| s.running_status.is_some()) + .await + .with_kind(ErrorKind::Unknown)?; + } + drop(temp); + Ok::<_, Error>(()) + }); + let cancel_handle = transition.cancellation_handle(); + jobs.add_job(transition.map(|x| { + if let Some(Err(err)) = x { + tracing::debug!("{:?}", err); + tracing::warn!("{}", err); + } + })); + let notified = self.0.synchronized.notified(); + + let mut old = None; + self.0.persistent_container.state.send_modify(|s| { + old = std::mem::replace( + &mut s.transition_state, + Some(TransitionState { + kind: TransitionKind::BackingUp, + cancel_handle, + }), + ) + }); + if let Some(t) = old { + t.abort().await; + } + notified.await; + + self.0.persistent_container.state.send_modify(|s| { + s.transition_state.take(); + }); + } +} diff --git a/core/startos/src/service/transition/mod.rs b/core/startos/src/service/transition/mod.rs index 25d225492..5c9710917 100644 --- a/core/startos/src/service/transition/mod.rs +++ b/core/startos/src/service/transition/mod.rs @@ -59,21 +59,23 @@ impl Drop for TransitionState { } #[derive(Debug, Clone)] -pub struct TempDesiredState(pub(super) Arc>); -impl TempDesiredState { +pub struct TempDesiredRestore(pub(super) Arc>, StartStop); +impl TempDesiredRestore { pub fn new(state: &Arc>) -> Self { - Self(state.clone()) + Self(state.clone(), state.borrow().desired_state) } pub fn stop(&self) { self.0 .send_modify(|s| s.temp_desired_state = Some(StartStop::Stop)); } - pub fn start(&self) { + pub fn restore(&self) -> StartStop { + let restore_state = self.1; self.0 - .send_modify(|s| s.temp_desired_state = Some(StartStop::Start)); + .send_modify(|s| s.temp_desired_state = Some(restore_state)); + restore_state } } -impl Drop for TempDesiredState { +impl Drop for TempDesiredRestore { fn drop(&mut self) { self.0.send_modify(|s| s.temp_desired_state = None); } diff --git a/core/startos/src/service/transition/restart.rs b/core/startos/src/service/transition/restart.rs index dbf066e6f..aaee84ddc 100644 --- a/core/startos/src/service/transition/restart.rs +++ b/core/startos/src/service/transition/restart.rs @@ -1,6 +1,6 @@ use futures::FutureExt; -use super::TempDesiredState; +use super::TempDesiredRestore; use crate::prelude::*; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; @@ -20,17 +20,30 @@ impl Handler for ServiceActor { } async fn handle(&mut self, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response { // So Need a handle to just a single field in the state - let temp = TempDesiredState::new(&self.0.persistent_container.state); + let temp = TempDesiredRestore::new(&self.0.persistent_container.state); let mut current = self.0.persistent_container.state.subscribe(); let transition = RemoteCancellable::new(async move { temp.stop(); - current.wait_for(|s| s.running_status.is_none()).await; - temp.start(); - current.wait_for(|s| s.running_status.is_some()).await; + current + .wait_for(|s| s.running_status.is_none()) + .await + .with_kind(ErrorKind::Unknown)?; + if temp.restore().is_start() { + current + .wait_for(|s| s.running_status.is_some()) + .await + .with_kind(ErrorKind::Unknown)?; + } drop(temp); + Ok::<_, Error>(()) }); let cancel_handle = transition.cancellation_handle(); - jobs.add_job(transition.map(|_| ())); + jobs.add_job(transition.map(|x| { + if let Some(Err(err)) = x { + tracing::debug!("{:?}", err); + tracing::warn!("{}", err); + } + })); let notified = self.0.synchronized.notified(); let mut old = None; @@ -46,10 +59,15 @@ impl Handler for ServiceActor { if let Some(t) = old { t.abort().await; } - notified.await + notified.await; + + self.0.persistent_container.state.send_modify(|s| { + s.transition_state.take(); + }); } } impl Service { + #[instrument(skip_all)] pub async fn restart(&self) -> Result<(), Error> { self.actor.send(Restart).await } diff --git a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts index 152a800bb..847a6090a 100644 --- a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts +++ b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts @@ -1,9 +1,12 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AddressInfo } from "./AddressInfo" +import type { ExportedHostnameInfo } from "./ExportedHostnameInfo" +import type { HostKind } from "./HostKind" +import type { ServiceInterfaceId } from "./ServiceInterfaceId" import type { ServiceInterfaceType } from "./ServiceInterfaceType" export type ExportServiceInterfaceParams = { - id: string + id: ServiceInterfaceId name: string description: string hasPrimary: boolean @@ -11,4 +14,6 @@ export type ExportServiceInterfaceParams = { masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType + hostKind: HostKind + hostnames: Array } From 0aefe385d088ee212ebb75f49845c90c06074b6c Mon Sep 17 00:00:00 2001 From: J H Date: Mon, 29 Apr 2024 16:38:46 -0600 Subject: [PATCH 2/8] wip: Getting the flow of backup/restore --- .../Systems/SystemForEmbassy/index.ts | 5 ++- core/startos/src/backup/backup_bulk.rs | 41 +++++++++++++++---- core/startos/src/service/mod.rs | 27 +++++++++--- core/startos/src/service/transition/backup.rs | 2 +- 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index f1d2f8b8e..8988c5446 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -420,7 +420,10 @@ export class SystemForEmbassy implements System { const container = await DockerProcedureContainer.of( effects, restoreBackup, - this.manifest.volumes, + { + ...this.manifest.volumes, + BACKUP: { type: "backup", readonly: true }, + }, ) await container.execFail( [restoreBackup.entrypoint, ...restoreBackup.args], diff --git a/core/startos/src/backup/backup_bulk.rs b/core/startos/src/backup/backup_bulk.rs index 928e4811a..edb2c8ec9 100644 --- a/core/startos/src/backup/backup_bulk.rs +++ b/core/startos/src/backup/backup_bulk.rs @@ -13,7 +13,7 @@ use tokio::io::AsyncWriteExt; use tracing::instrument; use ts_rs::TS; -use super::target::BackupTargetId; +use super::target::{BackupTargetId, PackageBackupInfo}; use super::PackageBackupReport; use crate::auth::check_password_against_db; use crate::backup::os::OsBackup; @@ -246,19 +246,43 @@ async fn perform_backup( backup_guard: BackupMountGuard, package_ids: &OrdSet, ) -> Result, Error> { + let db = ctx.db.peek().await; let mut backup_report = BTreeMap::new(); let backup_guard = Arc::new(backup_guard); + let mut package_backups: BTreeMap = + backup_guard.metadata.package_backups.clone(); for id in package_ids { if let Some(service) = &*ctx.services.get(id).await { + let backup_result = service + .backup(backup_guard.package_backup(id)) + .await + .err() + .map(|e| e.to_string()); + if backup_result.is_none() { + let manifest = db + .as_public() + .as_package_data() + .as_idx(id) + .or_not_found(id)? + .as_state_info() + .expect_installed()? + .as_manifest(); + + package_backups.insert( + id.clone(), + PackageBackupInfo { + os_version: manifest.as_os_version().de()?, + version: manifest.as_version().de()?, + title: manifest.as_title().de()?, + timestamp: Utc::now(), + }, + ); + } backup_report.insert( id.clone(), PackageBackupReport { - error: service - .backup(backup_guard.package_backup(id)) - .await - .err() - .map(|e| e.to_string()), + error: backup_result, }, ); } @@ -298,7 +322,7 @@ async fn perform_backup( } let luks_folder = Path::new("/media/embassy/config/luks"); if tokio::fs::metadata(&luks_folder).await.is_ok() { - dir_copy(&luks_folder, &luks_folder_bak, None).await?; + dir_copy(luks_folder, &luks_folder_bak, None).await?; } let timestamp = Some(Utc::now()); @@ -307,8 +331,9 @@ async fn perform_backup( backup_guard.unencrypted_metadata.full = true; backup_guard.metadata.version = crate::version::Current::new().semver().into(); backup_guard.metadata.timestamp = timestamp; + backup_guard.metadata.package_backups = package_backups; - backup_guard.save_and_unmount().await?; + backup_guard.save().await?; ctx.db .mutate(|v| { diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index fe8baeceb..6e3c53af9 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -296,13 +296,28 @@ impl Service { } pub async fn restore( - _ctx: RpcContext, - _s9pk: S9pk, - _guard: impl GenericMountGuard, - _progress: Option, + ctx: RpcContext, + s9pk: S9pk, + backup_source: impl GenericMountGuard, + progress: Option, ) -> Result { - // TODO - Err(Error::new(eyre!("not yet implemented"), ErrorKind::Unknown)) + dbg!("Restoring starting"); + let service = Service::install(ctx.clone(), s9pk, None, progress).await?; + + let backup_guard = service + .seed + .persistent_container + .mount_backup(backup_source.path()) + .await?; + service + .seed + .persistent_container + .execute(ProcedureName::RestoreBackup, Value::Null, None) + .await?; + backup_guard.unmount(true).await?; + + dbg!("Restoring done"); + Ok(service) } pub async fn shutdown(self) -> Result<(), Error> { diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index f52f7bb65..45f8fd9c9 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -41,7 +41,7 @@ impl Handler for ServiceActor { seed.persistent_container .execute(ProcedureName::CreateBackup, Value::Null, None) .await?; - drop(backup_guard); + backup_guard.unmount(true).await?; if temp.restore().is_start() { current From 89b4dbbabf10ad1b709c9a2d59692622f37ae60d Mon Sep 17 00:00:00 2001 From: J H Date: Tue, 30 Apr 2024 17:18:45 -0600 Subject: [PATCH 3/8] feat: Recover --- core/startos/src/service/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 6e3c53af9..b83efe34a 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -10,7 +10,7 @@ use persistent_container::PersistentContainer; use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, Handler, HandlerArgs}; use serde::{Deserialize, Serialize}; use start_stop::StartStop; -use tokio::sync::Notify; +use tokio::{fs::File, sync::Notify}; use ts_rs::TS; use crate::context::{CliContext, RpcContext}; @@ -365,6 +365,15 @@ impl Service { #[instrument(skip_all)] pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> { + let id = &self.seed.id; + let mut file = File::create(guard.path().join(id).with_extension("s9pk")).await?; + self.seed + .persistent_container + .s9pk + .clone() + .serialize(&mut file, true) + .await?; + drop(file); self.actor .send(transition::backup::Backup { path: guard.path().to_path_buf(), From a66dd7ea8b2b610bafd315e7e783d09142af3bab Mon Sep 17 00:00:00 2001 From: J H Date: Thu, 2 May 2024 13:59:27 -0600 Subject: [PATCH 4/8] Feature: Commit the full pass on the backup restore. --- core/startos/src/service/mod.rs | 17 ++--- core/startos/src/service/start_stop.rs | 1 + core/startos/src/service/transition/backup.rs | 71 ++++++++++-------- core/startos/src/service/transition/mod.rs | 2 + .../startos/src/service/transition/restart.rs | 52 +++++++------ .../startos/src/service/transition/restore.rs | 73 +++++++++++++++++++ core/startos/src/status/mod.rs | 15 +++- sdk/lib/osBindings/MainStatus.ts | 1 + 8 files changed, 161 insertions(+), 71 deletions(-) create mode 100644 core/startos/src/service/transition/restore.rs diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index b83efe34a..62ddf0868 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -301,22 +301,14 @@ impl Service { backup_source: impl GenericMountGuard, progress: Option, ) -> Result { - dbg!("Restoring starting"); let service = Service::install(ctx.clone(), s9pk, None, progress).await?; - let backup_guard = service - .seed - .persistent_container - .mount_backup(backup_source.path()) - .await?; service - .seed - .persistent_container - .execute(ProcedureName::RestoreBackup, Value::Null, None) + .actor + .send(transition::restore::Restore { + path: backup_source.path().to_path_buf(), + }) .await?; - backup_guard.unmount(true).await?; - - dbg!("Restoring done"); Ok(service) } @@ -454,6 +446,7 @@ impl Actor for ServiceActor { kinds.running_status, ) { (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting, + (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring, (Some(TransitionKind::BackingUp), _, Some(status)) => { MainStatus::BackingUp { started: Some(status.started), diff --git a/core/startos/src/service/start_stop.rs b/core/startos/src/service/start_stop.rs index bc24574ac..178176023 100644 --- a/core/startos/src/service/start_stop.rs +++ b/core/startos/src/service/start_stop.rs @@ -15,6 +15,7 @@ impl From for StartStop { fn from(value: MainStatus) -> Self { match value { MainStatus::Stopped => StartStop::Stop, + MainStatus::Restoring => StartStop::Stop, MainStatus::Restarting => StartStop::Start, MainStatus::Stopping { .. } => StartStop::Stop, MainStatus::Starting => StartStop::Start, diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index 45f8fd9c9..4a523c8e6 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use futures::FutureExt; +use futures::{FutureExt, TryFutureExt}; use models::ProcedureName; use super::TempDesiredRestore; @@ -17,7 +17,7 @@ pub(in crate::service) struct Backup { pub path: PathBuf, } impl Handler for ServiceActor { - type Response = (); + type Response = Result<(), Error>; fn conflicts_with(_: &Backup) -> ConflictBuilder { ConflictBuilder::everything() .except::() @@ -30,36 +30,44 @@ impl Handler for ServiceActor { let path = backup.path.clone(); let seed = self.0.clone(); - let transition = RemoteCancellable::new(async move { - temp.stop(); - current - .wait_for(|s| s.running_status.is_none()) - .await - .with_kind(ErrorKind::Unknown)?; - - let backup_guard = seed.persistent_container.mount_backup(path).await?; - seed.persistent_container - .execute(ProcedureName::CreateBackup, Value::Null, None) - .await?; - backup_guard.unmount(true).await?; - - if temp.restore().is_start() { + let state = self.0.persistent_container.state.clone(); + let transition = RemoteCancellable::new( + async move { + temp.stop(); current - .wait_for(|s| s.running_status.is_some()) + .wait_for(|s| s.running_status.is_none()) .await .with_kind(ErrorKind::Unknown)?; + + let backup_guard = seed.persistent_container.mount_backup(path).await?; + seed.persistent_container + .execute(ProcedureName::CreateBackup, Value::Null, None) + .await?; + backup_guard.unmount(true).await?; + + if temp.restore().is_start() { + current + .wait_for(|s| s.running_status.is_some()) + .await + .with_kind(ErrorKind::Unknown)?; + } + drop(temp); + state.send_modify(|s| { + s.transition_state.take(); + }); + Ok::<_, Error>(()) } - drop(temp); - Ok::<_, Error>(()) - }); + .map(|x| { + if let Err(err) = dbg!(x) { + tracing::debug!("{:?}", err); + tracing::warn!("{}", err); + } + }), + ); let cancel_handle = transition.cancellation_handle(); - jobs.add_job(transition.map(|x| { - if let Some(Err(err)) = x { - tracing::debug!("{:?}", err); - tracing::warn!("{}", err); - } - })); - let notified = self.0.synchronized.notified(); + let transition = transition.shared(); + let job_transition = transition.clone(); + jobs.add_job(job_transition.map(|_| ())); let mut old = None; self.0.persistent_container.state.send_modify(|s| { @@ -74,10 +82,9 @@ impl Handler for ServiceActor { if let Some(t) = old { t.abort().await; } - notified.await; - - self.0.persistent_container.state.send_modify(|s| { - s.transition_state.take(); - }); + match transition.await { + None => Err(Error::new(eyre!("Backup canceled"), ErrorKind::Unknown)), + Some(x) => Ok(x), + } } } diff --git a/core/startos/src/service/transition/mod.rs b/core/startos/src/service/transition/mod.rs index 5c9710917..7b7f10f2a 100644 --- a/core/startos/src/service/transition/mod.rs +++ b/core/startos/src/service/transition/mod.rs @@ -10,11 +10,13 @@ use crate::util::future::{CancellationHandle, RemoteCancellable}; pub mod backup; pub mod restart; +pub mod restore; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum TransitionKind { BackingUp, Restarting, + Restoring, } /// Used only in the manager/mod and is used to keep track of the state of the manager during the diff --git a/core/startos/src/service/transition/restart.rs b/core/startos/src/service/transition/restart.rs index aaee84ddc..07466def1 100644 --- a/core/startos/src/service/transition/restart.rs +++ b/core/startos/src/service/transition/restart.rs @@ -22,29 +22,37 @@ impl Handler for ServiceActor { // So Need a handle to just a single field in the state let temp = TempDesiredRestore::new(&self.0.persistent_container.state); let mut current = self.0.persistent_container.state.subscribe(); - let transition = RemoteCancellable::new(async move { - temp.stop(); - current - .wait_for(|s| s.running_status.is_none()) - .await - .with_kind(ErrorKind::Unknown)?; - if temp.restore().is_start() { + let state = self.0.persistent_container.state.clone(); + let transition = RemoteCancellable::new( + async move { + temp.stop(); current - .wait_for(|s| s.running_status.is_some()) + .wait_for(|s| s.running_status.is_none()) .await .with_kind(ErrorKind::Unknown)?; + if temp.restore().is_start() { + current + .wait_for(|s| s.running_status.is_some()) + .await + .with_kind(ErrorKind::Unknown)?; + } + drop(temp); + state.send_modify(|s| { + s.transition_state.take(); + }); + Ok::<_, Error>(()) } - drop(temp); - Ok::<_, Error>(()) - }); + .map(|x| { + if let Err(err) = x { + tracing::debug!("{:?}", err); + tracing::warn!("{}", err); + } + }), + ); let cancel_handle = transition.cancellation_handle(); - jobs.add_job(transition.map(|x| { - if let Some(Err(err)) = x { - tracing::debug!("{:?}", err); - tracing::warn!("{}", err); - } - })); - let notified = self.0.synchronized.notified(); + let transition = transition.shared(); + let job_transition = transition.clone(); + jobs.add_job(job_transition.map(|_| ())); let mut old = None; self.0.persistent_container.state.send_modify(|s| { @@ -59,11 +67,9 @@ impl Handler for ServiceActor { if let Some(t) = old { t.abort().await; } - notified.await; - - self.0.persistent_container.state.send_modify(|s| { - s.transition_state.take(); - }); + if transition.await.is_none() { + tracing::warn!("Service {} has been cancelled", &self.0.id); + } } } impl Service { diff --git a/core/startos/src/service/transition/restore.rs b/core/startos/src/service/transition/restore.rs new file mode 100644 index 000000000..57ee6cfa9 --- /dev/null +++ b/core/startos/src/service/transition/restore.rs @@ -0,0 +1,73 @@ +use std::path::PathBuf; + +use futures::{FutureExt, TryFutureExt}; +use models::ProcedureName; + +use super::TempDesiredRestore; +use crate::prelude::*; +use crate::service::config::GetConfig; +use crate::service::dependencies::DependencyConfig; +use crate::service::transition::{TransitionKind, TransitionState}; +use crate::service::ServiceActor; +use crate::util::actor::background::BackgroundJobQueue; +use crate::util::actor::{ConflictBuilder, Handler}; +use crate::util::future::RemoteCancellable; + +pub(in crate::service) struct Restore { + pub path: PathBuf, +} +impl Handler for ServiceActor { + type Response = Result<(), Error>; + fn conflicts_with(_: &Restore) -> ConflictBuilder { + ConflictBuilder::everything() + } + async fn handle(&mut self, restore: Restore, jobs: &BackgroundJobQueue) -> Self::Response { + // So Need a handle to just a single field in the state + let path = restore.path.clone(); + let seed = self.0.clone(); + + let state = self.0.persistent_container.state.clone(); + let transition = RemoteCancellable::new( + async move { + let backup_guard = seed.persistent_container.mount_backup(path).await?; + seed.persistent_container + .execute(ProcedureName::RestoreBackup, Value::Null, None) + .await?; + backup_guard.unmount(true).await?; + + state.send_modify(|s| { + s.transition_state.take(); + }); + Ok::<_, Error>(()) + } + .map(|x| { + if let Err(err) = dbg!(x) { + tracing::debug!("{:?}", err); + tracing::warn!("{}", err); + } + }), + ); + let cancel_handle = transition.cancellation_handle(); + let transition = transition.shared(); + let job_transition = transition.clone(); + jobs.add_job(job_transition.map(|_| ())); + + let mut old = None; + self.0.persistent_container.state.send_modify(|s| { + old = std::mem::replace( + &mut s.transition_state, + Some(TransitionState { + kind: TransitionKind::Restoring, + cancel_handle, + }), + ) + }); + if let Some(t) = old { + t.abort().await; + } + match transition.await { + None => Err(Error::new(eyre!("Restoring canceled"), ErrorKind::Unknown)), + Some(x) => Ok(x), + } + } +} diff --git a/core/startos/src/status/mod.rs b/core/startos/src/status/mod.rs index 09d10fb2d..520fe5089 100644 --- a/core/startos/src/status/mod.rs +++ b/core/startos/src/status/mod.rs @@ -25,6 +25,7 @@ pub struct Status { pub enum MainStatus { Stopped, Restarting, + Restoring, #[serde(rename_all = "camelCase")] Stopping { timeout: crate::util::serde::Duration, @@ -54,6 +55,7 @@ impl MainStatus { started: Some(_), .. } => true, MainStatus::Stopped + | MainStatus::Restoring | MainStatus::Stopping { .. } | MainStatus::Restarting | MainStatus::BackingUp { started: None, .. } => false, @@ -75,6 +77,7 @@ impl MainStatus { MainStatus::Running { started, .. } => Some(*started), MainStatus::BackingUp { started, .. } => *started, MainStatus::Stopped => None, + MainStatus::Restoring => None, MainStatus::Restarting => None, MainStatus::Stopping { .. } => None, MainStatus::Starting { .. } => None, @@ -84,9 +87,10 @@ impl MainStatus { let (started, health) = match self { MainStatus::Starting { .. } => (Some(Utc::now()), Default::default()), MainStatus::Running { started, health } => (Some(started.clone()), health.clone()), - MainStatus::Stopped | MainStatus::Stopping { .. } | MainStatus::Restarting => { - (None, Default::default()) - } + MainStatus::Stopped + | MainStatus::Stopping { .. } + | MainStatus::Restoring + | MainStatus::Restarting => (None, Default::default()), MainStatus::BackingUp { .. } => return self.clone(), }; MainStatus::BackingUp { started, health } @@ -96,7 +100,10 @@ impl MainStatus { match self { MainStatus::Running { health, .. } => Some(health), MainStatus::BackingUp { health, .. } => Some(health), - MainStatus::Stopped | MainStatus::Stopping { .. } | MainStatus::Restarting => None, + MainStatus::Stopped + | MainStatus::Restoring + | MainStatus::Stopping { .. } + | MainStatus::Restarting => None, MainStatus::Starting { .. } => None, } } diff --git a/sdk/lib/osBindings/MainStatus.ts b/sdk/lib/osBindings/MainStatus.ts index 534cad76f..1b9d8482e 100644 --- a/sdk/lib/osBindings/MainStatus.ts +++ b/sdk/lib/osBindings/MainStatus.ts @@ -6,6 +6,7 @@ import type { HealthCheckResult } from "./HealthCheckResult" export type MainStatus = | { status: "stopped" } | { status: "restarting" } + | { status: "restoring" } | { status: "stopping"; timeout: Duration } | { status: "starting" } | { From 64b43aa9a431885cf8f35834de293da02382bfe4 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Mon, 6 May 2024 10:24:57 -0600 Subject: [PATCH 5/8] use special type for backup instead of special id (#2614) --- .../DockerProcedureContainer.ts | 5 +---- .../startos/src/service/persistent_container.rs | 14 +++++++++----- core/startos/src/service/transition/backup.rs | 8 ++++++-- core/startos/src/service/transition/restore.rs | 11 ++++++----- sdk/lib/util/Overlay.ts | 17 +++++++++++++++++ 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index b92da4818..b4a5f5829 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -68,10 +68,7 @@ export class DockerProcedureContainer { }, }) } else if (volumeMount.type === "backup") { - await overlay.mount( - { type: "volume", id: mount, subpath: null, readonly: false }, - mounts[mount], - ) + await overlay.mount({ type: "backup", subpath: null }, mounts[mount]) } } } diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index 8b1dd459e..f9012c7be 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -1,7 +1,7 @@ -use std::path::Path; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Weak}; use std::time::Duration; -use std::{collections::BTreeMap, path::PathBuf}; use futures::future::ready; use futures::{Future, FutureExt}; @@ -223,7 +223,11 @@ impl PersistentContainer { } #[instrument(skip_all)] - pub async fn mount_backup(&self, backup_path: impl AsRef) -> Result { + pub async fn mount_backup( + &self, + backup_path: impl AsRef, + mount_type: MountType, + ) -> Result { let backup_path: PathBuf = backup_path.as_ref().to_path_buf(); let mountpoint = self .lxc_container @@ -235,10 +239,10 @@ impl PersistentContainer { ) })? .rootfs_dir() - .join("media/startos/volumes/BACKUP"); + .join("media/startos/backup"); tokio::fs::create_dir_all(&mountpoint).await?; let bind = Bind::new(&backup_path); - let mount_guard = MountGuard::mount(&bind, &mountpoint, MountType::ReadWrite).await; + let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await; Command::new("chown") .arg("100000:100000") .arg(mountpoint.as_os_str()) diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index 4a523c8e6..e6a5cb817 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -1,9 +1,10 @@ use std::path::PathBuf; -use futures::{FutureExt, TryFutureExt}; +use futures::FutureExt; use models::ProcedureName; use super::TempDesiredRestore; +use crate::disk::mount::filesystem::ReadWrite; use crate::prelude::*; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; @@ -39,7 +40,10 @@ impl Handler for ServiceActor { .await .with_kind(ErrorKind::Unknown)?; - let backup_guard = seed.persistent_container.mount_backup(path).await?; + let backup_guard = seed + .persistent_container + .mount_backup(path, ReadWrite) + .await?; seed.persistent_container .execute(ProcedureName::CreateBackup, Value::Null, None) .await?; diff --git a/core/startos/src/service/transition/restore.rs b/core/startos/src/service/transition/restore.rs index 57ee6cfa9..b32ade4f5 100644 --- a/core/startos/src/service/transition/restore.rs +++ b/core/startos/src/service/transition/restore.rs @@ -1,12 +1,10 @@ use std::path::PathBuf; -use futures::{FutureExt, TryFutureExt}; +use futures::FutureExt; use models::ProcedureName; -use super::TempDesiredRestore; +use crate::disk::mount::filesystem::ReadOnly; use crate::prelude::*; -use crate::service::config::GetConfig; -use crate::service::dependencies::DependencyConfig; use crate::service::transition::{TransitionKind, TransitionState}; use crate::service::ServiceActor; use crate::util::actor::background::BackgroundJobQueue; @@ -29,7 +27,10 @@ impl Handler for ServiceActor { let state = self.0.persistent_container.state.clone(); let transition = RemoteCancellable::new( async move { - let backup_guard = seed.persistent_container.mount_backup(path).await?; + let backup_guard = seed + .persistent_container + .mount_backup(path, ReadOnly) + .await?; seed.persistent_container .execute(ProcedureName::RestoreBackup, Value::Null, None) .await?; diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/Overlay.ts index fecb718d5..0e2dec58e 100644 --- a/sdk/lib/util/Overlay.ts +++ b/sdk/lib/util/Overlay.ts @@ -55,6 +55,17 @@ export class Overlay { ]) } else if (options.type === "pointer") { await this.effects.mount({ location: path, target: options }) + } else if (options.type === "backup") { + const subpath = options.subpath + ? options.subpath.startsWith("/") + ? options.subpath + : `/${options.subpath}` + : "/" + await execFile("mount", [ + "--bind", + `/media/startos/backup${subpath}`, + path, + ]) } else { throw new Error(`unknown type ${(options as any).type}`) } @@ -188,6 +199,7 @@ export type MountOptions = | MountOptionsVolume | MountOptionsAssets | MountOptionsPointer + | MountOptionsBackup export type MountOptionsVolume = { type: "volume" @@ -209,3 +221,8 @@ export type MountOptionsPointer = { subpath: string | null readonly: boolean } + +export type MountOptionsBackup = { + type: "backup" + subpath: string | null +} From 033defc943c6abe6cabc22742d2fc35eac3dd442 Mon Sep 17 00:00:00 2001 From: J H Date: Mon, 6 May 2024 11:45:18 -0600 Subject: [PATCH 6/8] fix: Allow compat docker style to run again --- core/startos/src/service/persistent_container.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index f9012c7be..6d0ed4b42 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -241,13 +241,13 @@ impl PersistentContainer { .rootfs_dir() .join("media/startos/backup"); tokio::fs::create_dir_all(&mountpoint).await?; - let bind = Bind::new(&backup_path); - let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await; Command::new("chown") .arg("100000:100000") .arg(mountpoint.as_os_str()) .invoke(ErrorKind::Filesystem) .await?; + let bind = Bind::new(&backup_path); + let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await; Command::new("chown") .arg("100000:100000") .arg(backup_path.as_os_str()) From 5c413df3ff35dc1299637c64fe152f7e047fc3b0 Mon Sep 17 00:00:00 2001 From: J H Date: Mon, 6 May 2024 12:42:57 -0600 Subject: [PATCH 7/8] fix: Backup for the js side --- .../SystemForEmbassy/polyfillEffects.ts | 31 +++++++++++++++++++ container-runtime/src/Models/Volume.ts | 9 ++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index ab6cb1c64..66a303e7b 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -267,6 +267,7 @@ export class PolyfillEffects implements oet.Effects { json: () => fetched.json(), } } + runRsync(rsyncOptions: { srcVolume: string dstVolume: string @@ -277,6 +278,36 @@ export class PolyfillEffects implements oet.Effects { id: () => Promise wait: () => Promise progress: () => Promise + } { + let secondRun: ReturnType | undefined + let firstRun = this._runRsync(rsyncOptions) + let waitValue = firstRun.wait().then((x) => { + secondRun = this._runRsync(rsyncOptions) + return secondRun.wait() + }) + const id = async () => { + return secondRun?.id?.() ?? firstRun.id() + } + const wait = () => waitValue + const progress = async () => { + const secondProgress = secondRun?.progress?.() + if (secondProgress) { + return (await secondProgress) / 2.0 + 0.5 + } + return (await firstRun.progress()) / 2.0 + } + return { id, wait, progress } + } + _runRsync(rsyncOptions: { + srcVolume: string + dstVolume: string + srcPath: string + dstPath: string + options: oet.BackupOptions + }): { + id: () => Promise + wait: () => Promise + progress: () => Promise } { const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions const command = "rsync" diff --git a/container-runtime/src/Models/Volume.ts b/container-runtime/src/Models/Volume.ts index ebf013b68..061bb1fd0 100644 --- a/container-runtime/src/Models/Volume.ts +++ b/container-runtime/src/Models/Volume.ts @@ -1,14 +1,17 @@ import * as fs from "node:fs/promises" +export const BACKUP = "backup" export class Volume { readonly path: string constructor( readonly volumeId: string, _path = "", ) { - const path = (this.path = `/media/startos/volumes/${volumeId}${ - !_path ? "" : `/${_path}` - }`) + if (volumeId.toLowerCase() === BACKUP) { + this.path = `/media/startos/backup${!_path ? "" : `/${_path}`}` + } else { + this.path = `/media/startos/volumes/${volumeId}${!_path ? "" : `/${_path}`}` + } } async exists() { return fs.stat(this.path).then( From 549891790d3cb5108fd1f93d7de142ff412757a6 Mon Sep 17 00:00:00 2001 From: J H Date: Mon, 6 May 2024 13:19:22 -0600 Subject: [PATCH 8/8] chore: Update some of the callbacks --- container-runtime/src/Adapters/RpcListener.ts | 2 +- container-runtime/src/Models/CallbackHolder.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index db5b786fc..faff253fe 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -81,7 +81,7 @@ const callbackType = object({ id: idType, method: literal("callback"), params: object({ - callback: string, + callback: number, args: array, }), }) diff --git a/container-runtime/src/Models/CallbackHolder.ts b/container-runtime/src/Models/CallbackHolder.ts index b562e8dd0..6539dda88 100644 --- a/container-runtime/src/Models/CallbackHolder.ts +++ b/container-runtime/src/Models/CallbackHolder.ts @@ -2,16 +2,16 @@ export class CallbackHolder { constructor() {} private root = (Math.random() + 1).toString(36).substring(7) private inc = 0 - private callbacks = new Map() + private callbacks = new Map() private newId() { - return this.root + (this.inc++).toString(36) + return this.inc++ } addCallback(callback: Function) { const id = this.newId() this.callbacks.set(id, callback) return id } - callCallback(index: string, args: any[]): Promise { + callCallback(index: number, args: any[]): Promise { const callback = this.callbacks.get(index) if (!callback) throw new Error(`Callback ${index} does not exist`) this.callbacks.delete(index)