Feature/backup+restore (#2613)

* feat: Implementation on the backup for the service.

* wip: Getting the flow of backup/restore

* feat: Recover

* Feature: Commit the full pass on the backup restore.

* use special type for backup instead of special id (#2614)

* fix: Allow compat docker style to run again

* fix: Backup for the js side

* chore: Update some of the callbacks

---------

Co-authored-by: Aiden McClelland <[email protected]>

container-runtime/src/Adapters/RpcListener.ts

@@ -81,7 +81,7 @@ const callbackType = object({
   id: idType,
   method: literal("callback"),
   params: object({
-    callback: string,
+    callback: number,
     args: array,
   }),
 })

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -68,7 +68,7 @@ export class DockerProcedureContainer {
             },
           })
         } else if (volumeMount.type === "backup") {
-          throw new Error("TODO")
+          await overlay.mount({ type: "backup", subpath: null }, mounts[mount])
         }
       }
     }

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -145,6 +145,15 @@ export class MainLoop {
               ...actionProcedure.args,
               JSON.stringify(timeChanged),
             ])
+            if (executed.exitCode === 0) {
+              await effects.setHealth({
+                id: healthId,
+                name: value.name,
+                result: "success",
+                message: actionProcedure["success-message"],
+              })
+              return
+            }
             if (executed.exitCode === 59) {
               await effects.setHealth({
                 id: healthId,

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -399,11 +399,10 @@ export class SystemForEmbassy implements System {
   ): Promise<void> {
     const backup = this.manifest.backup.create
     if (backup.type === "docker") {
-      const container = await DockerProcedureContainer.of(
-        effects,
-        backup,
-        this.manifest.volumes,
-      )
+      const container = await DockerProcedureContainer.of(effects, backup, {
+        ...this.manifest.volumes,
+        BACKUP: { type: "backup", readonly: false },
+      })
       await container.execFail([backup.entrypoint, ...backup.args], timeoutMs)
     } else {
       const moduleCode = await this.moduleCode
@@ -421,7 +420,10 @@ export class SystemForEmbassy implements System {
       const container = await DockerProcedureContainer.of(
         effects,
         restoreBackup,
-        this.manifest.volumes,
+        {
+          ...this.manifest.volumes,
+          BACKUP: { type: "backup", readonly: true },
+        },
       )
       await container.execFail(
         [restoreBackup.entrypoint, ...restoreBackup.args],
@@ -664,46 +666,6 @@ export class SystemForEmbassy implements System {
     }
     throw new Error(`Unknown type in the fetch properties: ${setConfigValue}`)
   }
-  private async health(
-    effects: HostSystemStartOs,
-    healthId: string,
-    timeSinceStarted: unknown,
-    timeoutMs: number | null,
-  ): Promise<void> {
-    const healthProcedure = this.manifest["health-checks"][healthId]
-    if (!healthProcedure) return
-    if (healthProcedure.type === "docker") {
-      const container = await DockerProcedureContainer.of(
-        effects,
-        healthProcedure,
-        this.manifest.volumes,
-      )
-      return JSON.parse(
-        (
-          await container.execFail(
-            [
-              healthProcedure.entrypoint,
-              ...healthProcedure.args,
-              JSON.stringify(timeSinceStarted),
-            ],
-            timeoutMs,
-          )
-        ).stdout.toString(),
-      )
-    } else if (healthProcedure.type === "script") {
-      const moduleCode = await this.moduleCode
-      const method = moduleCode.health?.[healthId]
-      if (!method) throw new Error("Expecting that the method health exists")
-      await method(
-        new PolyfillEffects(effects, this.manifest),
-        Number(timeSinceStarted),
-      ).then((x) => {
-        if ("result" in x) return x.result
-        if ("error" in x) throw new Error("Error getting config: " + x.error)
-        throw new Error("Error getting config: " + x["error-code"][1])
-      })
-    }
-  }
   private async action(
     effects: HostSystemStartOs,
     actionId: string,

container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts

@@ -57,6 +57,7 @@ export const matchManifest = object(
         matchProcedure,
         object({
           name: string,
+          ["success-message"]: string,
         }),
       ),
     ]),

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -267,6 +267,7 @@ export class PolyfillEffects implements oet.Effects {
       json: () => fetched.json(),
     }
   }
+
   runRsync(rsyncOptions: {
     srcVolume: string
     dstVolume: string
@@ -277,6 +278,36 @@ export class PolyfillEffects implements oet.Effects {
     id: () => Promise<string>
     wait: () => Promise<null>
     progress: () => Promise<number>
+  } {
+    let secondRun: ReturnType<typeof this._runRsync> | undefined
+    let firstRun = this._runRsync(rsyncOptions)
+    let waitValue = firstRun.wait().then((x) => {
+      secondRun = this._runRsync(rsyncOptions)
+      return secondRun.wait()
+    })
+    const id = async () => {
+      return secondRun?.id?.() ?? firstRun.id()
+    }
+    const wait = () => waitValue
+    const progress = async () => {
+      const secondProgress = secondRun?.progress?.()
+      if (secondProgress) {
+        return (await secondProgress) / 2.0 + 0.5
+      }
+      return (await firstRun.progress()) / 2.0
+    }
+    return { id, wait, progress }
+  }
+  _runRsync(rsyncOptions: {
+    srcVolume: string
+    dstVolume: string
+    srcPath: string
+    dstPath: string
+    options: oet.BackupOptions
+  }): {
+    id: () => Promise<string>
+    wait: () => Promise<null>
+    progress: () => Promise<number>
   } {
     const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions
     const command = "rsync"

container-runtime/src/Models/CallbackHolder.ts

@@ -2,16 +2,16 @@ export class CallbackHolder {
   constructor() {}
   private root = (Math.random() + 1).toString(36).substring(7)
   private inc = 0
-  private callbacks = new Map<string, Function>()
+  private callbacks = new Map<number, Function>()
   private newId() {
-    return this.root + (this.inc++).toString(36)
+    return this.inc++
   }
   addCallback(callback: Function) {
     const id = this.newId()
     this.callbacks.set(id, callback)
     return id
   }
-  callCallback(index: string, args: any[]): Promise<unknown> {
+  callCallback(index: number, args: any[]): Promise<unknown> {
     const callback = this.callbacks.get(index)
     if (!callback) throw new Error(`Callback ${index} does not exist`)
     this.callbacks.delete(index)

container-runtime/src/Models/Volume.ts

@@ -1,14 +1,17 @@
 import * as fs from "node:fs/promises"
 
+export const BACKUP = "backup"
 export class Volume {
   readonly path: string
   constructor(
     readonly volumeId: string,
     _path = "",
   ) {
-    const path = (this.path = `/media/startos/volumes/${volumeId}${
-      !_path ? "" : `/${_path}`
-    }`)
+    if (volumeId.toLowerCase() === BACKUP) {
+      this.path = `/media/startos/backup${!_path ? "" : `/${_path}`}`
+    } else {
+      this.path = `/media/startos/volumes/${volumeId}${!_path ? "" : `/${_path}`}`
+    }
   }
   async exists() {
     return fs.stat(this.path).then(

core/startos/src/backup/backup_bulk.rs

@@ -13,7 +13,7 @@ use tokio::io::AsyncWriteExt;
 use tracing::instrument;
 use ts_rs::TS;
 
-use super::target::BackupTargetId;
+use super::target::{BackupTargetId, PackageBackupInfo};
 use super::PackageBackupReport;
 use crate::auth::check_password_against_db;
 use crate::backup::os::OsBackup;
@@ -246,19 +246,43 @@ async fn perform_backup(
     backup_guard: BackupMountGuard<TmpMountGuard>,
     package_ids: &OrdSet<PackageId>,
 ) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
+    let db = ctx.db.peek().await;
     let mut backup_report = BTreeMap::new();
     let backup_guard = Arc::new(backup_guard);
+    let mut package_backups: BTreeMap<PackageId, PackageBackupInfo> =
+        backup_guard.metadata.package_backups.clone();
 
     for id in package_ids {
         if let Some(service) = &*ctx.services.get(id).await {
+            let backup_result = service
+                .backup(backup_guard.package_backup(id))
+                .await
+                .err()
+                .map(|e| e.to_string());
+            if backup_result.is_none() {
+                let manifest = db
+                    .as_public()
+                    .as_package_data()
+                    .as_idx(id)
+                    .or_not_found(id)?
+                    .as_state_info()
+                    .expect_installed()?
+                    .as_manifest();
+
+                package_backups.insert(
+                    id.clone(),
+                    PackageBackupInfo {
+                        os_version: manifest.as_os_version().de()?,
+                        version: manifest.as_version().de()?,
+                        title: manifest.as_title().de()?,
+                        timestamp: Utc::now(),
+                    },
+                );
+            }
             backup_report.insert(
                 id.clone(),
                 PackageBackupReport {
-                    error: service
-                        .backup(backup_guard.package_backup(id))
-                        .await
-                        .err()
-                        .map(|e| e.to_string()),
+                    error: backup_result,
                 },
             );
         }
@@ -298,7 +322,7 @@ async fn perform_backup(
     }
     let luks_folder = Path::new("/media/startos/config/luks");
     if tokio::fs::metadata(&luks_folder).await.is_ok() {
-        dir_copy(&luks_folder, &luks_folder_bak, None).await?;
+        dir_copy(luks_folder, &luks_folder_bak, None).await?;
     }
 
     let timestamp = Some(Utc::now());
@@ -307,8 +331,9 @@ async fn perform_backup(
     backup_guard.unencrypted_metadata.full = true;
     backup_guard.metadata.version = crate::version::Current::new().semver().into();
     backup_guard.metadata.timestamp = timestamp;
+    backup_guard.metadata.package_backups = package_backups;
 
-    backup_guard.save_and_unmount().await?;
+    backup_guard.save().await?;
 
     ctx.db
         .mutate(|v| {

core/startos/src/service/mod.rs

@@ -10,7 +10,7 @@ use persistent_container::PersistentContainer;
 use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor};
 use serde::{Deserialize, Serialize};
 use start_stop::StartStop;
-use tokio::sync::Notify;
+use tokio::{fs::File, sync::Notify};
 use ts_rs::TS;
 
 use crate::context::{CliContext, RpcContext};
@@ -296,13 +296,20 @@ impl Service {
     }
 
     pub async fn restore(
-        _ctx: RpcContext,
-        _s9pk: S9pk,
-        _guard: impl GenericMountGuard,
-        _progress: Option<InstallProgressHandles>,
+        ctx: RpcContext,
+        s9pk: S9pk,
+        backup_source: impl GenericMountGuard,
+        progress: Option<InstallProgressHandles>,
     ) -> Result<Self, Error> {
-        // TODO
-        Err(Error::new(eyre!("not yet implemented"), ErrorKind::Unknown))
+        let service = Service::install(ctx.clone(), s9pk, None, progress).await?;
+
+        service
+            .actor
+            .send(transition::restore::Restore {
+                path: backup_source.path().to_path_buf(),
+            })
+            .await?;
+        Ok(service)
     }
 
     pub async fn shutdown(self) -> Result<(), Error> {
@@ -348,9 +355,23 @@ impl Service {
         Ok(())
     }
 
-    pub async fn backup(&self, _guard: impl GenericMountGuard) -> Result<BackupReturn, Error> {
-        // TODO
-        Err(Error::new(eyre!("not yet implemented"), ErrorKind::Unknown))
+    #[instrument(skip_all)]
+    pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> {
+        let id = &self.seed.id;
+        let mut file = File::create(guard.path().join(id).with_extension("s9pk")).await?;
+        self.seed
+            .persistent_container
+            .s9pk
+            .clone()
+            .serialize(&mut file, true)
+            .await?;
+        drop(file);
+        self.actor
+            .send(transition::backup::Backup {
+                path: guard.path().to_path_buf(),
+            })
+            .await?;
+        Ok(())
     }
 
     pub fn container_id(&self) -> Result<ContainerId, Error> {
@@ -425,6 +446,7 @@ impl Actor for ServiceActor {
                         kinds.running_status,
                     ) {
                         (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting,
+                        (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring,
                         (Some(TransitionKind::BackingUp), _, Some(status)) => {
                             MainStatus::BackingUp {
                                 started: Some(status.started),

core/startos/src/service/persistent_container.rs

@@ -1,5 +1,5 @@
 use std::collections::BTreeMap;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
@@ -222,6 +222,40 @@ impl PersistentContainer {
         })
     }
 
+    #[instrument(skip_all)]
+    pub async fn mount_backup(
+        &self,
+        backup_path: impl AsRef<Path>,
+        mount_type: MountType,
+    ) -> Result<MountGuard, Error> {
+        let backup_path: PathBuf = backup_path.as_ref().to_path_buf();
+        let mountpoint = self
+            .lxc_container
+            .get()
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("PersistentContainer has been destroyed"),
+                    ErrorKind::Incoherent,
+                )
+            })?
+            .rootfs_dir()
+            .join("media/startos/backup");
+        tokio::fs::create_dir_all(&mountpoint).await?;
+        Command::new("chown")
+            .arg("100000:100000")
+            .arg(mountpoint.as_os_str())
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        let bind = Bind::new(&backup_path);
+        let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await;
+        Command::new("chown")
+            .arg("100000:100000")
+            .arg(backup_path.as_os_str())
+            .invoke(ErrorKind::Filesystem)
+            .await?;
+        mount_guard
+    }
+
     #[instrument(skip_all)]
     pub async fn init(&self, seed: Weak<ServiceActorSeed>) -> Result<(), Error> {
         let socket_server_context = EffectContext::new(seed);