fix: use bind mount - Failed at step STATE_DIRECTORY

Author: ryan4yin

hosts/idols-aquamarine/monitoring/module/default.nix

@@ -8,11 +8,10 @@ with lib; let
   cfg = config.services.my-victoriametrics;
   settingsFormat = pkgs.formats.yaml {};
 
-  workingDir = "/var/lib/" + cfg.stateDir;
   startCLIList =
     [
       "${cfg.package}/bin/victoria-metrics"
-      "-storageDataPath=${workingDir}"
+      "-storageDataPath=/var/lib/${cfg.stateDir}"
       "-httpListenAddr=${cfg.listenAddress}"
       "-retentionPeriod=${cfg.retentionPeriod}"
     ]
@@ -133,12 +132,10 @@ in {
 
         DynamicUser = true;
         User = "victoriametrics";
-        Group = "victoriametrics";
         RestartSec = 1;
         Restart = "on-failure";
         RuntimeDirectory = "victoriametrics";
         RuntimeDirectoryMode = "0700";
-        WorkingDirectory = workingDir;
         StateDirectory = cfg.stateDir;
         StateDirectoryMode = "0700";
 

hosts/idols-aquamarine/monitoring/victoriametrics.nix

@@ -3,13 +3,23 @@
   myvars,
   ...
 }: {
+  # Since victoriametrics use DynamicUser, the user & group do not exists before the service starts.
+  # this group is used as a supplementary Unix group for the service to access our data dir(/data/apps/xxx)
+  users.groups.victoriametrics-data = {};
+
   # Workaround for victoriametrics to store data in another place
   # https://www.freedesktop.org/software/systemd/man/latest/tmpfiles.d.html#Type
   systemd.tmpfiles.rules = [
-    "D /data/apps/victoriametrics 0751 victoriametrics victoriametrics - -"
-    "L+ /var/lib/victoriametrics - - - - /data/apps/victoriametrics"
+    "D /data/apps/victoriametrics 0770 root victoriametrics-data - -"
   ];
 
+  # Symlinks do not work with DynamicUser, so we should use bind mount here.
+  # https://github.com/systemd/systemd/issues/25097#issuecomment-1929074961
+  systemd.services.victoriametrics.serviceConfig = {
+    SupplementaryGroups = ["victoriametrics-data"];
+    BindPaths = ["/data/apps/victoriametrics:/var/lib/victoriametrics:rbind"];
+  };
+
   # https://victoriametrics.io/docs/victoriametrics/latest/configuration/configuration/
   services.my-victoriametrics = {
     enable = true;