Noodling around with a borg/borgmatic role

ansible/Makefile

@@ -9,3 +9,6 @@ setup:
 
 maintenance:
 	ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="maintenance"
+
+borg:
+	ansible-playbook -v -i inventory.yml --ask-vault-pass playbook.yml --tags="borg"

ansible/inventory.yml

@@ -1,18 +1,27 @@
 $ANSIBLE_VAULT;1.1;AES256
-34336239663063393937336138613833663531336632313232623866663935663362313833323664
-3130363533643430623433623763323335636364646161300a366564663834303664653761366638
-63343262306439303162333365316138613939353737613937373766313661643839393565333261
-3334376466653331360a616434636631613366326661316132376336333530653836306234303731
-64346436343832636264613165383031393239303732373061393132613430636465393033396461
-36323964323962326262386566643531396133353636333438343136373738373761336335646131
-66353735666666366261393131373263393738613136663236623938636332383039323538393737
-30646135343761366466646234386639333965343632393264376166333364333930393564323033
-39393135626365346264646638343030616335396333303035356539626535633363336666353762
-39613436303862356363383962616237656632356438303134323237353637333461326436643766
-35326461383133666164366539356231366236643532306337303535323536363466646239363562
-63356633393963313765326461656539333435343934383064656561666333653731636539653630
-62626166373335653461333164633535396566393031353235316263623964313532333662313133
-66343931326638346164363563626430623462643836636534363438343935653463613034306230
-31346266363933383731626166646366363863663362316362616162346330333234643832336638
-64383234653563366131363462383763653661646162336565643139653461313562323861373638
-33343439356232363138316361303966646231623034313165303338653063633064
+66333666636462666636653565366537326262313132313537326531306538343339663564373538
+3738633435613030303533346133386331373433386161360a663439653439383731343233633035
+63396532616136383466323633613039313937633431313466616337653366383163363763643737
+6532333834666435350a333065636538333264303664613130333230313432383233396664663135
+62656539353562386236633134643439373930396131303065663134653835353936353364303864
+61626163653730386632303662303335376534313038613237333633653031626335383637336637
+35636331343737623161623531313065323133666531646566323663616566323035653465623934
+38313131393965306334663766333934383064346535376533366461356366623539363230383038
+34653232666235646430393366343733333961366234313333656637633734633762656434356565
+35626236663963303063313139353434343837303765623733373531376466663365386639663037
+62383231623938393935376466313061336332656337316234383237663663336634383164343937
+61616166343332343735396334633438626561383465336664653138353737616539636135356665
+30663331653030393261343235623164366564376166373839366338373139656335623130323361
+37636661333937666565636163616639636163646434636538653930663530396663326638613138
+64633838306161376533363138333131363830633933353038353233613333343234616361303030
+34653438376430623538343135643633323133383731323633646439633931333637386434343337
+62616138613435393336313333333666383161376261316265356233396337326561653431636239
+37343938363063326531356135363733326463303937656562373638366336653537336466306537
+35353538343832343738346537353036633235636633366231323832363533643636306636383061
+36633766383638346264306533663965373636303034623166656538663634356230333133613537
+32663261313935373430333038333833326336613663623531393933326533326661386131313561
+32633935653062333836653833373932623039303664383362643365326166336661333764363333
+61623566376538656339313337666438623331646562353265353133386433383633386430626538
+66613531633661333435333065633539363032636262366537666533643133313838613761303465
+33626566623031313530396163376235376162303239336531333437613363633738363932663866
+6635303335663362383335653765323861303933396264383263

ansible/playbook.yml

@@ -10,11 +10,13 @@
       tags: [setup]
     - role: datalab
       tags: [deploy]
+    - role: borg
+      tags: [borg, setup]
     - role: nginx
-      tags: [setup, maintenance]
+      tags: [nginx, deploy, setup]
 
   tasks:
     - name: Keep all packages up-to-date
       ansible.builtin.include_role:
         name: apt_upgrade
-      tags: [setup, maintenance]
+      tags: [maintenance]

ansible/requirements.yml

@@ -6,3 +6,8 @@ collections:
     version: 1.5.4
   - name: community.general
     version: 8.2.0
+  - name: community.crypto
+    version: 2.21.1
+roles:
+  - name: borgbase.ansible_role_borgbackup
+    version: 1.0.1

ansible/roles/borg/defaults/main.yml

@@ -0,0 +1,45 @@
+---
+borg_encryption_passphrase: ''
+borg_exclude_patterns: []
+borg_one_file_system: true
+borg_exclude_from: []
+borg_encryption_passcommand: false
+borg_lock_wait_time: 5
+borg_ssh_key_type: "ed25519"
+borg_ssh_key_name: "id_{{ borg_ssh_key_type }}"
+borg_ssh_key_file_path: "{{ backup_user_info.home }}/.ssh/{{ borg_ssh_key_name }}"
+borg_ssh_command: false
+borg_remote_path: false
+borg_remote_rate_limit: 0
+borg_retention_policy:
+  keep_hourly: 3
+  keep_daily: 7
+  keep_weekly: 4
+  keep_monthly: 6
+borg_version: false
+borgmatic_timer_cron_name: "borgmatic"
+borgmatic_timer: cron
+borgmatic_timer_hour: "{{ range(0, 5) | random(seed=inventory_hostname) }}"
+borgmatic_timer_minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
+borg_install_method: "pip"
+borg_require_epel: "{{ ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' }}"
+
+borgmatic_hooks:
+  on_error:
+    - echo "`date` - Error while creating a backup."
+  before_backup:
+    - echo "`date` - Starting backup."
+  after_backup:
+    - echo "`date` - Finished backup."
+borgmatic_checks:
+  - name: repository
+    frequency: "4 weeks"
+  - name: archives
+    frequency: "6 weeks"
+borgmatic_check_last: 3
+borgmatic_store_atime: true
+borgmatic_store_ctime: true
+borgmatic_relocated_repo_access_is_ok: false
+borgmatic_version: ">=1.7.11"
+
+borg_venv_path: "/opt/borgmatic"

ansible/roles/borg/tasks/main.yml

@@ -0,0 +1,74 @@
+---
+  - name: Install borg
+    become: true
+    apt:
+      name: 
+      - borgbackup
+      state: present
+
+  - name: Install pipx
+    pip:
+      name: pipx
+      state: present
+
+  - name: Install borgmatic with pipx
+    community.general.pipx:
+      name: borgmatic
+      state: present
+
+  - name: Install MongoDB tools
+    become: true
+    apt:
+      deb: https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.10.0.deb
+
+  - name: Ensure /etc/borgmatic exists
+    become: true
+    ansible.builtin.file:
+      path: /etc/borgmatic
+      state: directory
+      mode: "0700"
+      owner: "{{ ansible_ssh_user }}"
+
+  - name: Add Borgmatic configuration
+    become: true
+    ansible.builtin.template:
+      src: config.yaml.j2
+      dest: "/etc/borgmatic/config.yaml"
+      mode: "0600"
+      owner: "{{ ansible_ssh_user }}"
+
+    vars:
+      borg_exclude_patterns: []
+      borg_exclude_from: []
+      borg_install_method: "package"
+      borg_user: "{{ ansible_user }}"
+      borg_source_directories:
+        - /data
+      borgmatic_hooks:
+        before_backup:
+          - echo "`date` - Starting backup."
+        mongodb_databases:
+          - name: all 
+            port: 27017
+            options: "--ssl"
+      borgmatic_timer: cron
+      borg_retention_policy:
+        keep_daily: 30
+        keep_weekly: 0
+        keep_monthly: 12
+        keep_yearly: 4
+      borg_one_file_system: true
+      borgmatic_store_atime: true
+      borgmatic_store_ctime: true
+      borg_encryption_passcommand: false
+      borg_remote_rate_limit: 0
+      borg_ssh_command: ssh
+      borg_lock_wait_time: 5
+
+  - name: Add Cron job for borgmatic
+    cron:
+      name: "borgmatic" 
+      hour: "2"
+      minute: "{{ range(0, 59) | random(seed=inventory_hostname) }}"
+      user: "{{ ansible_user }}"
+      job: "borgmatic -c /etc/borgmatic/config.yaml"

ansible/roles/borg/templates/config.yaml.j2

@@ -0,0 +1,183 @@
+#jinja2: lstrip_blocks: "True", trim_blocks: "True"
+---
+# Managed by Ansible, please don't edit manually
+
+# Full config: https://torsion.org/borgmatic/docs/reference/config.yaml
+location:
+{% if borg_source_directories is not defined or borg_source_directories | length == 0 %}
+    source_directories:
+        - /etc/hostname # prevent empty backupconfig
+{% else %}
+    source_directories:
+    {% for dir in borg_source_directories %}
+        - {{ dir }}
+    {% endfor %}
+{% endif %}
+
+    # Stay in same file system (do not cross mount points).
+    one_file_system: {{ borg_one_file_system }}
+    repositories:
+{% if borg_repository is iterable and (borg_repository is not string and borg_repository is not mapping) %}
+  {% for repo in borg_repository %}
+        - {{ repo }}
+  {% endfor %}
+{% elif borg_repository is defined and borg_repository is string %}
+        - {{ borg_repository }}
+{% endif %}
+
+    # Store atime into archive.
+    atime: {{ borgmatic_store_atime }}
+
+    # Store ctime into archive.
+    ctime: {{ borgmatic_store_ctime }}
+
+{% if borg_exclude_patterns %}
+    # Any paths matching these patterns are excluded from backups. Globs and tildes
+    # are expanded. See the output of "borg help patterns" for more details.
+    exclude_patterns:
+{% for dir in borg_exclude_patterns %}
+        - '{{ dir }}'
+{% endfor %}
+{% endif %}
+{% if borg_exclude_from %}
+    # Read exclude patterns from one or more separate named files, one pattern per
+    # line. See the output of "borg help patterns" for more details.
+    exclude_from:
+{% for dir in borg_exclude_from %}
+        - {{ dir }}
+{% endfor %}
+{% endif %}
+
+    # Exclude directories that contain a CACHEDIR.TAG file. See
+    # http://www.brynosaurus.com/cachedir/spec.html for details.
+    exclude_caches: true
+
+    # Exclude directories that contain a file with the given filename.
+    exclude_if_present: .nobackup
+
+    # Alternate Borg remote executable. Defaults to "borg".
+    # remote_path: borg1
+{% if borg_remote_path %}
+    remote_path: {{ borg_remote_path }}
+{% endif %}
+
+# Repository storage options. See
+# https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
+# https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
+# details.
+storage:
+{% if borg_encryption_passphrase %}
+    encryption_passphrase: {{ borg_encryption_passphrase }}
+
+{% endif %}
+    # The standard output of this command is used to unlock the encryption key. Only
+    # use on repositories that were initialized with passcommand/repokey encryption.
+    # Note that if both encryption_passcommand and encryption_passphrase are set,
+    # then encryption_passphrase takes precedence.
+    # encryption_passcommand: secret-tool lookup borg-repository repo-name
+{% if borg_encryption_passcommand %}
+    encryption_passcommand: {{ borg_encryption_passcommand }}
+{% endif %}
+
+    # Type of compression to use when creating archives. See
+    # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
+    # Defaults to no compression.
+    compression: auto,zstd
+
+    # Remote network upload rate limit in kiBytes/second.
+{% if borg_remote_rate_limit %}
+    remote_rate_limit: {{ borg_remote_rate_limit }}
+{% endif %}
+
+    # Command to use instead of just "ssh". This can be used to specify ssh options.
+    # ssh_command: ssh -i ~/.ssh/id_ed25519
+{% if borg_ssh_command %}
+    ssh_command: {{ borg_ssh_command }}
+{% endif %}
+
+    # Umask to be used for borg create.
+    umask: 0077
+
+    # Maximum seconds to wait for acquiring a repository/cache lock.
+    lock_wait: {{ borg_lock_wait_time }}
+
+    # Name of the archive. Borg placeholders can be used. See the output of
+    # "borg help placeholders" for details. Default is
+    # "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
+    # also specify a prefix in the retention section to avoid accidental pruning of
+    # archives with a different archive name format. And you should also specify a
+    # prefix in the consistency section as well.
+    archive_name_format: '{hostname}-{now:%Y-%m-%d-%H%M%S}'
+
+    # Bypass Borg error about a repository that has been moved.
+    relocated_repo_access_is_ok: {{ borgmatic_relocated_repo_access_is_ok }}
+
+# Retention policy for how many backups to keep in each category. See
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+# At least one of the "keep" options is required for pruning to work.
+retention:
+{% if borg_retention_policy.keep_secondly is defined %}
+    # Number of secondly archives to keep.
+    keep_secondly: {{ borg_retention_policy.keep_secondly }}
+{% endif %}
+
+{% if borg_retention_policy.keep_minutely is defined %}
+    # Number of minutely archives to keep.
+    keep_minutely: {{ borg_retention_policy.keep_minutely }}
+{% endif %}
+
+{% if borg_retention_policy.keep_hourly is defined %}
+    # Number of hourly archives to keep.
+    keep_hourly: {{ borg_retention_policy.keep_hourly }}
+{% endif %}
+
+{% if borg_retention_policy.keep_daily is defined %}
+    # Number of daily archives to keep.
+    keep_daily: {{ borg_retention_policy.keep_daily }}
+{% endif %}
+
+{% if borg_retention_policy.keep_weekly is defined %}
+    # Number of weekly archives to keep.
+    keep_weekly: {{ borg_retention_policy.keep_weekly }}
+{% endif %}
+
+{% if borg_retention_policy.keep_monthly is defined %}
+    # Number of monthly archives to keep.
+    keep_monthly: {{ borg_retention_policy.keep_monthly }}
+{% endif %}
+
+{% if borg_retention_policy.keep_yearly is defined %}
+    # Number of yearly archives to keep.
+    keep_yearly: {{ borg_retention_policy.keep_yearly }}
+{% endif %}
+
+# Consistency checks to run after backups. See
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
+consistency:
+    # List of one or more consistency checks to run: "repository",
+    # "archives", "data", and/or "extract". Defaults to
+    # "repository" and "archives". Set to "disabled" to disable
+    # all consistency checks. "repository" checks the consistency
+    # of the repository, "archives" checks all of the archives,
+    # "data" verifies the integrity of the data within the
+    # archives, and "extract" does an extraction dry-run of the
+    # most recent archive. Note that "data" implies "archives".
+    checks:
+    {% for checks in borgmatic_checks %}
+        - {{ checks }}
+    {% endfor %}
+
+    # Restrict the number of checked archives to the last n. Applies only to the "archives" check.
+    check_last: {{ borgmatic_check_last }}
+
+# Shell commands or scripts to execute before and after a backup or if an error has occurred.
+# IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
+# Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
+# prevent potential shell injection or privilege escalation.
+hooks:
+{% for hook in borgmatic_hooks %}
+    {{ hook }}:
+        {{ borgmatic_hooks[hook] | to_nice_yaml(indent=2) | trim | indent(8) }}
+{% endfor %}
+