From d6aa39ff04b3a5f24ba7785dfdf4bb13671bff5a Mon Sep 17 00:00:00 2001 From: Aner Zakobar Date: Wed, 29 Apr 2026 20:23:42 +0300 Subject: [PATCH] Added shell command for deploy, updated readme, backup script. --- README.org | 246 +++++++++++++++++++++----------------- scripts/offload-backup.sh | 118 ++++++++++++++++++ shells/defaultShell.nix | 5 + 3 files changed, 262 insertions(+), 107 deletions(-) create mode 100644 scripts/offload-backup.sh diff --git a/README.org b/README.org index 2db05ea..22f3818 100644 --- a/README.org +++ b/README.org @@ -108,21 +108,12 @@ nixos-rebuild switch \ The Pi builds its own config natively (no cross-compilation). sops-nix will now decrypt all secrets and start all services. -** Caddy plugin hash +You can also use the command: -The first deploy will fail at the Caddy build step because =lib.fakeHash= -is a placeholder. Copy the correct hash from the error output and replace -it in =modules/caddy.nix=: - -#+begin_src nix -caddyWithCloudflare = pkgs.caddy.withPlugins { - plugins = [ "github.com/caddy-dns/cloudflare@..." ]; - hash = "sha256-REPLACE_WITH_REAL_HASH="; # ← paste here -}; +#+begin_src bash +homey-deploy-rpi-main #+end_src -Then re-run the deploy command from Phase 3. - ** Ongoing deploys from workstation All future config changes follow the same pattern: @@ -131,24 +122,12 @@ All future config changes follow the same pattern: 2. Run: #+begin_src bash -nixos-rebuild switch \ - --flake .#pi-main \ - --target-host admin@192.168.1.100 \ - --build-host admin@192.168.1.100 \ - --use-remote-sudo +homey-deploy-rpi-main #+end_src NixOS activates the new config on the Pi immediately, with an automatic rollback if activation fails. -* Installation (legacy Helm) - -Install using - -#+begin_src bash -helm upgrade --install homey . -n homey -#+end_src - * Backing up Backups use [[https://restic.net/][restic]] and run automatically via systemd on a daily schedule. @@ -203,98 +182,151 @@ restic -r s3:https://... restore latest --target /mnt/data restic -r s3:https://... restore latest --target /mnt/data --include /mnt/data/gitea #+end_src -* LDAP Configuration +* Disaster Recovery -Logins are done to PHPLDAPADMIN +Full recovery from total host failure (dead Pi, dead SD card), assuming this +git repo and your workstation PGP key (=076AA297579A0064=) survive. -DN is like: +** Step 1 — Flash and boot a new Pi -cn=admin,dc=,dc=io -get-secret-val.sh homey openldap-admin password +Follow Phase 1 above to build and flash a fresh bootstrap image, then SSH in. -First thing we do is create an organization unit called users +** Step 2 — Regenerate the age key and re-encrypt secrets -To add a new user, we create a child entry to ou=users +The old Pi's age key is gone with the dead machine. Your workstation PGP key +is the fallback and can still decrypt =secrets/secrets.yaml=. -It has to be of type inetOrgPerson +On the Pi: -cn = Common Name, sn = Sur Name. -Select RDN = User Name (uid) (FROM DROP DOWN MENU) -UID = USERNAME, that is what is important. (In PHPLdapAdmin it is under User Name) - -Now we may continue! - -* GITEA - -Site Title: whatever - -SSH Server Domain: git. -SSH Server Port: 2222 -Gitea Base URL: http://git. - -Then add Administrator Account Settings: - -Administrator Username: gitea-admin -Password: from gitea-admin-pass -Email address must be populated - -That will work after a few minutes. - -Now we go into Authentication Sources - -Add a new LDAP Authentication source - -Authentication name: Home LDAP -Host: openldap -Port: 389 -Bind DN = cn=readonly,dc=,dc=io -Bind Password: openldap-ro password -User Search Base: ou=users,dc=,dc=io -user search filter = (uid=%s) -Admin filter (title=admin) -Username Attribute: uid -First Name Attribute: cn -Surname Attribute: sn -Email Attribute: mail - -* AUTHELIA - -https://github.com/authelia/authelia/blob/57d5fbd3f5c82e83296023dc1de6e4f5ff063c00/examples/compose/lite/authelia/configuration.yml -This fucking sucks -https://gist.github.com/james-d-elliott/5152d27c0781aee856a3383f1284998e - -* EVERYTHING -https://www.talkingquickly.co.uk/gitea-sso-with-keycloak-openldap-openid-connect - -* DRONE AND GITEA -? -https://dev.to/ruanbekker/self-hosted-cicd-with-gitea-and-drone-ci-200l - -* DAV - -https://gitlab.com/davical-project/davical/-/blob/master/config/example-config.php - -Line 800 ish for auth from reverse proxy - -* NEXTCLOUD - -I ran THIS command inside -su www-data -s /bin/bash -c php occ ldap:promote-group "admins" - -** When maintenence mode - -#+begin_example -kubectl exec --tty --stdin -n homey deploy/nextcloud -- su -l www-data -s /bin/bash -php /var/www/html/occ maintenance:mode --off +#+begin_src bash +sudo age-keygen -o /var/lib/sops-nix/key.txt +sudo age-keygen -y /var/lib/sops-nix/key.txt # copy this public key #+end_src -* I UNDERSTAND +On the workstation — replace the old age key in =secrets/.sops.yaml= with the +new public key, then re-encrypt: -I need to backup Chen's stuff -And... I need to Jellyfin +#+begin_src bash +sops updatekeys secrets/secrets.yaml +git add secrets/.sops.yaml secrets/secrets.yaml +git commit -m "replace Pi age key after host failure" +#+end_src -* PAPERLESS +** Step 3 — Deploy the full NixOS config -https://github.com/paperless-ngx/paperless-ngx/blob/74c44fe418a91a526b5dab1a91fde4aaebd28bb1/docker/compose/docker-compose.postgres.yml +#+begin_src bash +nixos-rebuild switch \ + --flake .#pi-main \ + --target-host admin@192.168.1.100 \ + --build-host admin@192.168.1.100 \ + --use-remote-sudo +#+end_src + +This brings up the OS and mounts =/mnt/data=. Services will fail to start +until data is restored — that is expected. + +** Step 4 — Restore data from restic + +Credentials are in =secrets/secrets.yaml= (=restic/password=, +=restic/s3_access_key_id=, =restic/s3_secret_access_key=). + +#+begin_src bash +ssh admin@192.168.1.100 + +export RESTIC_REPOSITORY="s3:https://s3.us-east-005.backblazeb2.com/zakobar-home-backup" +export RESTIC_PASSWORD="..." # restic/password from secrets +export AWS_ACCESS_KEY_ID="..." # restic/s3_access_key_id +export AWS_SECRET_ACCESS_KEY="..." # restic/s3_secret_access_key + +restic snapshots # verify repo is reachable +sudo restic restore latest --target /mnt/data +#+end_src + +If restoring from a USB offload disk instead of S3: + +#+begin_src bash +sudo restic -r /mnt/usb/homey-backup restore latest --target /mnt/data +#+end_src + +** Step 5 — Restore the Nextcloud database + +The raw Postgres data dir is excluded from restic; only the =pg_dump= SQL file +is backed up. After the data restore you will have +=/mnt/data/nextcloud/db-dump/nextcloud.sql= but an empty database. Import it: + +#+begin_src bash +sudo systemctl start podman-nextcloud-postgres +# Wait ~10 s for Postgres to be ready, then: +podman exec -i nextcloud-postgres \ + psql -U postgres nextcloud_db \ + < /mnt/data/nextcloud/db-dump/nextcloud.sql +#+end_src + +** Step 6 — Start services and verify + +#+begin_src bash +sudo systemctl start podman-openldap podman-authelia podman-gitea podman-nextcloud +#+end_src + +Manual checks after restart: + +- *Gitea*: Admin → Authentication Sources — verify the LDAP source is present. + It lives in Gitea's database (restored from restic) so it should survive + automatically. Confirm by logging in with an LDAP user. +- *Nextcloud*: Admin → LDAP/AD Integration — confirm the LDAP app is still + configured. If not, re-enter the settings from the LDAP Configuration + section of this file. + +** Key risks + +| Risk | Consequence | +|------+-------------| +| External HD also fails | Restore all data from restic — Nextcloud files may be large | +| Workstation PGP key lost | Cannot decrypt =secrets/secrets.yaml= — passwords must be reset manually per service | +| USB offload not yet implemented | =scripts/offload-backup.sh= does not exist yet; S3 is the only working backup tier | + +* Running commands in containers + +All services run as podman containers. Use =podman exec= to run commands +inside them. + +** General pattern + +Containers are started by systemd as root, so they live in root's podman +context. All =podman= commands must be run with =sudo=. + +#+begin_src bash +# List running containers +sudo podman ps + +# Run a command in a container +sudo podman exec + +# Run as a specific user +sudo podman exec -u + +# Interactive shell +sudo podman exec -it sh +#+end_src + +Container names match the service: =openldap=, =authelia=, =gitea=, +=nextcloud=, =nextcloud-postgres=, =jellyfin=, =transmission=. + +** Nextcloud — running occ commands + +=occ= must run as =www-data= inside the =nextcloud= container. + +#+begin_src bash +# General form +sudo podman exec -u www-data nextcloud php occ + +# Examples +sudo podman exec -u www-data nextcloud php occ status +sudo podman exec -u www-data nextcloud php occ maintenance:mode --off +sudo podman exec -u www-data nextcloud php occ preview:generate-all -vvv +sudo podman exec -u www-data nextcloud php occ ldap:promote-group "admins" +#+end_src + +Running without =-u www-data= will create files owned by root inside the +container, which breaks Nextcloud's file access. -For docker diff --git a/scripts/offload-backup.sh b/scripts/offload-backup.sh new file mode 100644 index 0000000..63450a7 --- /dev/null +++ b/scripts/offload-backup.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# offload-backup.sh — back up /mnt/data directly to a USB drive using restic. +# +# Run on the Pi (see homey-offload-backup in the dev shell). +# Scans for plugged-in USB partitions, lets you pick one, mounts it if needed, +# initialises a restic repo on it, and runs a backup of all service data dirs. +# +# The restic password is read from the sops-managed secret at runtime; +# no S3 credentials are needed — this is a direct local backup. +# +# Usage: sudo bash offload-backup.sh + +set -euo pipefail + +REPO_NAME="homey-backup" +DATA_DIR="/mnt/data" +PASSWORD_FILE="/run/secrets/restic/password" + +BACKUP_PATHS=( + "$DATA_DIR/openldap" + "$DATA_DIR/authelia" + "$DATA_DIR/gitea" + "$DATA_DIR/nextcloud" + "$DATA_DIR/jellyfin" + "$DATA_DIR/transmission" +) + +EXCLUDE_ARGS=( + --exclude "$DATA_DIR/nextcloud/db" + --exclude "$DATA_DIR/restic-cache" +) + +# --------------------------------------------------------------------------- +# Find USB partitions +# --------------------------------------------------------------------------- +echo "Scanning for USB drives..." +mapfile -t USB_PARTS < <( + lsblk -o NAME,SIZE,TRAN,LABEL,MOUNTPOINT -rn \ + | awk '$3 == "usb" && $2 != "" {print $1, $2, $4, $5}' +) + +if [ "${#USB_PARTS[@]}" -eq 0 ]; then + echo "No USB partitions found. Plug in a USB drive and try again." >&2 + exit 1 +fi + +echo "" +echo "Available USB partitions:" +for i in "${!USB_PARTS[@]}"; do + read -r dev size label mount <<< "${USB_PARTS[$i]}" + label="${label:-(no label)}" + mount="${mount:-(not mounted)}" + printf " [%d] /dev/%s %s label=%s mount=%s\n" \ + "$((i + 1))" "$dev" "$size" "$label" "$mount" +done +echo "" +read -rp "Select a partition [1-${#USB_PARTS[@]}]: " CHOICE + +if ! [[ "$CHOICE" =~ ^[0-9]+$ ]] \ + || [ "$CHOICE" -lt 1 ] \ + || [ "$CHOICE" -gt "${#USB_PARTS[@]}" ]; then + echo "Invalid selection." >&2 + exit 1 +fi + +read -r SELECTED_DEV _ _ EXISTING_MOUNT <<< "${USB_PARTS[$((CHOICE - 1))]}" +SELECTED_DEV="/dev/$SELECTED_DEV" + +# --------------------------------------------------------------------------- +# Mount if needed +# --------------------------------------------------------------------------- +MOUNTED_HERE=false +MOUNT_DIR="" + +if [ -n "$EXISTING_MOUNT" ]; then + MOUNT_DIR="$EXISTING_MOUNT" + echo "Using existing mount at $MOUNT_DIR" +else + MOUNT_DIR=$(mktemp -d) + echo "Mounting $SELECTED_DEV at $MOUNT_DIR..." + mount "$SELECTED_DEV" "$MOUNT_DIR" + MOUNTED_HERE=true +fi + +cleanup() { + if [ "$MOUNTED_HERE" = true ] && [ -n "$MOUNT_DIR" ]; then + echo "Unmounting $MOUNT_DIR..." + umount "$MOUNT_DIR" + rmdir "$MOUNT_DIR" + fi +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Initialise restic repo if this is the first run +# --------------------------------------------------------------------------- +REPO="$MOUNT_DIR/$REPO_NAME" + +if ! restic -r "$REPO" --password-file "$PASSWORD_FILE" snapshots &>/dev/null 2>&1; then + echo "Initialising restic repository at $REPO..." + restic -r "$REPO" --password-file "$PASSWORD_FILE" init +fi + +# --------------------------------------------------------------------------- +# Run the backup +# --------------------------------------------------------------------------- +echo "" +echo "Backing up to $REPO..." +restic -r "$REPO" \ + --password-file "$PASSWORD_FILE" \ + --cache-dir "$DATA_DIR/restic-cache" \ + backup \ + "${BACKUP_PATHS[@]}" \ + "${EXCLUDE_ARGS[@]}" + +echo "" +echo "Snapshots on this drive:" +restic -r "$REPO" --password-file "$PASSWORD_FILE" snapshots diff --git a/shells/defaultShell.nix b/shells/defaultShell.nix index c662dfe..bc45a12 100644 --- a/shells/defaultShell.nix +++ b/shells/defaultShell.nix @@ -13,5 +13,10 @@ pkgs.mkShell { sudo nixos-rebuild switch \ --flake .#pi-main '') + (pkgs.writeShellScriptBin "homey-offload-backup" '' + set -euo pipefail + scp scripts/offload-backup.sh admin@192.168.1.100:/tmp/homey-offload-backup.sh + ssh -t admin@192.168.1.100 'sudo bash /tmp/homey-offload-backup.sh; rm /tmp/homey-offload-backup.sh' + '') ]; }