minecraft: update mods + add modernfix + debugify

Applies patch from: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/1370
That I am working on. Also updates version to latest (at this time) git

.gitattributes

@@ -1,10 +1,3 @@
-secrets/murmur_password filter=git-crypt diff=git-crypt
-secrets/hashedPass filter=git-crypt diff=git-crypt
-secrets/minecraft-whitelist.nix filter=git-crypt diff=git-crypt
-secrets/wg0.conf filter=git-crypt diff=git-crypt
-secrets/caddy_auth filter=git-crypt diff=git-crypt
-secrets/matrix_reg_token filter=git-crypt diff=git-crypt
-secrets/owntracks_caddy_auth filter=git-crypt diff=git-crypt
-secrets/secureboot.tar filter=git-crypt diff=git-crypt
-secrets/zfs-key filter=git-crypt diff=git-crypt
-secrets/slskd_env filter=git-crypt diff=git-crypt
+secrets/** filter=git-crypt diff=git-crypt
+usb-secrets/usb-secrets-key* filter=git-crypt diff=git-crypt
+

.gitignore

@@ -0,0 +1 @@
+/result

configuration.nix

@@ -6,12 +6,20 @@
   username,
   eth_interface,
   service_configs,
+  options,
   ...
 }:
 {
   imports = [
-    ./hardware.nix
-    ./zfs.nix
+    ./modules/hardware.nix
+    ./modules/zfs.nix
+    ./modules/impermanence.nix
+    ./modules/usb-secrets.nix
+    ./modules/age-secrets.nix
+    ./modules/secureboot.nix
+    ./modules/no-rgb.nix
+    ./modules/security.nix
+    ./modules/ntfy-alerts.nix
 
     ./services/postgresql.nix
     ./services/jellyfin.nix
@@ -19,14 +27,48 @@
     ./services/immich.nix
     ./services/gitea.nix
     ./services/minecraft.nix
+
     ./services/wg.nix
     ./services/qbittorrent.nix
+    ./services/jellyfin-qbittorrent-monitor.nix
     ./services/bitmagnet.nix
-    ./services/matrix.nix
-    ./services/owntracks.nix
+
+    ./services/arr/prowlarr.nix
+    ./services/arr/sonarr.nix
+    ./services/arr/radarr.nix
+    ./services/arr/bazarr.nix
+    ./services/arr/jellyseerr.nix
+    ./services/arr/recyclarr.nix
+    ./services/arr/init.nix
+
     ./services/soulseek.nix
+
+    ./services/ups.nix
+
+    ./services/bitwarden.nix
+
+    ./services/matrix.nix
+    ./services/coturn.nix
+    ./services/livekit.nix
+
+    ./services/monero.nix
+    ./services/xmrig.nix
+
+    # KEEP UNTIL 2028
+    ./services/caddy_senior_project.nix
+
+    ./services/graphing-calculator.nix
+
+    ./services/ssh.nix
+
+    ./services/syncthing.nix
+
+    ./services/ntfy.nix
+    ./services/ntfy-alerts.nix
   ];
 
+  services.kmscon.enable = true;
+
   systemd.targets = {
     sleep.enable = false;
     suspend.enable = false;
@@ -34,28 +76,50 @@
     hybrid-sleep.enable = false;
   };
 
+  # Disable serial getty on ttyS0 to prevent dmesg warnings
+  systemd.services."serial-getty@ttyS0".enable = false;
+
+  # srvos enables vim, i don't want to use vim, disable it here:
+  programs.vim = {
+    defaultEditor = false;
+  }
+  // lib.optionalAttrs (options.programs.vim ? enable) {
+    enable = false;
+  };
+
   powerManagement = {
     powertop.enable = true;
     enable = true;
     cpuFreqGovernor = "powersave";
   };
 
+  # https://github.com/NixOS/nixpkgs/issues/101459#issuecomment-758306434
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "soft";
+      item = "nofile";
+      value = "4096";
+    }
+  ];
+
   nix = {
     # optimize the store
     optimise.automatic = true;
 
-    # enable flakes!
-    settings = {
-      experimental-features = [
-        "nix-command"
-        "flakes"
-      ];
+    # garbage collection
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 7d";
     };
   };
 
+  hardware.intelgpu.driver = "xe";
+
   boot = {
-    # 6.12 LTS until 2027
-    kernelPackages = pkgs.linuxPackages_6_12;
+    # 6.12 LTS until 2026
+    kernelPackages = pkgs.linuxPackages_6_12_hardened;
 
     loader = {
       # Use the systemd-boot EFI boot loader.
@@ -67,26 +131,41 @@
 
     initrd = {
       compressor = "zstd";
+      supportedFilesystems = [ "f2fs" ];
     };
 
-    loader.systemd-boot.enable = lib.mkForce false;
+    # BBR congestion control handles variable-latency VPN connections much
+    # better than CUBIC by probing bandwidth continuously rather than
+    # reacting to packet loss.
+    kernelModules = [ "tcp_bbr" ];
 
-    lanzaboote = {
-      enable = true;
-      pkiBundle = "/var/lib/sbctl";
+    kernel.sysctl = {
+      # Use BBR + fair queuing for smooth throughput through the WireGuard VPN
+      "net.core.default_qdisc" = "fq";
+      "net.ipv4.tcp_congestion_control" = "bbr";
+
+      # Disable slow-start after idle: prevents TCP from resetting window
+      # size on each burst cycle (the primary cause of the 0 -> 40 MB/s spikes)
+      "net.ipv4.tcp_slow_start_after_idle" = 0;
+
+      # Larger socket buffers to accommodate the VPN bandwidth-delay product
+      # (22ms RTT * target throughput). Current 2.5MB max is too small.
+      "net.core.rmem_max" = 16777216;
+      "net.core.wmem_max" = 16777216;
+      "net.ipv4.tcp_rmem" = "4096 87380 16777216";
+      "net.ipv4.tcp_wmem" = "4096 65536 16777216";
+
+      # Higher backlog for the large number of concurrent torrent connections
+      "net.core.netdev_max_backlog" = 5000;
+
+      # Minecraft server optimizations
+      # Disable autogroup for better scheduling of game server threads
+      "kernel.sched_autogroup_enabled" = 0;
+      # Huge pages for Minecraft JVM (4000MB heap / 2MB per page + ~200 overhead)
+      "vm.nr_hugepages" = 2200;
     };
   };
 
-  system.activationScripts = {
-    # extract all my secureboot keys
-    "secureboot-keys".text = ''
-      #!/bin/sh
-      rm -fr ${config.boot.lanzaboote.pkiBundle} || true
-      mkdir -p ${config.boot.lanzaboote.pkiBundle}
-      ${pkgs.gnutar}/bin/tar xf ${./secrets/secureboot.tar} -C ${config.boot.lanzaboote.pkiBundle}
-    '';
-  };
-
   environment.etc = {
     "issue".text = "";
   };
@@ -94,20 +173,10 @@
   # Set your time zone.
   time.timeZone = "America/New_York";
 
-  # Enable the OpenSSH daemon.
-  services.openssh = {
-    enable = true;
-    settings = {
-      AllowUsers = [ username "root" ];
-      PasswordAuthentication = false;
-      PermitRootLogin = "yes"; # for deploying configs
-    };
-  };
-
   hardware.graphics = {
     enable = true;
     extraPackages = with pkgs; [
-      vaapiVdpau
+      libva-vdpau-driver
       intel-compute-runtime # OpenCL filter support (hardware tonemapping and subtitle burn-in)
       vpl-gpu-rt # QSV on 11th gen or newer
     ];
@@ -145,93 +214,16 @@
 
     lsof
 
-    (pkgs.writeShellApplication {
-      name = "disk-smart-test";
-      runtimeInputs = with pkgs; [
-        gnugrep
-        coreutils
-        smartmontools
-      ];
-
-      # i gotta fix that
-      excludeShellChecks = [ "SC2010" ];
-
-      text = ''
-        #!/bin/sh
-        set -e
-        if [[ $EUID -ne 0 ]]; then
-          echo "This command requires root."
-          exit 2
-        fi
-
-        DISKS=$(ls /dev/sd* | grep -v "[0-9]$")
-        for i in $DISKS; do
-          echo -n "$i "
-          smartctl -a "$i" | grep "SMART overall-health self-assessment test result:" | cut -d' ' -f6
-        done
-      '';
-    })
-
-    (pkgs.writeShellApplication {
-      name = "reflac";
-      runtimeInputs = with pkgs; [ flac ];
-      excludeShellChecks = [ "2086" ];
-
-      text = builtins.readFile (
-        pkgs.fetchurl {
-          url = "https://raw.githubusercontent.com/chungy/reflac/refs/heads/master/reflac";
-          sha256 = "61c6cc8be3d276c6714e68b55e5de0e6491f50bbf195233073dbce14a1e278a7";
-        }
-      );
-    })
+    reflac
 
     pfetch-rs
 
     sbctl
+
+    # add `skdump`
+    libatasmart
   ];
 
-  systemd.services.no-rgb =
-    let
-      no-rgb = (
-        pkgs.writeShellApplication {
-          name = "no-rgb";
-          runtimeInputs = with pkgs; [
-            openrgb
-            coreutils
-            gnugrep
-          ];
-
-          text = ''
-            #!/bin/sh
-            set -e
-
-            NUM_DEVICES=$(openrgb --noautoconnect --list-devices | grep -cE '^[0-9]+: ')
-
-            for i in $(seq 0 $((NUM_DEVICES - 1))); do
-              openrgb --noautoconnect --device "$i" --mode direct --color 000000
-            done
-          '';
-        }
-      );
-    in
-    {
-      description = "disable rgb";
-      serviceConfig = {
-        ExecStart = "${no-rgb}/bin/${no-rgb.name}";
-        Type = "oneshot";
-      };
-      wantedBy = [ "multi-user.target" ];
-    };
-
-  services.hardware.openrgb = {
-    enable = true;
-    package = pkgs.openrgb-with-all-plugins;
-    motherboard = "amd";
-  };
-
-  services.udev.packages = [ pkgs.openrgb-with-all-plugins ];
-  hardware.i2c.enable = true;
-
   networking = {
     nameservers = [
       "1.1.1.1"
@@ -241,13 +233,15 @@
     hostName = hostname;
     hostId = "0f712d56";
     firewall.enable = true;
+    firewall.trustedInterfaces = [ "wg-br" ];
     useDHCP = false;
-    # enableIPv6 = false;
+    enableIPv6 = false;
 
     interfaces.${eth_interface} = {
       ipv4.addresses = [
         {
-          address = "10.1.1.102";
+          address = "192.168.1.50";
+          # address = "10.1.1.102";
           prefixLength = 24;
         }
       ];
@@ -259,7 +253,8 @@
       ];
     };
     defaultGateway = {
-      address = "10.1.1.1";
+      #address = "10.1.1.1";
+      address = "192.168.1.1";
       interface = eth_interface;
     };
     # TODO! fix this
@@ -269,7 +264,7 @@
     # };
   };
 
-  users.groups.${service_configs.torrent_group} = { };
+  users.groups.${service_configs.media_group} = { };
 
   users.users.${username} = {
     isNormalUser = true;
@@ -277,19 +272,11 @@
       "wheel"
       "video"
       "render"
-      service_configs.torrent_group
-    ];
-
-    hashedPasswordFile = builtins.toString ./secrets/hashedPass;
-
-    openssh.authorizedKeys.keys = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO4jL6gYOunUlUtPvGdML0cpbKSsPNqQ1jit4E7U1RyH" # laptop
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBJjT5QZ3zRDb+V6Em20EYpSEgPW5e/U+06uQGJdraxi" # desktop
+      service_configs.media_group
     ];
+    hashedPasswordFile = config.age.secrets.hashedPass.path;
   };
 
-  users.users.root.openssh.authorizedKeys.keys = config.users.users.${username}.openssh.authorizedKeys.keys;
-
   # https://nixos.wiki/wiki/Fish#Setting_fish_as_your_shell
   programs.fish.enable = true;
   programs.bash = {
@@ -333,7 +320,7 @@
   # };
 
   # systemd.tmpfiles.rules = [
-  #   "d /tank/music 775 ${username} users"
+  #   "Z /tank/music 775 ${username} users"
   # ];
 
   system.stateVersion = "24.11";

deploy.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-nixos-rebuild switch --flake .#muffin --target-host root@server --build-host root@server --verbose

disk-config.nix

@@ -1,4 +1,9 @@
+{ inputs, ... }:
 {
+  imports = [
+    inputs.disko.nixosModules.disko
+  ];
+
   disko.devices = {
     disk = {
       main = {
@@ -15,17 +20,40 @@
                 mountpoint = "/boot";
               };
             };
-            root = {
+            persistent = {
+              size = "20G";
+              content = {
+                type = "filesystem";
+                format = "f2fs";
+                mountpoint = "/persistent";
+              };
+            };
+            nix = {
               size = "100%";
               content = {
                 type = "filesystem";
                 format = "f2fs";
-                mountpoint = "/";
+                mountpoint = "/nix";
               };
             };
+
           };
         };
       };
     };
+    nodev = {
+      "/" = {
+        fsType = "tmpfs";
+        mountOptions = [
+          "defaults"
+          "size=2G"
+          "mode=755"
+        ];
+      };
+    };
   };
+
+  fileSystems."/persistent".neededForBoot = true;
+  fileSystems."/nix".neededForBoot = true;
+
 }

flake.lock

@@ -1,12 +1,57 @@
 {
   "nodes": {
+    "agenix": {
+      "inputs": {
+        "darwin": [],
+        "home-manager": [
+          "home-manager"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1770165109,
+        "narHash": "sha256-9VnK6Oqai65puVJ4WYtCTvlJeXxMzAp/69HhQuTdl/I=",
+        "owner": "ryantm",
+        "repo": "agenix",
+        "rev": "b027ee29d959fda4b60b57566d64c98a202e0feb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ryantm",
+        "repo": "agenix",
+        "type": "github"
+      }
+    },
+    "arr-init": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772249948,
+        "narHash": "sha256-v68tO12mTCET68eZG583U+OlBL4f6kAoHS9iKA/xLzQ=",
+        "ref": "refs/heads/main",
+        "rev": "d21eb9f5b0a30bb487de7c0afbbbaf19324eaa49",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      }
+    },
     "crane": {
       "locked": {
-        "lastModified": 1741148495,
-        "narHash": "sha256-EV8KUaIZ2/CdBXlutXrHoZYbWPeB65p5kKZk71gvDRI=",
+        "lastModified": 1771796463,
+        "narHash": "sha256-9bCDuUzpwJXcHMQYMS1yNuzYMmKO/CCwCexpjWOl62I=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "75390a36cd0c2cdd5f1aafd8a9f827d7107f2e53",
+        "rev": "3d3de3313e263e04894f284ac18177bd26169bad",
         "type": "github"
       },
       "original": {
@@ -15,6 +60,28 @@
         "type": "github"
       }
     },
+    "deploy-rs": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "utils": "utils"
+      },
+      "locked": {
+        "lastModified": 1770019181,
+        "narHash": "sha256-hwsYgDnby50JNVpTRYlF3UR/Rrpt01OrxVuryF40CFY=",
+        "owner": "serokell",
+        "repo": "deploy-rs",
+        "rev": "77c906c0ba56aabdbc72041bf9111b565cdd6171",
+        "type": "github"
+      },
+      "original": {
+        "owner": "serokell",
+        "repo": "deploy-rs",
+        "type": "github"
+      }
+    },
     "disko": {
       "inputs": {
         "nixpkgs": [
@@ -22,11 +89,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1741786315,
-        "narHash": "sha256-VT65AE2syHVj6v/DGB496bqBnu1PXrrzwlw07/Zpllc=",
+        "lastModified": 1771881364,
+        "narHash": "sha256-A5uE/hMium5of/QGC6JwF5TGoDAfpNtW00T0s9u/PN8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "0d8c6ad4a43906d14abd5c60e0ffe7b587b213de",
+        "rev": "a4cb7bf73f264d40560ba527f9280469f1f081c6",
         "type": "github"
       },
       "original": {
@@ -54,43 +121,38 @@
     "flake-compat_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1733328505,
-        "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
-        "owner": "edolstra",
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
         "repo": "flake-compat",
-        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
+        "owner": "NixOS",
         "repo": "flake-compat",
         "type": "github"
       }
     },
-    "flake-parts": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "lanzaboote",
-          "nixpkgs"
-        ]
-      },
+    "flake-compat_3": {
+      "flake": false,
       "locked": {
-        "lastModified": 1740872218,
-        "narHash": "sha256-ZaMw0pdoUKigLpv9HiNDH2Pjnosg7NBYMJlHTIsHEUo=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "3876f6b87db82f33775b1ef5ea343986105db764",
+        "lastModified": 1747046372,
+        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
         "type": "github"
       },
       "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
+        "owner": "edolstra",
+        "repo": "flake-compat",
         "type": "github"
       }
     },
     "flake-utils": {
       "inputs": {
-        "systems": "systems"
+        "systems": "systems_4"
       },
       "locked": {
         "lastModified": 1731533236,
@@ -110,7 +172,7 @@
       "inputs": {
         "nixpkgs": [
           "lanzaboote",
-          "pre-commit-hooks-nix",
+          "pre-commit",
           "nixpkgs"
         ]
       },
@@ -135,37 +197,77 @@
         ]
       },
       "locked": {
-        "lastModified": 1742234739,
-        "narHash": "sha256-zFL6zsf/5OztR1NSNQF33dvS1fL/BzVUjabZq4qrtY4=",
+        "lastModified": 1772020340,
+        "narHash": "sha256-aqBl3GNpCadMoJ/hVkWTijM1Aeilc278MjM+LA3jK6g=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "f6af7280a3390e65c2ad8fd059cdc303426cbd59",
+        "rev": "36e38ca0d9afe4c55405fdf22179a5212243eecc",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "release-24.11",
+        "ref": "release-25.11",
         "repo": "home-manager",
         "type": "github"
       }
     },
+    "home-manager_2": {
+      "inputs": {
+        "nixpkgs": [
+          "impermanence",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1768598210,
+        "narHash": "sha256-kkgA32s/f4jaa4UG+2f8C225Qvclxnqs76mf8zvTVPg=",
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "rev": "c47b2cc64a629f8e075de52e4742de688f930dc6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "type": "github"
+      }
+    },
+    "impermanence": {
+      "inputs": {
+        "home-manager": "home-manager_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1769548169,
+        "narHash": "sha256-03+JxvzmfwRu+5JafM0DLbxgHttOQZkUtDWBmeUkN8Y=",
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "rev": "7b1d382faf603b6d264f58627330f9faa5cba149",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "type": "github"
+      }
+    },
     "lanzaboote": {
       "inputs": {
         "crane": "crane",
-        "flake-compat": "flake-compat",
-        "flake-parts": "flake-parts",
         "nixpkgs": [
           "nixpkgs"
         ],
-        "pre-commit-hooks-nix": "pre-commit-hooks-nix",
+        "pre-commit": "pre-commit",
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1741442524,
-        "narHash": "sha256-tVcxLDLLho8dWcO81Xj/3/ANLdVs0bGyCPyKjp70JWk=",
+        "lastModified": 1772216104,
+        "narHash": "sha256-1TnGN26vnCEQk5m4AavJZxGZTb/6aZyphemRPRwFUfs=",
         "owner": "nix-community",
         "repo": "lanzaboote",
-        "rev": "d8099586d9a84308ffedac07880e7f07a0180ff4",
+        "rev": "dbe5112de965bbbbff9f0729a9789c20a65ab047",
         "type": "github"
       },
       "original": {
@@ -176,18 +278,18 @@
     },
     "nix-minecraft": {
       "inputs": {
-        "flake-compat": "flake-compat_2",
-        "flake-utils": "flake-utils",
+        "flake-compat": "flake-compat_3",
         "nixpkgs": [
           "nixpkgs"
-        ]
+        ],
+        "systems": "systems_3"
       },
       "locked": {
-        "lastModified": 1742522051,
-        "narHash": "sha256-uDlj+5J7eTuFkDaNl9cYf++gJdEW23Z4zSuDcNANIQc=",
+        "lastModified": 1772160153,
+        "narHash": "sha256-lk5IxQzY9ZeeEyjKNT7P6dFnlRpQgkus4Ekc/+slypY=",
         "owner": "Infinidoge",
         "repo": "nix-minecraft",
-        "rev": "57464e795fd31ceef845d7ce454d3b83e80e283e",
+        "rev": "deca3fb710b502ba10cd5cdc8f66c2cc184b92df",
         "type": "github"
       },
       "original": {
@@ -198,11 +300,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1742376361,
-        "narHash": "sha256-VFMgJkp/COvkt5dnkZB4D2szVdmF6DGm5ZdVvTUy61c=",
+        "lastModified": 1771969195,
+        "narHash": "sha256-qwcDBtrRvJbrrnv1lf/pREQi8t2hWZxVAyeMo7/E9sw=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "daaae13dff0ecc692509a1332ff9003d9952d7a9",
+        "rev": "41c6b421bdc301b2624486e11905c9af7b8ec68e",
         "type": "github"
       },
       "original": {
@@ -214,42 +316,39 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1742562948,
-        "narHash": "sha256-QUnzAW7CW0sCkFN1Kez/8UVq8EbBGNKOfHZHIZON0XQ=",
+        "lastModified": 1772047000,
+        "narHash": "sha256-7DaQVv4R97cii/Qdfy4tmDZMB2xxtyIvNGSwXBBhSmo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e7a04ccc42104e0554f0a2325930fe98db9a5325",
+        "rev": "1267bb4920d0fc06ea916734c11b0bf004bbe17e",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11-small",
+        "ref": "nixos-25.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "nixpkgs-qbt": {
+    "nixpkgs_2": {
       "locked": {
-        "lastModified": 1738103934,
-        "narHash": "sha256-MhDdcDDdK2uscLU370r3V9PQcejx+2LVbMG8bjCXMb0=",
+        "lastModified": 1764517877,
+        "narHash": "sha256-pp3uT4hHijIC8JUK5MEqeAWmParJrgBVzHLNfJDZxg4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4f4706686c921ef202712a00da1c96f0100f6921",
+        "rev": "2d293cbfa5a793b4c50d17c05ef9e385b90edf6c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "pull/287923/head",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "pre-commit-hooks-nix": {
+    "pre-commit": {
       "inputs": {
-        "flake-compat": [
-          "lanzaboote",
-          "flake-compat"
-        ],
+        "flake-compat": "flake-compat_2",
         "gitignore": "gitignore",
         "nixpkgs": [
           "lanzaboote",
@@ -257,11 +356,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1740915799,
-        "narHash": "sha256-JvQvtaphZNmeeV+IpHgNdiNePsIpHD5U/7QN5AeY44A=",
+        "lastModified": 1771858127,
+        "narHash": "sha256-Gtre9YoYl3n25tJH2AoSdjuwcqij5CPxL3U3xysYD08=",
         "owner": "cachix",
         "repo": "pre-commit-hooks.nix",
-        "rev": "42b1ba089d2034d910566bf6b40830af6b8ec732",
+        "rev": "49bbbfc218bf3856dfa631cead3b052d78248b83",
         "type": "github"
       },
       "original": {
@@ -272,14 +371,22 @@
     },
     "root": {
       "inputs": {
+        "agenix": "agenix",
+        "arr-init": "arr-init",
+        "deploy-rs": "deploy-rs",
         "disko": "disko",
         "home-manager": "home-manager",
+        "impermanence": "impermanence",
         "lanzaboote": "lanzaboote",
         "nix-minecraft": "nix-minecraft",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-qbt": "nixpkgs-qbt",
-        "vpn-confinement": "vpn-confinement"
+        "senior_project-website": "senior_project-website",
+        "srvos": "srvos",
+        "trackerlist": "trackerlist",
+        "vpn-confinement": "vpn-confinement",
+        "website": "website",
+        "ytbn-graphing-software": "ytbn-graphing-software"
       }
     },
     "rust-overlay": {
@@ -290,11 +397,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1741228283,
-        "narHash": "sha256-VzqI+k/eoijLQ5am6rDFDAtFAbw8nltXfLBC6SIEJAE=",
+        "lastModified": 1771988922,
+        "narHash": "sha256-Fc6FHXtfEkLtuVJzd0B6tFYMhmcPLuxr90rWfb/2jtQ=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "38e9826bc4296c9daf18bc1e6aa299f3e932a403",
+        "rev": "f4443dc3f0b6c5e6b77d923156943ce816d1fcb9",
         "type": "github"
       },
       "original": {
@@ -303,6 +410,63 @@
         "type": "github"
       }
     },
+    "rust-overlay_2": {
+      "inputs": {
+        "nixpkgs": [
+          "ytbn-graphing-software",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1764729618,
+        "narHash": "sha256-z4RA80HCWv2los1KD346c+PwNPzMl79qgl7bCVgz8X0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "52764074a85145d5001bf0aa30cb71936e9ad5b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "senior_project-website": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1771869552,
+        "narHash": "sha256-veaVrRWCSy7HYAAjUFLw8HASKcj+3f0W+sCwS3QiaM4=",
+        "owner": "Titaniumtown",
+        "repo": "senior-project-website",
+        "rev": "28a2b93492dac877dce0b38f078eacf74fce26e7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Titaniumtown",
+        "repo": "senior-project-website",
+        "type": "github"
+      }
+    },
+    "srvos": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772071250,
+        "narHash": "sha256-LDWvJDR1J8xE8TBJjzWnOA0oVP/l9xBFC4npQPJDHN4=",
+        "owner": "nix-community",
+        "repo": "srvos",
+        "rev": "5cd73bcf984b72d8046e1175d13753de255adfb9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "srvos",
+        "type": "github"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -318,13 +482,92 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_4": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "trackerlist": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1772233783,
+        "narHash": "sha256-2jPUBKpPuT4dCXwVFuZvTH3QyURixsfJZD7Zqs0atPY=",
+        "owner": "ngosang",
+        "repo": "trackerslist",
+        "rev": "85c4f103f130b070a192343c334f50c2f56b61a9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ngosang",
+        "repo": "trackerslist",
+        "type": "github"
+      }
+    },
+    "utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "vpn-confinement": {
       "locked": {
-        "lastModified": 1742138327,
-        "narHash": "sha256-Y71Mjej98CjaUKa1ecAIOo0eJ1B3ZVQl2ng6xl7/s9Y=",
+        "lastModified": 1767604552,
+        "narHash": "sha256-FddhMxnc99KYOZ/S3YNqtDSoxisIhVtJ7L4s8XD2u0A=",
         "owner": "Maroka-chan",
         "repo": "VPN-Confinement",
-        "rev": "38eeb3bc501900b48d1caf8c52a5b7f2fb7a52c5",
+        "rev": "a6b2da727853886876fd1081d6bb2880752937f3",
         "type": "github"
       },
       "original": {
@@ -332,6 +575,42 @@
         "repo": "VPN-Confinement",
         "type": "github"
       }
+    },
+    "website": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1768266466,
+        "narHash": "sha256-d4dZzEcIKuq4DhNtXczaflpRifAtcOgNr45W2Bexnps=",
+        "ref": "refs/heads/main",
+        "rev": "06011a27456b3b9f983ef1aa142b5773bcb52b6e",
+        "revCount": 23,
+        "type": "git",
+        "url": "https://git.gardling.com/titaniumtown/website"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.gardling.com/titaniumtown/website"
+      }
+    },
+    "ytbn-graphing-software": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs_2",
+        "rust-overlay": "rust-overlay_2"
+      },
+      "locked": {
+        "lastModified": 1765615270,
+        "narHash": "sha256-12C6LccKRe5ys0iRd+ob+BliswUSmqOKWhMTI8fNpr0=",
+        "ref": "refs/heads/main",
+        "rev": "ac6265eae734363f95909df9a3739bf6360fa721",
+        "revCount": 1130,
+        "type": "git",
+        "url": "https://git.gardling.com/titaniumtown/YTBN-Graphing-Software"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.gardling.com/titaniumtown/YTBN-Graphing-Software"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -2,7 +2,7 @@
   description = "Flake for server muffin";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11-small";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
 
     lanzaboote = {
       url = "github:nix-community/lanzaboote";
@@ -18,10 +18,8 @@
 
     vpn-confinement.url = "github:Maroka-chan/VPN-Confinement";
 
-    nixpkgs-qbt.url = "github:NixOS/nixpkgs/pull/287923/head";
-
     home-manager = {
-      url = "github:nix-community/home-manager/release-24.11";
+      url = "github:nix-community/home-manager/release-25.11";
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
@@ -29,54 +27,119 @@
       url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    srvos = {
+      url = "github:nix-community/srvos";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    deploy-rs = {
+      url = "github:serokell/deploy-rs";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    impermanence = {
+      url = "github:nix-community/impermanence";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    agenix = {
+      url = "github:ryantm/agenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.home-manager.follows = "home-manager";
+      inputs.darwin.follows = "";
+    };
+
+    senior_project-website = {
+      url = "github:Titaniumtown/senior-project-website";
+      flake = false;
+    };
+
+    website = {
+      url = "git+https://git.gardling.com/titaniumtown/website";
+      flake = false;
+    };
+
+    trackerlist = {
+      url = "github:ngosang/trackerslist";
+      flake = false;
+    };
+
+    ytbn-graphing-software = {
+      url = "git+https://git.gardling.com/titaniumtown/YTBN-Graphing-Software";
+    };
+
+    arr-init = {
+      url = "git+ssh://gitea@git.gardling.com/titaniumtown/arr-init";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =
     {
+      self,
       nixpkgs,
       nix-minecraft,
       nixos-hardware,
       vpn-confinement,
-      nixpkgs-qbt,
       home-manager,
       lanzaboote,
       disko,
+      srvos,
+      deploy-rs,
+      impermanence,
+      arr-init,
       ...
     }@inputs:
     let
       username = "primary";
       hostname = "muffin";
       eth_interface = "enp4s0";
+      system = "x86_64-linux";
 
       service_configs = rec {
         zpool_ssds = "tank";
         zpool_hdds = "hdds";
         torrents_path = "/torrents";
-        services_dir = "/${zpool_ssds}/services";
+        services_dir = "/services";
         music_dir = "/${zpool_ssds}/music";
-        torrent_group = "media";
+        media_group = "media";
+
+        cpu_arch = "znver3";
 
-        # TODO: add checks to make sure none of these collide
         ports = {
+          http = 80;
           https = 443;
           jellyfin = 8096; # no services.jellyfin option for this
           torrent = 6011;
-          ollama = 11434;
           bitmagnet = 3333;
-          owntracks = 3825;
           gitea = 2283;
           immich = 2284;
           soulseek_web = 5030;
           soulseek_listen = 50300;
+          llama_cpp = 8991;
+          vaultwarden = 8222;
+          syncthing_gui = 8384;
+          syncthing_protocol = 22000;
+          syncthing_discovery = 21027;
+          minecraft = 25565;
+          matrix = 6167;
+          matrix_federation = 8448;
+          coturn = 3478;
+          coturn_tls = 5349;
+          ntfy = 2586;
+          livekit = 7880;
+          lk_jwt = 8081;
+          prowlarr = 9696;
+          sonarr = 8989;
+          radarr = 7878;
+          bazarr = 6767;
+          jellyseerr = 5055;
         };
 
         https = {
           certs = services_dir + "/http_certs";
-          # TODO! generate website from repo directly using hugo
-          data_dir = services_dir + "/http";
           domain = "gardling.com";
-          wg_ip = "192.168.15.1";
-          matrix_hostname = "matrix.${service_configs.https.domain}";
         };
 
         gitea = {
@@ -86,6 +149,7 @@
 
         postgres = {
           socket = "/run/postgresql";
+          dataDir = services_dir + "/sql";
         };
 
         immich = {
@@ -103,16 +167,88 @@
         };
 
         jellyfin = {
-          dir = services_dir + "/jellyfin";
+          dataDir = services_dir + "/jellyfin";
+          cacheDir = services_dir + "/jellyfin_cache";
         };
 
-        owntracks = {
-          data_dir = services_dir + "/owntracks";
+        slskd = rec {
+          base = "/var/lib/slskd";
+          downloads = base + "/downloads";
+          incomplete = base + "/incomplete";
         };
+
+        vaultwarden = {
+          path = "/var/lib/vaultwarden";
+        };
+
+        monero = {
+          dataDir = services_dir + "/monero";
+        };
+
+        matrix = {
+          dataDir = "/var/lib/continuwuity";
+          domain = "matrix.${https.domain}";
+        };
+
+        ntfy = {
+          domain = "ntfy.${https.domain}";
+        };
+
+        livekit = {
+          domain = "livekit.${https.domain}";
+        };
+
+        syncthing = {
+          dataDir = services_dir + "/syncthing";
+          signalBackupDir = "/${zpool_ssds}/bak/signal";
+          grayjayBackupDir = "/${zpool_ssds}/bak/grayjay";
+        };
+
+        prowlarr = {
+          dataDir = services_dir + "/prowlarr";
+        };
+
+        sonarr = {
+          dataDir = services_dir + "/sonarr";
+        };
+
+        radarr = {
+          dataDir = services_dir + "/radarr";
+        };
+
+        bazarr = {
+          dataDir = services_dir + "/bazarr";
+        };
+
+        jellyseerr = {
+          configDir = services_dir + "/jellyseerr";
+        };
+
+        recyclarr = {
+          dataDir = services_dir + "/recyclarr";
+        };
+
+        media = {
+          moviesDir = torrents_path + "/media/movies";
+          tvDir = torrents_path + "/media/tv";
+        };
+      };
+
+      pkgs = import nixpkgs {
+        inherit system;
+        targetPlatform = system;
+        buildPlatform = builtins.currentSystem;
+      };
+      lib = import ./modules/lib.nix { inherit inputs pkgs service_configs; };
+      testSuite = import ./tests/tests.nix {
+        inherit pkgs lib inputs;
+        config = self.nixosConfigurations.muffin.config;
       };
     in
     {
-      nixosConfigurations.${hostname} = nixpkgs.lib.nixosSystem {
+      formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-tree;
+      nixosConfigurations.${hostname} = lib.nixosSystem {
+        inherit system;
         specialArgs = {
           inherit
             username
@@ -122,45 +258,91 @@
             inputs
             ;
         };
-        modules =
-          [
-            ./disk-config.nix
-            disko.nixosModules.disko
-            ./configuration.nix
-
-            vpn-confinement.nixosModules.default
-
-            # import the `services.qbittorrent` module
-            (nixpkgs-qbt + "/nixos/modules/services/torrent/qbittorrent.nix")
-
-            # get nix-minecraft working!
-            nix-minecraft.nixosModules.minecraft-servers
+        modules = [
+          # SAFETY! make sure no ports collide
+          (
+            { lib, ... }:
             {
-              nixpkgs.overlays = [ nix-minecraft.overlay ];
+              config.assertions = [
+                {
+                  assertion =
+                    let
+                      ports = lib.attrValues service_configs.ports;
+                      uniquePorts = lib.unique ports;
+                    in
+                    (lib.length ports) == (lib.length uniquePorts);
+                  message = "Duplicate ports detected in 'ports' configuration";
+                }
+              ];
             }
+          )
 
-            lanzaboote.nixosModules.lanzaboote
+          # sets up things like the watchdog
+          srvos.nixosModules.server
 
-            home-manager.nixosModules.home-manager
-            (
-              {
-                pkgs,
-                username,
-                home-manager,
-                stateVersion,
-                ...
-              }:
-              {
-                home-manager.users.${username} = import ./home.nix;
-              }
-            )
-          ]
-          ++ (with nixos-hardware.nixosModules; [
-            common-cpu-amd-pstate
-            common-cpu-amd-zenpower
-            common-pc-ssd
-            common-gpu-intel
-          ]);
+          # diff terminal support
+          srvos.nixosModules.mixins-terminfo
+
+          ./disk-config.nix
+          ./configuration.nix
+
+          {
+            nixpkgs.overlays = [
+              nix-minecraft.overlay
+              (import ./modules/overlays.nix)
+            ];
+            nixpkgs.config.allowUnfreePredicate =
+              pkg:
+              builtins.elem (nixpkgs.lib.getName pkg) [
+                "minecraft-server"
+              ];
+          }
+
+          lanzaboote.nixosModules.lanzaboote
+
+          arr-init.nixosModules.default
+
+          home-manager.nixosModules.home-manager
+          (
+            {
+              home-manager,
+              ...
+            }:
+            {
+              home-manager.users.${username} = import ./modules/home.nix;
+            }
+          )
+        ]
+        ++ (with nixos-hardware.nixosModules; [
+          common-cpu-amd-pstate
+          common-cpu-amd-zenpower
+          common-pc-ssd
+          common-gpu-intel
+        ]);
       };
+
+      deploy.nodes.muffin = {
+        hostname = "server-public";
+        profiles.system = {
+          sshUser = "root";
+          user = "root";
+          path = deploy-rs.lib.${system}.activate.nixos self.nixosConfigurations.muffin;
+        };
+      };
+
+      checks.${system} = testSuite;
+
+      packages.${system} = {
+        tests = pkgs.linkFarm "all-tests" (
+          pkgs.lib.mapAttrsToList (name: test: {
+            name = name;
+            path = test;
+          }) testSuite
+        );
+      }
+      // (pkgs.lib.mapAttrs' (name: test: {
+        name = "test-${name}";
+        value = test;
+      }) testSuite);
     };
 }

home.nix

@@ -1,36 +0,0 @@
-{
-  pkgs,
-  username,
-  stateVersion,
-  ...
-}:
-{
-  home.stateVersion = "24.11";
-  programs.fish =
-    let
-      eza = "${pkgs.eza}/bin/eza --color=always --group-directories-first";
-      coreutils = "${pkgs.coreutils}/bin";
-    in
-    {
-      enable = true;
-
-      interactiveShellInit = ''
-        #disable greeting
-        set fish_greeting
-
-        #fixes gnupg password entry
-        export GPG_TTY=(${coreutils}/tty)
-
-        #pfetch on shell start (disable pkgs because of execution time)
-        PF_INFO="ascii title os host kernel uptime memory editor wm" ${pkgs.pfetch-rs}/bin/pfetch
-      '';
-
-      shellAliases = {
-        # from DistroTube's dot files: Changing "ls" to "eza"
-        ls = "${eza} -al";
-        la = "${eza} -a";
-        ll = "${eza} -l";
-        lt = "${eza} -aT";
-      };
-    };
-}

modules/age-secrets.nix

@@ -0,0 +1,84 @@
+{
+  config,
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.agenix.nixosModules.default
+  ];
+
+  # Configure all agenix secrets
+  age.secrets = {
+    # ZFS encryption key
+    zfs-key = {
+      file = ../secrets/zfs-key.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Secureboot keys archive
+    secureboot-tar = {
+      file = ../secrets/secureboot.tar.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # System passwords
+    hashedPass = {
+      file = ../secrets/hashedPass.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Service authentication
+    caddy_auth = {
+      file = ../secrets/caddy_auth.age;
+      mode = "0400";
+      owner = "caddy";
+      group = "caddy";
+    };
+
+    jellyfin-api-key = {
+      file = ../secrets/jellyfin-api-key.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    slskd_env = {
+      file = ../secrets/slskd_env.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # Network configuration
+    wg0-conf = {
+      file = ../secrets/wg0.conf.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    # ntfy-alerts secrets
+    ntfy-alerts-topic = {
+      file = ../secrets/ntfy-alerts-topic.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    ntfy-alerts-token = {
+      file = ../secrets/ntfy-alerts-token.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+  };
+}

modules/hardware.nix

@@ -19,7 +19,6 @@
 
   swapDevices = [ ];
 
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
   hardware.cpu.amd.updateMicrocode = true;
   hardware.enableRedistributableFirmware = true;
 }

modules/home.nix

@@ -0,0 +1,31 @@
+{
+  pkgs,
+  lib,
+  ...
+}:
+{
+  home.stateVersion = "24.11";
+  programs.fish = {
+    enable = true;
+
+    interactiveShellInit = ''
+      # disable greeting
+      set fish_greeting
+
+      # pfetch on shell start (disable pkgs because of execution time)
+      PF_INFO="ascii title os host kernel uptime memory editor wm" ${lib.getExe pkgs.pfetch-rs}
+    '';
+
+    shellAliases =
+      let
+        eza = "${lib.getExe pkgs.eza} --color=always --group-directories-first";
+      in
+      {
+        # from DistroTube's dot files: Changing "ls" to "eza"
+        ls = "${eza} -al";
+        la = "${eza} -a";
+        ll = "${eza} -l";
+        lt = "${eza} -aT";
+      };
+  };
+}

modules/impermanence.nix

@@ -0,0 +1,70 @@
+{
+  config,
+  lib,
+  pkgs,
+  username,
+  service_configs,
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.impermanence.nixosModules.impermanence
+  ];
+
+  environment.persistence."/persistent" = {
+    hideMounts = true;
+    directories = [
+      "/var/log"
+      "/var/lib/systemd/coredump"
+      "/var/lib/nixos"
+
+      "/var/lib/systemd/timers"
+
+      # ZFS cache directory - persisting the directory instead of the file
+      # avoids "device busy" errors when ZFS atomically updates the cache
+      "/etc/zfs"
+    ];
+
+    files = [
+      # Machine ID
+      "/etc/machine-id"
+    ];
+
+    users.${username} = {
+      files = [
+        ".local/share/fish/fish_history"
+      ];
+    };
+
+    users.root = {
+      files = [
+        ".local/share/fish/fish_history"
+      ];
+    };
+  };
+
+  # Store SSH host keys directly in /persistent to survive tmpfs root wipes.
+  # This is more reliable than bind mounts for service-generated files.
+  services.openssh.hostKeys = [
+    {
+      path = "/persistent/etc/ssh/ssh_host_ed25519_key";
+      type = "ed25519";
+    }
+    {
+      path = "/persistent/etc/ssh/ssh_host_rsa_key";
+      type = "rsa";
+      bits = 4096;
+    }
+  ];
+
+  # Enforce root ownership on /persistent/etc. The impermanence activation
+  # script copies ownership from /persistent/etc to /etc via
+  # `chown --reference`. If /persistent/etc ever gets non-root ownership,
+  # sshd StrictModes rejects /etc/ssh/authorized_keys.d/root and root SSH
+  # breaks while non-root users still work.
+  # Use "z" (set ownership, non-recursive) not "d" (create only, no-op on existing).
+  systemd.tmpfiles.rules = [
+    "z /persistent/etc 0755 root root"
+  ];
+}

modules/lib.nix

@@ -0,0 +1,184 @@
+{
+  inputs,
+  pkgs,
+  service_configs,
+  ...
+}:
+inputs.nixpkgs.lib.extend (
+  final: prev:
+  let
+    lib = prev;
+  in
+  {
+    # stolen from: https://stackoverflow.com/a/42398526
+    optimizeWithFlags =
+      pkg: flags:
+      lib.overrideDerivation pkg (
+        old:
+        let
+          newflags = lib.foldl' (acc: x: "${acc} ${x}") "" flags;
+          oldflags = if (lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else "";
+        in
+        {
+          NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}";
+          # stdenv = pkgs.clang19Stdenv;
+        }
+      );
+
+    optimizePackage =
+      pkg:
+      final.optimizeWithFlags pkg [
+        "-O3"
+        "-march=${service_configs.cpu_arch}"
+        "-mtune=${service_configs.cpu_arch}"
+      ];
+
+    vpnNamespaceOpenPort =
+      port: service:
+      { ... }:
+      {
+        vpnNamespaces.wg = {
+          portMappings = [
+            {
+              from = port;
+              to = port;
+            }
+          ];
+
+          openVPNPorts = [
+            {
+              port = port;
+              protocol = "both";
+            }
+          ];
+        };
+        systemd.services.${service}.vpnConfinement = {
+          enable = true;
+          vpnNamespace = "wg";
+        };
+      };
+
+    serviceMountWithZpool =
+      serviceName: zpool: dirs:
+      { pkgs, config, ... }:
+      {
+        systemd.services."${serviceName}-mounts" = {
+          wants = [ "zfs.target" ] ++ lib.optionals (zpool != "") [ "zfs-import-${zpool}.service" ];
+          after = lib.optionals (zpool != "") [ "zfs-import-${zpool}.service" ];
+          before = [ "${serviceName}.service" ];
+
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = [
+              (lib.getExe (
+                pkgs.writeShellApplication {
+                  name = "ensure-zfs-mounts-with-pool-${serviceName}-${zpool}";
+                  runtimeInputs = with pkgs; [
+                    gawk
+                    coreutils
+                    config.boot.zfs.package
+                  ];
+
+                  text = ''
+                    set -euo pipefail
+
+                    echo "Ensuring ZFS mounts for service: ${serviceName} (pool: ${zpool})"
+                    echo "Directories: ${lib.strings.concatStringsSep ", " dirs}"
+
+                    # Validate mounts exist (ensureZfsMounts already has proper PATH)
+                    ${lib.getExe pkgs.ensureZfsMounts} ${lib.strings.concatStringsSep " " dirs}
+
+                    # Additional runtime check: verify paths are on correct zpool
+                    ${lib.optionalString (zpool != "") ''
+                      echo "Verifying ZFS mountpoints are on pool '${zpool}'..."
+
+                      if ! zfs_list_output=$(zfs list -H -o name,mountpoint 2>&1); then
+                        echo "ERROR: Failed to query ZFS datasets: $zfs_list_output" >&2
+                        exit 1
+                      fi
+
+                      # shellcheck disable=SC2043
+                      for target in ${lib.strings.concatStringsSep " " dirs}; do
+                        echo "Checking: $target"
+
+                        # Find dataset that has this mountpoint
+                        dataset=$(echo "$zfs_list_output" | awk -v target="$target" '$2 == target {print $1; exit}')
+
+                        if [ -z "$dataset" ]; then
+                          echo "ERROR: No ZFS dataset found for mountpoint: $target" >&2
+                          exit 1
+                        fi
+
+                        # Extract pool name from dataset (first part before /)
+                        actual_pool=$(echo "$dataset" | cut -d'/' -f1)
+
+                        if [ "$actual_pool" != "${zpool}" ]; then
+                          echo "ERROR: ZFS pool mismatch for $target" >&2
+                          echo " Expected pool: ${zpool}" >&2
+                          echo " Actual pool:   $actual_pool" >&2
+                          echo " Dataset:       $dataset" >&2
+                          exit 1
+                        fi
+
+                        echo "$target is on $dataset (pool: $actual_pool)"
+                      done
+
+                      echo "All paths verified successfully on pool '${zpool}'"
+                    ''}
+
+                    echo "Mount validation completed for ${serviceName} (pool: ${zpool})"
+                  '';
+                }
+              ))
+            ];
+          };
+        };
+
+        systemd.services.${serviceName} = {
+          wants = [
+            "${serviceName}-mounts.service"
+          ];
+          after = [
+            "${serviceName}-mounts.service"
+          ];
+          requires = [
+            "${serviceName}-mounts.service"
+          ];
+        };
+
+        # assert that the pool is even enabled
+        #assertions = lib.optionals (zpool != "") [
+        #  {
+        #    assertion = builtins.elem zpool config.boot.zfs.extraPools;
+        #    message = "${zpool} is not enabled in `boot.zfs.extraPools`";
+        #  }
+        #];
+      };
+
+    serviceFilePerms =
+      serviceName: tmpfilesRules:
+      { pkgs, ... }:
+      let
+        confFile = pkgs.writeText "${serviceName}-file-perms.conf" (
+          lib.concatStringsSep "\n" tmpfilesRules
+        );
+      in
+      {
+        systemd.services."${serviceName}-file-perms" = {
+          after = [ "${serviceName}-mounts.service" ];
+          before = [ "${serviceName}.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            ExecStart = "${pkgs.systemd}/bin/systemd-tmpfiles --create ${confFile}";
+          };
+        };
+
+        systemd.services.${serviceName} = {
+          wants = [ "${serviceName}-file-perms.service" ];
+          after = [ "${serviceName}-file-perms.service" ];
+        };
+      };
+  }
+)

modules/no-rgb.nix

@@ -0,0 +1,66 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  systemd.services.no-rgb =
+    let
+      no-rgb = (
+        pkgs.writeShellApplication {
+          name = "no-rgb";
+          runtimeInputs = with pkgs; [
+            openrgb
+            coreutils
+            gnugrep
+          ];
+
+          text = ''
+            # Retry loop to wait for hardware to be ready
+            NUM_DEVICES=0
+            for attempt in 1 2 3 4 5; do
+              DEVICE_LIST=$(openrgb --noautoconnect --list-devices 2>/dev/null) || DEVICE_LIST=""
+              NUM_DEVICES=$(echo "$DEVICE_LIST" | grep -cE '^[0-9]+: ') || NUM_DEVICES=0
+              if [ "$NUM_DEVICES" -gt 0 ]; then
+                break
+              fi
+              if [ "$attempt" -lt 5 ]; then
+                sleep 2
+              fi
+            done
+
+            # If no devices found after retries, exit gracefully
+            if [ "$NUM_DEVICES" -eq 0 ]; then
+              exit 0
+            fi
+
+            # Disable RGB on each device
+            for i in $(seq 0 $((NUM_DEVICES - 1))); do
+              openrgb --noautoconnect --device "$i" --mode direct --color 000000 || true
+            done
+          '';
+        }
+      );
+    in
+    {
+      description = "disable rgb";
+      after = [ "systemd-udev-settle.service" ];
+      serviceConfig = {
+        ExecStart = lib.getExe no-rgb;
+        Type = "oneshot";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+
+  services.hardware.openrgb = {
+    enable = true;
+    package = pkgs.openrgb-with-all-plugins;
+    motherboard = "amd";
+  };
+
+  services.udev.packages = [ pkgs.openrgb-with-all-plugins ];
+  hardware.i2c.enable = true;
+}

modules/ntfy-alerts.nix

@@ -0,0 +1,132 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.ntfyAlerts;
+
+  curl = "${pkgs.curl}/bin/curl";
+  hostname = config.networking.hostName;
+
+  # Build the curl auth args as a proper bash array fragment
+  authCurlArgs =
+    if cfg.tokenFile != null then
+      ''
+        if [ -f "${cfg.tokenFile}" ]; then
+          TOKEN=$(cat "${cfg.tokenFile}" 2>/dev/null || echo "")
+          if [ -n "$TOKEN" ]; then
+            AUTH_ARGS=(-H "Authorization: Bearer $TOKEN")
+          fi
+        fi
+      ''
+    else
+      "";
+
+  # Systemd failure alert script
+  systemdAlertScript = pkgs.writeShellScript "ntfy-systemd-alert" ''
+    set -euo pipefail
+
+    UNIT_NAME="$1"
+    SERVER_URL="${cfg.serverUrl}"
+    TOPIC=$(cat "${cfg.topicFile}" 2>/dev/null | tr -d '[:space:]')
+    if [ -z "$TOPIC" ]; then
+      echo "ERROR: Could not read topic from ${cfg.topicFile}"
+      exit 1
+    fi
+
+    # Get journal output for context
+    JOURNAL_OUTPUT=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 15 --no-pager 2>/dev/null || echo "No journal output available")
+
+    # Build auth args
+    AUTH_ARGS=()
+    ${authCurlArgs}
+
+    # Send notification
+    ${curl} -sf --max-time 15 -X POST \
+      "$SERVER_URL/$TOPIC" \
+      -H "Title: [${hostname}] Service failed: $UNIT_NAME" \
+      -H "Priority: high" \
+      -H "Tags: warning" \
+      "''${AUTH_ARGS[@]}" \
+      -d "$JOURNAL_OUTPUT" || true
+  '';
+
+in
+{
+  options.services.ntfyAlerts = {
+    enable = lib.mkEnableOption "ntfy push notifications for system alerts";
+
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "The ntfy server URL (e.g. https://ntfy.example.com)";
+      example = "https://ntfy.example.com";
+    };
+
+    topicFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to a file containing the ntfy topic name to publish alerts to.";
+      example = "/run/agenix/ntfy-alerts-topic";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Path to a file containing the ntfy auth token.
+        If set, uses Authorization: Bearer header for authentication.
+      '';
+      example = "/run/secrets/ntfy-token";
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    # Per-service OnFailure for monitored services
+    systemd.services = {
+      "ntfy-alert@" = {
+        description = "Send ntfy notification for failed service %i";
+
+        unitConfig.OnFailure = lib.mkForce "";
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${systemdAlertScript} %i";
+          TimeoutSec = 30;
+        };
+      };
+
+      # TODO: sanoid's ExecStartPre runs `zfs allow` which blocks on TXG sync;
+      # on the hdds pool (slow spinning disks + large async frees) this causes
+      # 30+ minute hangs and guaranteed timeouts. Suppress until we fix sanoid
+      # to run as root without `zfs allow`. See: nixpkgs#72060, openzfs/zfs#14180
+      "sanoid".unitConfig.OnFailure = lib.mkForce "";
+    };
+
+    # Global OnFailure drop-in for all services
+    systemd.packages = [
+      (pkgs.writeTextDir "etc/systemd/system/service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=ntfy-alert@%p.service
+      '')
+
+      # Sanoid-specific drop-in to override the global OnFailure (see TODO above)
+      (pkgs.writeTextDir "etc/systemd/system/sanoid.service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=
+      '')
+    ];
+    # ZED (ZFS Event Daemon) ntfy notification settings
+    services.zfs.zed = {
+      enableMail = false;
+      settings = {
+        ZED_NTFY_URL = cfg.serverUrl;
+        ZED_NTFY_TOPIC = "$(cat ${cfg.topicFile} | tr -d '[:space:]')";
+        ZED_NTFY_ACCESS_TOKEN = lib.mkIf (cfg.tokenFile != null) "$(cat ${cfg.tokenFile})";
+        ZED_NOTIFY_VERBOSE = true;
+      };
+    };
+
+  };
+}

modules/overlays.nix

@@ -0,0 +1,46 @@
+final: prev: {
+  ensureZfsMounts = prev.writeShellApplication {
+    name = "zfsEnsureMounted";
+    runtimeInputs = with prev; [
+      zfs
+      gawk
+      coreutils
+    ];
+
+    text = ''
+       #!/bin/sh
+
+       if [[ "$#" -eq "0" ]]; then
+          echo "no arguments passed"
+          exit 1
+       fi
+
+      MOUNTED=$(zfs list -o mountpoint,mounted -H | awk '$NF == "yes" {NF--; print}')
+
+       MISSING=""
+       for target in "$@"; do
+          if ! grep -Fxq "$target" <<< "$MOUNTED"; then
+             MISSING="$MISSING $target"
+          fi
+       done
+
+       if [[ -n "$MISSING" ]]; then
+          echo "FAILURE, missing:$MISSING" 1>&2
+          exit 1
+       fi
+    '';
+  };
+
+  reflac = prev.writeShellApplication {
+    name = "reflac";
+    runtimeInputs = with prev; [ flac ];
+    excludeShellChecks = [ "2086" ];
+
+    text = builtins.readFile (
+      prev.fetchurl {
+        url = "https://raw.githubusercontent.com/chungy/reflac/refs/heads/master/reflac";
+        sha256 = "61c6cc8be3d276c6714e68b55e5de0e6491f50bbf195233073dbce14a1e278a7";
+      }
+    );
+  };
+}

modules/secureboot.nix

@@ -0,0 +1,41 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  boot = {
+    loader.systemd-boot.enable = lib.mkForce false;
+
+    lanzaboote = {
+      enable = true;
+      # needed to be in `/etc/secureboot` for sbctl to work
+      pkiBundle = "/etc/secureboot";
+    };
+
+  };
+  system.activationScripts = {
+    # extract secureboot keys from agenix-decrypted tar
+    "secureboot-keys" = {
+      deps = [ "agenix" ];
+      text = ''
+        #!/bin/sh
+        # Check if keys already exist (e.g., from disko-install)
+        if [[ -d ${config.boot.lanzaboote.pkiBundle} && -f ${config.boot.lanzaboote.pkiBundle}/db.key ]]; then
+          echo "Secureboot keys already present, skipping extraction"
+          chown -R root:wheel ${config.boot.lanzaboote.pkiBundle}
+          chmod -R 500 ${config.boot.lanzaboote.pkiBundle}
+        else
+          echo "Extracting secureboot keys from agenix"
+          rm -fr ${config.boot.lanzaboote.pkiBundle} || true
+          mkdir -p ${config.boot.lanzaboote.pkiBundle}
+          ${pkgs.gnutar}/bin/tar xf ${config.age.secrets.secureboot-tar.path} -C ${config.boot.lanzaboote.pkiBundle}
+          chown -R root:wheel ${config.boot.lanzaboote.pkiBundle}
+          chmod -R 500 ${config.boot.lanzaboote.pkiBundle}
+        fi
+      '';
+    };
+  };
+}

modules/security.nix

@@ -0,0 +1,37 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+{
+  # memory allocator
+  # BREAKS REDIS-IMMICH
+  # environment.memoryAllocator.provider = "graphene-hardened";
+
+  # disable coredumps
+  systemd.coredump.enable = false;
+
+  services = {
+    dbus.implementation = "broker";
+    /*
+      logrotate.enable = true;
+      journald = {
+        storage = "volatile"; # Store logs in memory
+        upload.enable = false; # Disable remote log upload (the default)
+        extraConfig = ''
+          SystemMaxUse=500M
+          SystemMaxFileSize=50M
+        '';
+      };
+    */
+  };
+
+  services.fail2ban = {
+    enable = true;
+    # Use iptables actions for compatibility
+    banaction = "iptables-multiport";
+    banaction-allports = "iptables-allports";
+  };
+}

modules/usb-secrets.nix

@@ -0,0 +1,22 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  # Mount USB secrets drive via fileSystems
+  fileSystems."/mnt/usb-secrets" = {
+    device = "/dev/disk/by-label/SECRETS";
+    fsType = "vfat";
+    options = [
+      "ro"
+      "uid=root"
+      "gid=root"
+      "umask=377"
+    ];
+    neededForBoot = true;
+  };
+
+  age.identityPaths = [ "/mnt/usb-secrets/usb-secrets-key" ];
+}

modules/zfs.nix

@@ -1,29 +1,40 @@
 {
+  config,
   service_configs,
   pkgs,
   ...
 }:
 let
+  # DO NOT CHANGE
+  # path is set via a zfs property
   zfs-key = "/etc/zfs-key";
 in
 {
   system.activationScripts = {
+    # Copy decrypted ZFS key from agenix to expected location
+    # /etc is on tmpfs due to impermanence, so no persistent storage risk
     "zfs-key".text = ''
       #!/bin/sh
-      rm -fr ${zfs-key} || true
-      cp ${./secrets/zfs-key} ${zfs-key}
-      chmod 0500 ${zfs-key}
-      chown root:wheel ${zfs-key}
+      rm -f ${zfs-key} || true
+      cp ${config.age.secrets.zfs-key.path} ${zfs-key}
+      chmod 0400 ${zfs-key}
+      chown root:root ${zfs-key}
     '';
   };
 
-  boot.zfs.package = pkgs.zfs_unstable;
+  boot.zfs.package = pkgs.zfs;
   boot.initrd.kernelModules = [ "zfs" ];
 
-  boot.kernelParams = [
-    # 2048MB
-    "zfs.zfs_arc_max=2048000000"
-  ];
+  boot.kernelParams =
+    let
+      gb = 20;
+      mb = gb * 1000;
+      kb = mb * 1000;
+      b = kb * 1000;
+    in
+    [
+      "zfs.zfs_arc_max=${builtins.toString b}"
+    ];
 
   boot.supportedFilesystems = [ "zfs" ];
   boot.zfs.extraPools = [
@@ -53,7 +64,7 @@ in
       yearly = 0;
     };
 
-    datasets."${service_configs.zpool_ssds}/services/jellyfin_cache" = {
+    datasets."${service_configs.zpool_ssds}/services/jellyfin/cache" = {
       recursive = true;
       autoprune = true;
       autosnap = true;

scripts/install.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DISK="${1:-}"
+FLAKE_DIR="$(dirname "$(realpath "$0")")"
+
+if [[ -z "$DISK" ]]; then
+    echo "Usage: $0 <disk_device>"
+    echo "Example: $0 /dev/nvme0n1"
+    echo "         $0 /dev/sda"
+    exit 1
+fi
+
+if [[ ! -b "$DISK" ]]; then
+    echo "Error: $DISK is not a block device"
+    exit 1
+fi
+
+echo "Installing NixOS to $DISK using flake at $FLAKE_DIR"
+
+# Create temporary directories
+mkdir -p /tmp/secureboot
+mkdir -p /tmp/persistent
+
+# Function to cleanup on exit
+cleanup() {
+    echo "Cleaning up..."
+    rm -rf /tmp/secureboot 2>/dev/null || true
+    rm -rf /tmp/persistent 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Decrypt secureboot keys using the key in the repo
+echo "Decrypting secureboot keys..."
+if [[ ! -f "$FLAKE_DIR/usb-secrets/usb-secrets-key" ]]; then
+    echo "Error: usb-secrets-key not found at $FLAKE_DIR/usb-secrets/usb-secrets-key"
+    exit 1
+fi
+
+nix-shell -p age --run "age -d -i '$FLAKE_DIR/usb-secrets/usb-secrets-key' '$FLAKE_DIR/secrets/secureboot.tar.age'" | \
+    tar -x -C /tmp/secureboot
+
+echo "Secureboot keys extracted"
+
+# Extract persistent partition secrets
+echo "Extracting persistent partition contents..."
+if [[ -f "$FLAKE_DIR/secrets/persistent.tar" ]]; then
+    tar -xzf "$FLAKE_DIR/secrets/persistent.tar" -C /tmp/persistent
+    echo "Persistent partition contents extracted"
+else
+    echo "Warning: persistent.tar not found, skipping persistent secrets"
+fi
+
+# Check if disko-install is available
+if ! command -v disko-install >/dev/null 2>&1; then
+    echo "Running disko-install via nix..."
+    DISKO_INSTALL="nix run github:nix-community/disko#disko-install --"
+else
+    DISKO_INSTALL="disko-install"
+fi
+
+echo "Running disko-install to partition, format, and install NixOS..."
+
+# Build the extra-files arguments
+EXTRA_FILES_ARGS=(
+    --extra-files /tmp/secureboot /etc/secureboot
+    --extra-files "$FLAKE_DIR/usb-secrets/usb-secrets-key" /mnt/usb-secrets/usb-secrets-key
+)
+
+# Add each top-level item from persistent separately to avoid nesting
+# cp -ar creates /dst/src when copying directories, so we need to copy each item
+#
+# Also disko-install actually copies the files from extra-files, so we are good here
+if [[ -d /tmp/persistent ]] && [[ -n "$(ls -A /tmp/persistent 2>/dev/null)" ]]; then
+    for item in /tmp/persistent/*; do
+        if [[ -e "$item" ]]; then
+            basename=$(basename "$item")
+            EXTRA_FILES_ARGS+=(--extra-files "$item" "/persistent/$basename")
+        fi
+    done
+fi
+
+# Run disko-install with secureboot keys available
+sudo $DISKO_INSTALL \
+    --mode format \
+    --flake "$FLAKE_DIR#muffin" \
+    --disk main "$DISK" \
+    "${EXTRA_FILES_ARGS[@]}"

services/arr/bazarr.nix

@@ -0,0 +1,34 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "bazarr" service_configs.zpool_ssds [
+      service_configs.bazarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "bazarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "bazarr" [
+      "Z ${service_configs.bazarr.dataDir} 0700 ${config.services.bazarr.user} ${config.services.bazarr.group}"
+    ])
+  ];
+
+  services.bazarr = {
+    enable = true;
+    listenPort = service_configs.ports.bazarr;
+  };
+
+  services.caddy.virtualHosts."bazarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.bazarr}
+  '';
+
+  users.users.${config.services.bazarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/arr/init.nix

@@ -0,0 +1,115 @@
+{ config, service_configs, ... }:
+{
+  services.arrInit = {
+    prowlarr = {
+      enable = true;
+      serviceName = "prowlarr";
+      port = service_configs.ports.prowlarr;
+      dataDir = service_configs.prowlarr.dataDir;
+      apiVersion = "v1";
+      networkNamespacePath = "/run/netns/wg";
+      syncedApps = [
+        {
+          name = "Sonarr";
+          implementation = "Sonarr";
+          configContract = "SonarrSettings";
+          prowlarrUrl = "http://localhost:${builtins.toString service_configs.ports.prowlarr}";
+          baseUrl = "http://${config.vpnNamespaces.wg.bridgeAddress}:${builtins.toString service_configs.ports.sonarr}";
+          apiKeyFrom = "${service_configs.sonarr.dataDir}/config.xml";
+          syncCategories = [
+            5000
+            5010
+            5020
+            5030
+            5040
+            5045
+            5050
+            5090
+          ];
+          serviceName = "sonarr";
+        }
+        {
+          name = "Radarr";
+          implementation = "Radarr";
+          configContract = "RadarrSettings";
+          prowlarrUrl = "http://localhost:${builtins.toString service_configs.ports.prowlarr}";
+          baseUrl = "http://${config.vpnNamespaces.wg.bridgeAddress}:${builtins.toString service_configs.ports.radarr}";
+          apiKeyFrom = "${service_configs.radarr.dataDir}/config.xml";
+          syncCategories = [
+            2000
+            2010
+            2020
+            2030
+            2040
+            2045
+            2050
+            2060
+            2070
+            2080
+          ];
+          serviceName = "radarr";
+        }
+      ];
+    };
+
+    sonarr = {
+      enable = true;
+      serviceName = "sonarr";
+      port = service_configs.ports.sonarr;
+      dataDir = service_configs.sonarr.dataDir;
+      rootFolders = [ service_configs.media.tvDir ];
+      downloadClients = [
+        {
+          name = "qBittorrent";
+          implementation = "QBittorrent";
+          configContract = "QBittorrentSettings";
+          fields = {
+            host = config.vpnNamespaces.wg.namespaceAddress;
+            port = service_configs.ports.torrent;
+            useSsl = false;
+            tvCategory = "tvshows";
+          };
+        }
+      ];
+    };
+
+    radarr = {
+      enable = true;
+      serviceName = "radarr";
+      port = service_configs.ports.radarr;
+      dataDir = service_configs.radarr.dataDir;
+      rootFolders = [ service_configs.media.moviesDir ];
+      downloadClients = [
+        {
+          name = "qBittorrent";
+          implementation = "QBittorrent";
+          configContract = "QBittorrentSettings";
+          fields = {
+            host = config.vpnNamespaces.wg.namespaceAddress;
+            port = service_configs.ports.torrent;
+            useSsl = false;
+            movieCategory = "movies";
+          };
+        }
+      ];
+    };
+  };
+
+  services.bazarrInit = {
+    enable = true;
+    dataDir = "/var/lib/bazarr";
+    port = service_configs.ports.bazarr;
+    sonarr = {
+      enable = true;
+      dataDir = service_configs.sonarr.dataDir;
+      port = service_configs.ports.sonarr;
+      serviceName = "sonarr";
+    };
+    radarr = {
+      enable = true;
+      dataDir = service_configs.radarr.dataDir;
+      port = service_configs.ports.radarr;
+      serviceName = "radarr";
+    };
+  };
+}

services/arr/jellyseerr.nix

@@ -0,0 +1,43 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "jellyseerr" service_configs.zpool_ssds [
+      service_configs.jellyseerr.configDir
+    ])
+    (lib.serviceFilePerms "jellyseerr" [
+      "Z ${service_configs.jellyseerr.configDir} 0700 jellyseerr jellyseerr"
+    ])
+  ];
+
+  services.jellyseerr = {
+    enable = true;
+    port = service_configs.ports.jellyseerr;
+    configDir = service_configs.jellyseerr.configDir;
+  };
+
+  systemd.services.jellyseerr.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "jellyseerr";
+    Group = "jellyseerr";
+    ReadWritePaths = [ service_configs.jellyseerr.configDir ];
+  };
+
+  users.users.jellyseerr = {
+    isSystemUser = true;
+    group = "jellyseerr";
+    home = service_configs.jellyseerr.configDir;
+  };
+
+  users.groups.jellyseerr = { };
+
+  services.caddy.virtualHosts."jellyseerr.${service_configs.https.domain}".extraConfig = ''
+    # import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.jellyseerr}
+  '';
+}

services/arr/prowlarr.nix

@@ -0,0 +1,30 @@
+{
+  pkgs,
+  service_configs,
+  config,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "prowlarr" service_configs.zpool_ssds [
+      service_configs.prowlarr.dataDir
+    ])
+    (lib.vpnNamespaceOpenPort service_configs.ports.prowlarr "prowlarr")
+  ];
+
+  services.prowlarr = {
+    enable = true;
+    dataDir = service_configs.prowlarr.dataDir;
+    settings.server.port = service_configs.ports.prowlarr;
+  };
+
+  systemd.services.prowlarr.serviceConfig = {
+    ExecStartPre = "+${pkgs.coreutils}/bin/chown -R prowlarr /var/lib/prowlarr";
+  };
+
+  services.caddy.virtualHosts."prowlarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy ${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString service_configs.ports.prowlarr}
+  '';
+}

services/arr/radarr.nix

@@ -0,0 +1,36 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "radarr" service_configs.zpool_ssds [
+      service_configs.radarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "radarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "radarr" [
+      "Z ${service_configs.radarr.dataDir} 0700 ${config.services.radarr.user} ${config.services.radarr.group}"
+    ])
+  ];
+
+  services.radarr = {
+    enable = true;
+    dataDir = service_configs.radarr.dataDir;
+    settings.server.port = service_configs.ports.radarr;
+    settings.update.mechanism = "external";
+  };
+
+  services.caddy.virtualHosts."radarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.radarr}
+  '';
+
+  users.users.${config.services.radarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/arr/recyclarr.nix

@@ -0,0 +1,202 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  radarrConfig = "${service_configs.radarr.dataDir}/config.xml";
+  sonarrConfig = "${service_configs.sonarr.dataDir}/config.xml";
+  appDataDir = "${service_configs.recyclarr.dataDir}/data";
+
+  # Runs as root (via + prefix) to read API keys, writes secrets.yml for recyclarr
+  generateSecrets = pkgs.writeShellScript "recyclarr-generate-secrets" ''
+    RADARR_KEY=$(${pkgs.gnugrep}/bin/grep -oP '(?<=<ApiKey>)[^<]+' ${radarrConfig})
+    SONARR_KEY=$(${pkgs.gnugrep}/bin/grep -oP '(?<=<ApiKey>)[^<]+' ${sonarrConfig})
+    cat > ${appDataDir}/secrets.yml <<EOF
+    movies_api_key: $RADARR_KEY
+    series_api_key: $SONARR_KEY
+    EOF
+    chown recyclarr:recyclarr ${appDataDir}/secrets.yml
+    chmod 600 ${appDataDir}/secrets.yml
+  '';
+in
+{
+  imports = [
+    (lib.serviceMountWithZpool "recyclarr" service_configs.zpool_ssds [
+      service_configs.recyclarr.dataDir
+    ])
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d ${service_configs.recyclarr.dataDir} 0755 recyclarr recyclarr -"
+    "d ${appDataDir} 0755 recyclarr recyclarr -"
+  ];
+
+  services.recyclarr = {
+    enable = true;
+    command = "sync";
+    schedule = "daily";
+    user = "recyclarr";
+    group = "recyclarr";
+
+    configuration = {
+      radarr.movies = {
+        base_url = "http://localhost:${builtins.toString service_configs.ports.radarr}";
+
+        include = [
+          { template = "radarr-quality-definition-movie"; }
+          { template = "radarr-quality-profile-remux-web-2160p"; }
+          { template = "radarr-custom-formats-remux-web-2160p"; }
+        ];
+
+        quality_profiles = [
+          {
+            name = "Remux + WEB 2160p";
+            upgrade = {
+              allowed = true;
+              until_quality = "Remux-2160p";
+            };
+            qualities = [
+              { name = "Remux-2160p"; }
+              {
+                name = "WEB 2160p";
+                qualities = [
+                  "WEBDL-2160p"
+                  "WEBRip-2160p"
+                ];
+              }
+              { name = "Remux-1080p"; }
+              { name = "Bluray-1080p"; }
+              {
+                name = "WEB 1080p";
+                qualities = [
+                  "WEBDL-1080p"
+                  "WEBRip-1080p"
+                ];
+              }
+              { name = "HDTV-1080p"; }
+            ];
+          }
+        ];
+
+        custom_formats = [
+          # Upscaled
+          {
+            trash_ids = [ "bfd8eb01832d646a0a89c4deb46f8564" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = -10000;
+              }
+            ];
+          }
+          # x265 (HD) - override template -10000 penalty
+          {
+            trash_ids = [ "dc98083864ea246d05a42df0d05f81cc" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = 0;
+              }
+            ];
+          }
+          # x265 (no HDR/DV) - override template -10000 penalty
+          {
+            trash_ids = [ "839bea857ed2c0a8e084f3cbdbd65ecb" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = 0;
+              }
+            ];
+          }
+        ];
+      };
+
+      sonarr.series = {
+        base_url = "http://localhost:${builtins.toString service_configs.ports.sonarr}";
+
+        include = [
+          { template = "sonarr-quality-definition-series"; }
+          { template = "sonarr-v4-quality-profile-web-2160p"; }
+          { template = "sonarr-v4-custom-formats-web-2160p"; }
+        ];
+
+        quality_profiles = [
+          {
+            name = "WEB-2160p";
+            upgrade = {
+              allowed = true;
+              until_quality = "WEB 2160p";
+            };
+            qualities = [
+              {
+                name = "WEB 2160p";
+                qualities = [
+                  "WEBDL-2160p"
+                  "WEBRip-2160p"
+                ];
+              }
+              { name = "Bluray-1080p Remux"; }
+              { name = "Bluray-1080p"; }
+              {
+                name = "WEB 1080p";
+                qualities = [
+                  "WEBDL-1080p"
+                  "WEBRip-1080p"
+                ];
+              }
+              { name = "HDTV-1080p"; }
+            ];
+          }
+        ];
+
+        custom_formats = [
+          # Upscaled
+          {
+            trash_ids = [ "23297a736ca77c0fc8e70f8edd7ee56c" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = -10000;
+              }
+            ];
+          }
+          # x265 (HD) - override template -10000 penalty
+          {
+            trash_ids = [ "47435ece6b99a0b477caf360e79ba0bb" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = 0;
+              }
+            ];
+          }
+          # x265 (no HDR/DV) - override template -10000 penalty
+          {
+            trash_ids = [ "9b64dff695c2115facf1b6ea59c9bd07" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = 0;
+              }
+            ];
+          }
+        ];
+      };
+    };
+  };
+
+  # Add secrets generation before recyclarr runs
+  systemd.services.recyclarr = {
+    after = [
+      "network-online.target"
+      "radarr.service"
+      "sonarr.service"
+    ];
+    wants = [ "network-online.target" ];
+    serviceConfig.ExecStartPre = "+${generateSecrets}";
+  };
+}

services/arr/sonarr.nix

@@ -0,0 +1,42 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "sonarr" service_configs.zpool_ssds [
+      service_configs.sonarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "sonarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "sonarr" [
+      "Z ${service_configs.sonarr.dataDir} 0700 ${config.services.sonarr.user} ${config.services.sonarr.group}"
+    ])
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d /torrents/media 2775 root ${service_configs.media_group} -"
+    "d ${service_configs.media.tvDir} 2775 root ${service_configs.media_group} -"
+    "d ${service_configs.media.moviesDir} 2775 root ${service_configs.media_group} -"
+  ];
+
+  services.sonarr = {
+    enable = true;
+    dataDir = service_configs.sonarr.dataDir;
+    settings.server.port = service_configs.ports.sonarr;
+    settings.update.mechanism = "external";
+  };
+
+  services.caddy.virtualHosts."sonarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.sonarr}
+  '';
+
+  users.users.${config.services.sonarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/bitmagnet.nix

@@ -2,24 +2,13 @@
   pkgs,
   service_configs,
   config,
+  lib,
   ...
 }:
 {
-  vpnNamespaces.wg = {
-    portMappings = [
-      {
-        from = service_configs.ports.bitmagnet;
-        to = service_configs.ports.bitmagnet;
-      }
-    ];
-
-    openVPNPorts = [
-      {
-        port = service_configs.ports.bitmagnet;
-        protocol = "both";
-      }
-    ];
-  };
+  imports = [
+    (lib.vpnNamespaceOpenPort service_configs.ports.bitmagnet "bitmagnet")
+  ];
 
   services.bitmagnet = {
     enable = true;
@@ -36,13 +25,7 @@
   };
 
   services.caddy.virtualHosts."bitmagnet.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
-    ${builtins.readFile ../secrets/caddy_auth}
-    reverse_proxy ${service_configs.https.wg_ip}:${builtins.toString service_configs.ports.bitmagnet}
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy ${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString service_configs.ports.bitmagnet}
   '';
-
-  systemd.services.bitmagnet.vpnConfinement = {
-    enable = true;
-    vpnNamespace = "wg";
-  };
 }

services/bitwarden.nix

@@ -0,0 +1,60 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "vaultwarden" service_configs.zpool_ssds [
+      service_configs.vaultwarden.path
+      config.services.vaultwarden.backupDir
+    ])
+    (lib.serviceMountWithZpool "backup-vaultwarden" service_configs.zpool_ssds [
+      service_configs.vaultwarden.path
+      config.services.vaultwarden.backupDir
+    ])
+    (lib.serviceFilePerms "vaultwarden" [
+      "Z ${service_configs.vaultwarden.path} 0700 vaultwarden vaultwarden"
+      "Z ${config.services.vaultwarden.backupDir} 0700 vaultwarden vaultwarden"
+    ])
+  ];
+
+  services.vaultwarden = {
+    enable = true;
+    backupDir = "/${service_configs.zpool_ssds}/bak/vaultwarden";
+    config = {
+      # Refer to https://github.com/dani-garcia/vaultwarden/blob/main/.env.template
+      DOMAIN = "https://bitwarden.${service_configs.https.domain}";
+      SIGNUPS_ALLOWED = false;
+
+      ROCKET_ADDRESS = "127.0.0.1";
+      ROCKET_PORT = service_configs.ports.vaultwarden;
+      ROCKET_LOG = "critical";
+    };
+  };
+
+  services.caddy.virtualHosts."bitwarden.${service_configs.https.domain}".extraConfig = ''
+    encode zstd gzip
+
+    reverse_proxy :${toString config.services.vaultwarden.config.ROCKET_PORT} {
+        header_up X-Real-IP {remote_host}
+    }
+  '';
+
+  # Protect Vaultwarden login from brute force attacks
+  services.fail2ban.jails.vaultwarden = {
+    enabled = true;
+    settings = {
+      backend = "systemd";
+      port = "http,https";
+      # defaults: maxretry=5, findtime=10m, bantime=10m
+    };
+    filter.Definition = {
+      failregex = ''^.*Username or password is incorrect\. Try again\. IP: <HOST>\..*$'';
+      ignoreregex = "";
+      journalmatch = "_SYSTEMD_UNIT=vaultwarden.service";
+    };
+  };
+}

services/caddy.nix

@@ -1,18 +1,61 @@
 {
   config,
   service_configs,
-  username,
   pkgs,
+  lib,
+  inputs,
   ...
 }:
+
+let
+  theme = pkgs.fetchFromGitHub {
+    owner = "kaiiiz";
+    repo = "hugo-theme-monochrome";
+    rev = "d17e05715e91f41a842f2656e6bdd70cba73de91";
+    sha256 = "h9I2ukugVrldIC3SXefS0L3R245oa+TuRChOCJJgF24=";
+  };
+
+  hugo-neko = pkgs.fetchFromGitHub {
+    owner = "ystepanoff";
+    repo = "hugo-neko";
+    rev = "5a50034acbb1ae0cec19775af64e7167ca22725e";
+    sha256 = "VLwr4zEeFQU/b+vj0XTLSuEiosuNFu2du4uud7m8bnw=";
+  };
+
+  hugoWebsite = pkgs.stdenv.mkDerivation {
+    pname = "hugo-site";
+    version = "0.1";
+
+    src = inputs.website;
+
+    nativeBuildInputs = with pkgs; [
+      hugo
+      go
+      git
+    ];
+
+    installPhase = ''
+      rm -fr themes/theme modules/hugo-neko
+      cp -r ${theme} themes/theme
+      cp -r ${hugo-neko} modules/hugo-neko
+      hugo --minify -d $out;
+    '';
+  };
+in
 {
+  imports = [
+    (lib.serviceMountWithZpool "caddy" service_configs.zpool_ssds [
+      config.services.caddy.dataDir
+    ])
+  ];
+
   services.caddy = {
     enable = true;
     email = "titaniumtown@proton.me";
     virtualHosts = {
       ${service_configs.https.domain} = {
         extraConfig = ''
-          root * ${service_configs.https.data_dir}
+          root * ${hugoWebsite}
           file_server browse
         '';
 
@@ -22,7 +65,7 @@
   };
 
   systemd.tmpfiles.rules = [
-    "d ${service_configs.https.data_dir} 770 ${config.services.caddy.user} ${config.services.caddy.group}"
+    "d ${config.services.caddy.dataDir} 700 ${config.services.caddy.user} ${config.services.caddy.group}"
   ];
 
   systemd.packages = with pkgs; [ nssTools ];
@@ -31,25 +74,34 @@
     service_configs.ports.https
 
     # http (but really acmeCA challenges)
-    80
-
-    # for matrix federation
-    8448
+    service_configs.ports.http
   ];
 
   networking.firewall.allowedUDPPorts = [
     service_configs.ports.https
-
-    # for matrix federation
-    8448
   ];
 
-  users.users.${config.services.caddy.user}.extraGroups = [
-    # for `map.gardling.com`
-    "minecraft"
-  ];
+  # Protect Caddy basic auth endpoints from brute force attacks
+  services.fail2ban.jails.caddy-auth = {
+    enabled = true;
+    settings = {
+      backend = "auto";
+      port = "http,https";
+      logpath = "/var/log/caddy/access-*.log";
+      # defaults: maxretry=5, findtime=10m, bantime=10m
 
-  users.users.${username}.extraGroups = [
-    config.services.caddy.group
-  ];
+      # Ignore local network IPs - NAT hairpinning causes all LAN traffic to
+      # appear from the router IP (192.168.1.1). Banning it blocks all internal access.
+      ignoreip = "127.0.0.1/8 ::1 192.168.1.0/24";
+    };
+    filter.Definition = {
+      # Only match 401s where an Authorization header was actually sent.
+      # Without this, the normal HTTP Basic Auth challenge-response flow
+      # (browser probes without credentials, gets 401, then resends with
+      # credentials) counts every page visit as a "failure."
+      failregex = ''^.*"remote_ip":"<HOST>".*"Authorization":\["REDACTED"\].*"status":401.*$'';
+      ignoreregex = "";
+      datepattern = ''"ts":{Epoch}\.'';
+    };
+  };
 }

services/caddy_senior_project.nix

@@ -0,0 +1,39 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  inputs,
+  ...
+}:
+let
+  theme = pkgs.fetchFromGitHub {
+    owner = "kaiiiz";
+    repo = "hugo-theme-monochrome";
+    rev = "d17e05715e91f41a842f2656e6bdd70cba73de91";
+    sha256 = "h9I2ukugVrldIC3SXefS0L3R245oa+TuRChOCJJgF24=";
+  };
+
+  hugoWebsite = pkgs.stdenv.mkDerivation {
+    pname = "hugo-site";
+    version = "0.1";
+
+    src = inputs.senior_project-website;
+
+    nativeBuildInputs = with pkgs; [
+      hugo
+    ];
+
+    installPhase = ''
+      rm -fr themes/theme
+      cp -rv ${theme} themes/theme
+      hugo --minify -d $out;
+    '';
+  };
+in
+{
+  services.caddy.virtualHosts."senior-project.${service_configs.https.domain}".extraConfig = ''
+    root * ${hugoWebsite}
+    file_server browse
+  '';
+}

services/coturn.nix

@@ -0,0 +1,59 @@
+{
+  config,
+  lib,
+  service_configs,
+  ...
+}:
+{
+  services.coturn = {
+    enable = true;
+    realm = service_configs.https.domain;
+    use-auth-secret = true;
+    static-auth-secret = lib.strings.trim (builtins.readFile ../secrets/coturn_static_auth_secret);
+    listening-port = service_configs.ports.coturn;
+    tls-listening-port = service_configs.ports.coturn_tls;
+    no-cli = true;
+
+    # recommended security settings from Synapse's coturn docs
+    extraConfig = ''
+      denied-peer-ip=10.0.0.0-10.255.255.255
+      denied-peer-ip=192.168.0.0-192.168.255.255
+      denied-peer-ip=172.16.0.0-172.31.255.255
+      denied-peer-ip=0.0.0.0-0.255.255.255
+      denied-peer-ip=100.64.0.0-100.127.255.255
+      denied-peer-ip=169.254.0.0-169.254.255.255
+      denied-peer-ip=192.0.0.0-192.0.0.255
+      denied-peer-ip=198.18.0.0-198.19.255.255
+      denied-peer-ip=198.51.100.0-198.51.100.255
+      denied-peer-ip=203.0.113.0-203.0.113.255
+      denied-peer-ip=240.0.0.0-255.255.255.255
+      denied-peer-ip=::1
+      denied-peer-ip=64:ff9b::-64:ff9b::ffff:ffff
+      denied-peer-ip=::ffff:0.0.0.0-::ffff:255.255.255.255
+      denied-peer-ip=100::-100::ffff:ffff:ffff:ffff
+      denied-peer-ip=2001::-2001:1ff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=2002::-2002:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=fc00::-fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+      denied-peer-ip=fe80::-febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+    '';
+  };
+
+  # coturn needs these ports open
+  networking.firewall = {
+    allowedTCPPorts = [
+      service_configs.ports.coturn
+      service_configs.ports.coturn_tls
+    ];
+    allowedUDPPorts = [
+      service_configs.ports.coturn
+      service_configs.ports.coturn_tls
+    ];
+    # relay port range
+    allowedUDPPortRanges = [
+      {
+        from = config.services.coturn.min-port;
+        to = config.services.coturn.max-port;
+      }
+    ];
+  };
+}

services/gitea.nix

@@ -1,10 +1,18 @@
 {
+  pkgs,
+  lib,
   config,
   service_configs,
-  username,
   ...
 }:
 {
+  imports = [
+    (lib.serviceMountWithZpool "gitea" service_configs.zpool_ssds [ config.services.gitea.stateDir ])
+    (lib.serviceFilePerms "gitea" [
+      "Z ${config.services.gitea.stateDir} 0700 ${config.services.gitea.user} ${config.services.gitea.group}"
+    ])
+  ];
+
   services.gitea = {
     enable = true;
     appName = "Simon Gardling's Gitea instance";
@@ -36,11 +44,6 @@
     reverse_proxy :${builtins.toString config.services.gitea.settings.server.HTTP_PORT}
   '';
 
-  systemd.tmpfiles.rules = [
-    # 0700 for ssh permission reasons
-    "d ${config.services.gitea.stateDir} 0700 ${config.services.gitea.user} ${config.services.gitea.group}"
-  ];
-
   services.postgresql = {
     ensureDatabases = [ config.services.gitea.user ];
     ensureUsers = [
@@ -54,7 +57,18 @@
 
   services.openssh.settings.AllowUsers = [ config.services.gitea.user ];
 
-  users.users.${username}.extraGroups = [
-    config.services.gitea.group
-  ];
+  # Protect Gitea login from brute force attacks
+  services.fail2ban.jails.gitea = {
+    enabled = true;
+    settings = {
+      backend = "systemd";
+      port = "http,https";
+      # defaults: maxretry=5, findtime=10m, bantime=10m
+    };
+    filter.Definition = {
+      failregex = "^.*Failed authentication attempt for .* from <HOST>:.*$";
+      ignoreregex = "";
+      journalmatch = "_SYSTEMD_UNIT=gitea.service";
+    };
+  };
 }

services/graphing-calculator.nix

@@ -0,0 +1,16 @@
+{
+  service_configs,
+  inputs,
+  pkgs,
+  ...
+}:
+let
+  graphing-calculator =
+    inputs.ytbn-graphing-software.packages.${pkgs.stdenv.targetPlatform.system}.web;
+in
+{
+  services.caddy.virtualHosts."graphing.${service_configs.https.domain}".extraConfig = ''
+    root * ${graphing-calculator}
+    file_server browse
+  '';
+}

services/immich.nix

@@ -2,10 +2,22 @@
   service_configs,
   pkgs,
   config,
-  username,
+  lib,
   ...
 }:
 {
+  imports = [
+    (lib.serviceMountWithZpool "immich-server" service_configs.zpool_ssds [
+      config.services.immich.mediaLocation
+    ])
+    (lib.serviceMountWithZpool "immich-machine-learning" service_configs.zpool_ssds [
+      config.services.immich.mediaLocation
+    ])
+    (lib.serviceFilePerms "immich-server" [
+      "Z ${config.services.immich.mediaLocation} 0770 ${config.services.immich.user} ${config.services.immich.group}"
+    ])
+  ];
+
   services.immich = {
     enable = true;
     mediaLocation = service_configs.immich.dir;
@@ -21,10 +33,6 @@
     reverse_proxy :${builtins.toString config.services.immich.port}
   '';
 
-  systemd.tmpfiles.rules = [
-    "d ${config.services.immich.mediaLocation} 0770 ${config.services.immich.user} ${config.services.immich.group}"
-  ];
-
   environment.systemPackages = with pkgs; [
     immich-go
   ];
@@ -34,7 +42,18 @@
     "render"
   ];
 
-  users.users.${username}.extraGroups = [
-    config.services.immich.group
-  ];
+  # Protect Immich login from brute force attacks
+  services.fail2ban.jails.immich = {
+    enabled = true;
+    settings = {
+      backend = "systemd";
+      port = "http,https";
+      # defaults: maxretry=5, findtime=10m, bantime=10m
+    };
+    filter.Definition = {
+      failregex = "^.*Failed login attempt for user .* from ip address <HOST>.*$";
+      ignoreregex = "";
+      journalmatch = "_SYSTEMD_UNIT=immich-server.service";
+    };
+  };
 }

services/jellyfin-qbittorrent-monitor.nix

@@ -0,0 +1,57 @@
+{
+  pkgs,
+  service_configs,
+  config,
+  ...
+}:
+{
+  systemd.services."jellyfin-qbittorrent-monitor" = {
+    description = "Monitor Jellyfin streaming and control qBittorrent rate limits";
+    after = [
+      "network.target"
+      "jellyfin.service"
+      "qbittorrent.service"
+    ];
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = pkgs.writeShellScript "jellyfin-monitor-start" ''
+        export JELLYFIN_API_KEY=$(cat $CREDENTIALS_DIRECTORY/jellyfin-api-key)
+        exec ${
+          pkgs.python3.withPackages (ps: with ps; [ requests ])
+        }/bin/python ${./jellyfin-qbittorrent-monitor.py}
+      '';
+      Restart = "always";
+      RestartSec = "10s";
+
+      # Security hardening
+      DynamicUser = true;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      ProtectKernelTunables = true;
+      ProtectKernelModules = true;
+      ProtectControlGroups = true;
+      MemoryDenyWriteExecute = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      RemoveIPC = true;
+
+      # Load credentials from agenix secrets
+      LoadCredential = "jellyfin-api-key:${config.age.secrets.jellyfin-api-key.path}";
+    };
+
+    environment = {
+      JELLYFIN_URL = "http://localhost:${builtins.toString service_configs.ports.jellyfin}";
+      QBITTORRENT_URL = "http://${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString service_configs.ports.torrent}";
+      CHECK_INTERVAL = "30";
+      # Bandwidth budget configuration
+      TOTAL_BANDWIDTH_BUDGET = "30000000"; # 30 Mbps in bits per second
+      SERVICE_BUFFER = "5000000"; # 5 Mbps reserved for other services (bps)
+      DEFAULT_STREAM_BITRATE = "10000000"; # 10 Mbps fallback when bitrate unknown (bps)
+      MIN_TORRENT_SPEED = "100"; # KB/s - below this, pause torrents instead
+      STREAM_BITRATE_HEADROOM = "1.1"; # multiplier per stream for bitrate fluctuations
+    };
+  };
+}

services/jellyfin-qbittorrent-monitor.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+
+import requests
+import time
+import logging
+import sys
+import signal
+import json
+import ipaddress
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class ServiceUnavailable(Exception):
+    """Raised when a monitored service is temporarily unavailable."""
+
+    pass
+
+
+class JellyfinQBittorrentMonitor:
+    def __init__(
+        self,
+        jellyfin_url="http://localhost:8096",
+        qbittorrent_url="http://localhost:8080",
+        check_interval=30,
+        jellyfin_api_key=None,
+        streaming_start_delay=10,
+        streaming_stop_delay=60,
+        total_bandwidth_budget=30000000,
+        service_buffer=5000000,
+        default_stream_bitrate=10000000,
+        min_torrent_speed=100,
+        stream_bitrate_headroom=1.1,
+    ):
+        self.jellyfin_url = jellyfin_url
+        self.qbittorrent_url = qbittorrent_url
+        self.check_interval = check_interval
+        self.jellyfin_api_key = jellyfin_api_key
+        self.total_bandwidth_budget = total_bandwidth_budget
+        self.service_buffer = service_buffer
+        self.default_stream_bitrate = default_stream_bitrate
+        self.min_torrent_speed = min_torrent_speed
+        self.stream_bitrate_headroom = stream_bitrate_headroom
+        self.last_streaming_state = None
+        self.current_state = "unlimited"
+        self.torrents_paused = False
+        self.last_alt_limits = None
+        self.running = True
+        self.session = requests.Session()  # Use session for cookies
+        self.last_active_streams = []
+
+        # Hysteresis settings to prevent rapid switching
+        self.streaming_start_delay = streaming_start_delay
+        self.streaming_stop_delay = streaming_stop_delay
+        self.last_state_change = 0
+
+        # Local network ranges (RFC 1918 private networks + localhost)
+        self.local_networks = [
+            ipaddress.ip_network("10.0.0.0/8"),
+            ipaddress.ip_network("172.16.0.0/12"),
+            ipaddress.ip_network("192.168.0.0/16"),
+            ipaddress.ip_network("127.0.0.0/8"),
+            ipaddress.ip_network("::1/128"),  # IPv6 localhost
+            ipaddress.ip_network("fe80::/10"),  # IPv6 link-local
+        ]
+
+    def is_local_ip(self, ip_address: str) -> bool:
+        """Check if an IP address is from a local network"""
+        try:
+            ip = ipaddress.ip_address(ip_address)
+            return any(ip in network for network in self.local_networks)
+        except ValueError:
+            logger.warning(f"Invalid IP address format: {ip_address}")
+            return True  # Treat invalid IPs as local for safety
+
+    def signal_handler(self, signum, frame):
+        logger.info("Received shutdown signal, cleaning up...")
+        self.running = False
+        self.restore_normal_limits()
+        sys.exit(0)
+
+    def check_jellyfin_sessions(self) -> list[dict]:
+        headers = (
+            {"X-Emby-Token": self.jellyfin_api_key} if self.jellyfin_api_key else {}
+        )
+
+        try:
+            response = requests.get(
+                f"{self.jellyfin_url}/Sessions", headers=headers, timeout=10
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to check Jellyfin sessions: {e}")
+            raise ServiceUnavailable(f"Jellyfin unavailable: {e}") from e
+
+        try:
+            sessions = response.json()
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse Jellyfin response: {e}")
+            raise ServiceUnavailable(f"Jellyfin returned invalid JSON: {e}") from e
+
+        active_streams = []
+        for session in sessions:
+            if (
+                "NowPlayingItem" in session
+                and not session.get("PlayState", {}).get("IsPaused", True)
+                and not self.is_local_ip(session.get("RemoteEndPoint", ""))
+            ):
+                item = session["NowPlayingItem"]
+                item_type = item.get("Type", "").lower()
+                if item_type in ["movie", "episode", "video"]:
+                    user = session.get("UserName", "Unknown")
+                    stream_name = f"{user}: {item.get('Name', 'Unknown')}"
+                    if session.get("TranscodingInfo") and session[
+                        "TranscodingInfo"
+                    ].get("Bitrate"):
+                        bitrate = session["TranscodingInfo"]["Bitrate"]
+                    elif item.get("Bitrate"):
+                        bitrate = item["Bitrate"]
+                    elif item.get("MediaSources", [{}])[0].get("Bitrate"):
+                        bitrate = item["MediaSources"][0]["Bitrate"]
+                    else:
+                        bitrate = self.default_stream_bitrate
+
+                    bitrate = min(int(bitrate), 100_000_000)
+                    # Add headroom to account for bitrate fluctuations
+                    bitrate = int(bitrate * self.stream_bitrate_headroom)
+                    active_streams.append({"name": stream_name, "bitrate_bps": bitrate})
+
+        return active_streams
+
+    def check_qbittorrent_alternate_limits(self) -> bool:
+        try:
+            response = self.session.get(
+                f"{self.qbittorrent_url}/api/v2/transfer/speedLimitsMode", timeout=10
+            )
+            if response.status_code == 200:
+                return response.text.strip() == "1"
+            else:
+                logger.warning(
+                    f"SpeedLimitsMode endpoint returned HTTP {response.status_code}"
+                )
+                raise ServiceUnavailable(
+                    f"qBittorrent returned HTTP {response.status_code}"
+                )
+        except requests.exceptions.RequestException as e:
+            logger.error(f"SpeedLimitsMode endpoint failed: {e}")
+            raise ServiceUnavailable(f"qBittorrent unavailable: {e}") from e
+
+    def use_alt_limits(self, enable: bool) -> None:
+        action = "enabled" if enable else "disabled"
+        try:
+            current_throttle = self.check_qbittorrent_alternate_limits()
+
+            if current_throttle == enable:
+                logger.debug(
+                    f"Alternate speed limits already {action}, no action needed"
+                )
+                return
+
+            response = self.session.post(
+                f"{self.qbittorrent_url}/api/v2/transfer/toggleSpeedLimitsMode",
+                timeout=10,
+            )
+            response.raise_for_status()
+            new_state = self.check_qbittorrent_alternate_limits()
+            if new_state == enable:
+                logger.info(f"Alternate speed limits {action}")
+            else:
+                logger.warning(
+                    f"Toggle may have failed: expected {enable}, got {new_state}"
+                )
+
+        except ServiceUnavailable:
+            logger.warning(
+                f"qBittorrent unavailable, cannot {action} alternate speed limits"
+            )
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to {action} alternate speed limits: {e}")
+
+    def pause_all_torrents(self) -> None:
+        try:
+            response = self.session.post(
+                f"{self.qbittorrent_url}/api/v2/torrents/stop",
+                data={"hashes": "all"},
+                timeout=10,
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to pause torrents: {e}")
+
+    def resume_all_torrents(self) -> None:
+        try:
+            response = self.session.post(
+                f"{self.qbittorrent_url}/api/v2/torrents/start",
+                data={"hashes": "all"},
+                timeout=10,
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to resume torrents: {e}")
+
+    def set_alt_speed_limits(self, dl_kbs: float, ul_kbs: float) -> None:
+        try:
+            payload = {
+                "alt_dl_limit": int(dl_kbs * 1024),
+                "alt_up_limit": int(ul_kbs * 1024),
+            }
+            response = self.session.post(
+                f"{self.qbittorrent_url}/api/v2/app/setPreferences",
+                data={"json": json.dumps(payload)},
+                timeout=10,
+            )
+            response.raise_for_status()
+            self.last_alt_limits = (dl_kbs, ul_kbs)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to set alternate speed limits: {e}")
+
+    def restore_normal_limits(self) -> None:
+        if self.torrents_paused:
+            logger.info("Resuming all torrents before shutdown...")
+            self.resume_all_torrents()
+            self.torrents_paused = False
+
+        if self.current_state != "unlimited":
+            logger.info("Restoring normal speed limits before shutdown...")
+        self.use_alt_limits(False)
+        self.current_state = "unlimited"
+
+    def sync_qbittorrent_state(self) -> None:
+        try:
+            if self.current_state == "unlimited":
+                actual_state = self.check_qbittorrent_alternate_limits()
+                if actual_state:
+                    logger.warning(
+                        "qBittorrent state mismatch detected: expected alt speed OFF, got ON. Re-syncing..."
+                    )
+                    self.use_alt_limits(False)
+            elif self.current_state == "throttled":
+                if self.last_alt_limits:
+                    self.set_alt_speed_limits(*self.last_alt_limits)
+                actual_state = self.check_qbittorrent_alternate_limits()
+                if not actual_state:
+                    logger.warning(
+                        "qBittorrent state mismatch detected: expected alt speed ON, got OFF. Re-syncing..."
+                    )
+                    self.use_alt_limits(True)
+            elif self.current_state == "paused":
+                self.pause_all_torrents()
+                self.torrents_paused = True
+        except ServiceUnavailable:
+            pass
+
+    def should_change_state(self, new_streaming_state: bool) -> bool:
+        """Apply hysteresis to prevent rapid state changes"""
+        now = time.time()
+
+        if new_streaming_state == self.last_streaming_state:
+            return False
+
+        time_since_change = now - self.last_state_change
+
+        if new_streaming_state and not self.last_streaming_state:
+            if time_since_change >= self.streaming_start_delay:
+                self.last_state_change = now
+                return True
+            else:
+                remaining = self.streaming_start_delay - time_since_change
+                logger.info(
+                    f"Streaming started - waiting {remaining:.1f}s before enforcing limits"
+                )
+
+        elif not new_streaming_state and self.last_streaming_state:
+            if time_since_change >= self.streaming_stop_delay:
+                self.last_state_change = now
+                return True
+            else:
+                remaining = self.streaming_stop_delay - time_since_change
+                logger.info(
+                    f"Streaming stopped - waiting {remaining:.1f}s before restoring unlimited mode"
+                )
+
+        return False
+
+    def run(self):
+        logger.info("Starting Jellyfin-qBittorrent monitor")
+        logger.info(f"Jellyfin URL: {self.jellyfin_url}")
+        logger.info(f"qBittorrent URL: {self.qbittorrent_url}")
+        logger.info(f"Check interval: {self.check_interval}s")
+        logger.info(f"Streaming start delay: {self.streaming_start_delay}s")
+        logger.info(f"Streaming stop delay: {self.streaming_stop_delay}s")
+        logger.info(f"Total bandwidth budget: {self.total_bandwidth_budget} bps")
+        logger.info(f"Service buffer: {self.service_buffer} bps")
+        logger.info(f"Default stream bitrate: {self.default_stream_bitrate} bps")
+        logger.info(f"Minimum torrent speed: {self.min_torrent_speed} KB/s")
+        logger.info(f"Stream bitrate headroom: {self.stream_bitrate_headroom}x")
+
+        signal.signal(signal.SIGINT, self.signal_handler)
+        signal.signal(signal.SIGTERM, self.signal_handler)
+
+        while self.running:
+            try:
+                self.sync_qbittorrent_state()
+
+                try:
+                    active_streams = self.check_jellyfin_sessions()
+                except ServiceUnavailable:
+                    logger.warning("Jellyfin unavailable, maintaining current state")
+                    time.sleep(self.check_interval)
+                    continue
+
+                streaming_active = len(active_streams) > 0
+
+                if active_streams:
+                    for stream in active_streams:
+                        logger.debug(
+                            f"Active stream: {stream['name']} ({stream['bitrate_bps']} bps)"
+                        )
+
+                if active_streams != self.last_active_streams:
+                    if streaming_active:
+                        stream_names = ", ".join(
+                            stream["name"] for stream in active_streams
+                        )
+                        logger.info(
+                            f"Active streams ({len(active_streams)}): {stream_names}"
+                        )
+                    elif len(active_streams) == 0 and self.last_streaming_state:
+                        logger.info("No active streaming sessions")
+
+                if self.should_change_state(streaming_active):
+                    self.last_streaming_state = streaming_active
+
+                streaming_state = bool(self.last_streaming_state)
+                total_streaming_bps = sum(
+                    stream["bitrate_bps"] for stream in active_streams
+                )
+                remaining_bps = (
+                    self.total_bandwidth_budget
+                    - self.service_buffer
+                    - total_streaming_bps
+                )
+                remaining_kbs = max(0, remaining_bps) / 8 / 1024
+
+                if not streaming_state:
+                    desired_state = "unlimited"
+                elif streaming_active:
+                    if remaining_kbs >= self.min_torrent_speed:
+                        desired_state = "throttled"
+                    else:
+                        desired_state = "paused"
+                else:
+                    desired_state = self.current_state
+
+                if desired_state != self.current_state:
+                    if desired_state == "unlimited":
+                        action = "resume torrents, disable alt speed"
+                    elif desired_state == "throttled":
+                        action = (
+                            "set alt limits "
+                            f"dl={int(remaining_kbs)}KB/s ul={int(remaining_kbs)}KB/s, enable alt speed"
+                        )
+                    else:
+                        action = "pause torrents"
+
+                    logger.info(
+                        "State change %s -> %s | streams=%d total_bps=%d remaining_bps=%d action=%s",
+                        self.current_state,
+                        desired_state,
+                        len(active_streams),
+                        total_streaming_bps,
+                        remaining_bps,
+                        action,
+                    )
+
+                if desired_state == "unlimited":
+                    if self.torrents_paused:
+                        self.resume_all_torrents()
+                        self.torrents_paused = False
+                    self.use_alt_limits(False)
+                elif desired_state == "throttled":
+                    if self.torrents_paused:
+                        self.resume_all_torrents()
+                        self.torrents_paused = False
+                    self.set_alt_speed_limits(remaining_kbs, remaining_kbs)
+                    self.use_alt_limits(True)
+                else:
+                    if not self.torrents_paused:
+                        self.pause_all_torrents()
+                        self.torrents_paused = True
+
+                self.current_state = desired_state
+                self.last_active_streams = active_streams
+                time.sleep(self.check_interval)
+
+            except KeyboardInterrupt:
+                break
+            except Exception as e:
+                logger.error(f"Unexpected error in monitoring loop: {e}")
+                time.sleep(self.check_interval)
+
+        self.restore_normal_limits()
+        logger.info("Monitor stopped")
+
+
+if __name__ == "__main__":
+    import os
+
+    # Configuration from environment variables
+    jellyfin_url = os.getenv("JELLYFIN_URL", "http://localhost:8096")
+    qbittorrent_url = os.getenv("QBITTORRENT_URL", "http://localhost:8080")
+    check_interval = int(os.getenv("CHECK_INTERVAL", "30"))
+    jellyfin_api_key = os.getenv("JELLYFIN_API_KEY")
+    streaming_start_delay = int(os.getenv("STREAMING_START_DELAY", "10"))
+    streaming_stop_delay = int(os.getenv("STREAMING_STOP_DELAY", "60"))
+    total_bandwidth_budget = int(os.getenv("TOTAL_BANDWIDTH_BUDGET", "30000000"))
+    service_buffer = int(os.getenv("SERVICE_BUFFER", "5000000"))
+    default_stream_bitrate = int(os.getenv("DEFAULT_STREAM_BITRATE", "10000000"))
+    min_torrent_speed = int(os.getenv("MIN_TORRENT_SPEED", "100"))
+    stream_bitrate_headroom = float(os.getenv("STREAM_BITRATE_HEADROOM", "1.1"))
+
+    monitor = JellyfinQBittorrentMonitor(
+        jellyfin_url=jellyfin_url,
+        qbittorrent_url=qbittorrent_url,
+        check_interval=check_interval,
+        jellyfin_api_key=jellyfin_api_key,
+        streaming_start_delay=streaming_start_delay,
+        streaming_stop_delay=streaming_stop_delay,
+        total_bandwidth_budget=total_bandwidth_budget,
+        service_buffer=service_buffer,
+        default_stream_bitrate=default_stream_bitrate,
+        min_torrent_speed=min_torrent_speed,
+        stream_bitrate_headroom=stream_bitrate_headroom,
+    )
+
+    monitor.run()

services/jellyfin.nix

@@ -2,44 +2,59 @@
   pkgs,
   config,
   service_configs,
-  username,
+  lib,
   ...
 }:
 {
-  environment.systemPackages = with pkgs; [
-    jellyfin
-    jellyfin-web
-    jellyfin-ffmpeg
+  imports = [
+    (lib.serviceMountWithZpool "jellyfin" service_configs.zpool_ssds [
+      config.services.jellyfin.dataDir
+      config.services.jellyfin.cacheDir
+    ])
+    (lib.serviceFilePerms "jellyfin" [
+      "Z ${config.services.jellyfin.dataDir} 0700 ${config.services.jellyfin.user} ${config.services.jellyfin.group}"
+      "Z ${config.services.jellyfin.cacheDir} 0700 ${config.services.jellyfin.user} ${config.services.jellyfin.group}"
+    ])
   ];
 
-  services.jellyfin = rec {
+  services.jellyfin = {
     enable = true;
     # used for local streaming
     openFirewall = true;
+    package = pkgs.jellyfin.override { jellyfin-ffmpeg = (lib.optimizePackage pkgs.jellyfin-ffmpeg); };
 
-    dataDir = service_configs.jellyfin.dir;
-    cacheDir = dataDir + "_cache";
+    inherit (service_configs.jellyfin) dataDir cacheDir;
   };
 
   services.caddy.virtualHosts."jellyfin.${service_configs.https.domain}".extraConfig = ''
-    reverse_proxy :${builtins.toString service_configs.ports.jellyfin}
+    reverse_proxy :${builtins.toString service_configs.ports.jellyfin} {
+      header_up X-Real-IP {remote_host}
+      header_up X-Forwarded-For {remote_host}
+      header_up X-Forwarded-Proto {scheme}
+    }
     request_body {
     	max_size 4096MB
     }
   '';
 
-  systemd.tmpfiles.rules = [
-    "d ${config.services.jellyfin.dataDir} 0700 ${config.services.jellyfin.user} ${config.services.jellyfin.group}"
-    "d ${config.services.jellyfin.cacheDir} 0700 ${config.services.jellyfin.user} ${config.services.jellyfin.group}"
-  ];
-
   users.users.${config.services.jellyfin.user}.extraGroups = [
     "video"
     "render"
-    service_configs.torrent_group
+    service_configs.media_group
   ];
 
-  users.users.${username}.extraGroups = [
-    config.services.jellyfin.group
-  ];
+  # Protect Jellyfin login from brute force attacks
+  services.fail2ban.jails.jellyfin = {
+    enabled = true;
+    settings = {
+      backend = "auto";
+      port = "http,https";
+      logpath = "${config.services.jellyfin.dataDir}/log/log_*.log";
+      # defaults: maxretry=5, findtime=10m, bantime=10m
+    };
+    filter.Definition = {
+      failregex = ''^.*Authentication request for .* has been denied \(IP: "<ADDR>"\)\..*$'';
+      ignoreregex = "";
+    };
+  };
 }

services/livekit.nix

@@ -0,0 +1,53 @@
+{
+  service_configs,
+  ...
+}:
+let
+  keyFile = ../secrets/livekit_keys;
+
+  ports = service_configs.ports;
+in
+{
+  services.livekit = {
+    enable = true;
+    inherit keyFile;
+    openFirewall = true;
+
+    settings = {
+      port = ports.livekit;
+      bind_addresses = [ "127.0.0.1" ];
+
+      rtc = {
+        port_range_start = 50100;
+        port_range_end = 50200;
+        use_external_ip = true;
+      };
+
+      # Disable LiveKit's built-in TURN; coturn is already running
+      turn = {
+        enabled = false;
+      };
+
+      logging = {
+        level = "info";
+      };
+    };
+  };
+
+  services.lk-jwt-service = {
+    enable = true;
+    inherit keyFile;
+    livekitUrl = "wss://${service_configs.livekit.domain}";
+    port = ports.lk_jwt;
+  };
+
+  services.caddy.virtualHosts."${service_configs.livekit.domain}".extraConfig = ''
+    @jwt path /sfu/get /healthz
+    handle @jwt {
+      reverse_proxy :${builtins.toString ports.lk_jwt}
+    }
+    handle {
+      reverse_proxy :${builtins.toString ports.livekit}
+    }
+  '';
+}

services/matrix.nix

@@ -1,39 +1,53 @@
 {
-  pkgs,
   config,
+  pkgs,
   service_configs,
   lib,
   ...
 }:
+let
+  package =
+    let
+      src = pkgs.fetchFromGitea {
+        domain = "forgejo.ellis.link";
+        owner = "continuwuation";
+        repo = "continuwuity";
+        rev = "052c4dfa2165fdc4839fed95b71446120273cf23";
+        hash = "sha256-kQV4glRrKczoJpn9QIMgB5ac+saZQjSZPel+9K9Ykcs=";
+      };
+    in
+    pkgs.matrix-continuwuity.overrideAttrs (old: {
+      inherit src;
+      cargoDeps = pkgs.rustPlatform.fetchCargoVendor {
+        inherit src;
+        name = "${old.pname}-vendor";
+        hash = "sha256-vlOXQL8wwEGFX+w0G/eIeHW3J1UDzhJ501kYhAghDV8=";
+      };
+
+      patches = (old.patches or [ ]) ++ [
+
+      ];
+    });
+in
 {
-  services.matrix-conduit.settings.global.registration_token =
-    builtins.readFile ../secrets/matrix_reg_token;
+  imports = [
+    (lib.serviceMountWithZpool "continuwuity" service_configs.zpool_ssds [
+      "/var/lib/private/continuwuity"
+    ])
+    (lib.serviceFilePerms "continuwuity" [
+      "Z /var/lib/private/continuwuity 0770 ${config.services.matrix-continuwuity.user} ${config.services.matrix-continuwuity.group}"
+    ])
+  ];
 
-  services.caddy.virtualHosts.${service_configs.https.domain}.extraConfig = lib.mkBefore ''
-    header /.well-known/matrix/* Content-Type application/json
-    header /.well-known/matrix/* Access-Control-Allow-Origin *
-    respond /.well-known/matrix/server `{"m.server": "${service_configs.https.matrix_hostname}:443"}`
-    respond /.well-known/matrix/client `{"m.server":{"base_url":"https://${service_configs.https.matrix_hostname}"},"m.homeserver":{"base_url":"https://${service_configs.https.matrix_hostname}"},"org.matrix.msc3575.proxy":{"base_url":"https://${config.services.matrix-conduit.settings.global.server_name}"}}`
-  '';
-
-  services.caddy.virtualHosts."${service_configs.https.matrix_hostname}".extraConfig = ''
-    reverse_proxy :${builtins.toString config.services.matrix-conduit.settings.global.port}
-  '';
-
-  # Exact duplicate
-  services.caddy.virtualHosts."${service_configs.https.matrix_hostname}:8448".extraConfig =
-    config.services.caddy.virtualHosts."${config.services.matrix-conduit.settings.global.server_name
-    }".extraConfig;
-
-  services.matrix-conduit = {
+  services.matrix-continuwuity = {
     enable = true;
-    package = pkgs.conduwuit;
+    inherit package;
 
     settings.global = {
-      port = 6167;
+      port = [ service_configs.ports.matrix ];
       server_name = service_configs.https.domain;
-      database_backend = "rocksdb";
       allow_registration = true;
+      registration_token = lib.strings.trim (builtins.readFile ../secrets/matrix_reg_token);
 
       new_user_displayname_suffix = "";
 
@@ -44,12 +58,42 @@
         "envs.net"
       ];
 
-      # without this, conduit fails to start
-      address = "0.0.0.0";
+      address = [
+        "0.0.0.0"
+      ];
+
+      # TURN server config (coturn)
+      turn_secret = config.services.coturn.static-auth-secret;
+      turn_uris = [
+        "turn:${service_configs.https.domain}?transport=udp"
+        "turn:${service_configs.https.domain}?transport=tcp"
+      ];
+      turn_ttl = 86400;
     };
   };
 
-  systemd.tmpfiles.rules = [
-    "d /var/lib/private/matrix-conduit 0770 conduit conduit"
+  services.caddy.virtualHosts.${service_configs.https.domain}.extraConfig = lib.mkBefore ''
+    header /.well-known/matrix/* Content-Type application/json
+    header /.well-known/matrix/* Access-Control-Allow-Origin *
+    respond /.well-known/matrix/server `{"m.server": "${service_configs.matrix.domain}:${builtins.toString service_configs.ports.https}"}`
+    respond /.well-known/matrix/client `{"m.server":{"base_url":"https://${service_configs.matrix.domain}"},"m.homeserver":{"base_url":"https://${service_configs.matrix.domain}"},"org.matrix.msc3575.proxy":{"base_url":"https://${config.services.matrix-continuwuity.settings.global.server_name}"},"org.matrix.msc4143.rtc_foci":[{"type":"livekit","livekit_service_url":"https://${service_configs.livekit.domain}"}]}`
+  '';
+
+  services.caddy.virtualHosts."${service_configs.matrix.domain}".extraConfig = ''
+    reverse_proxy :${builtins.toString service_configs.ports.matrix}
+  '';
+
+  # Exact duplicate for federation port
+  services.caddy.virtualHosts."${service_configs.matrix.domain}:${builtins.toString service_configs.ports.matrix_federation}".extraConfig =
+    config.services.caddy.virtualHosts."${service_configs.matrix.domain}".extraConfig;
+
+  # for federation
+  networking.firewall.allowedTCPPorts = [
+    service_configs.ports.matrix_federation
+  ];
+
+  # for federation
+  networking.firewall.allowedUDPPorts = [
+    service_configs.ports.matrix_federation
   ];
 }

services/minecraft.nix

@@ -2,26 +2,25 @@
   pkgs,
   service_configs,
   lib,
-  username,
+  config,
+  inputs,
   ...
 }:
-let
-  heap_size = "4000M";
-in
 {
-  environment.systemPackages = [
-    (pkgs.writeScriptBin "mc-console" ''
-      #!/bin/sh
-      ${pkgs.tmux}/bin/tmux -S /run/minecraft/${service_configs.minecraft.server_name}.sock attach
-    '')
+  imports = [
+    (lib.serviceMountWithZpool "minecraft-server-${service_configs.minecraft.server_name}"
+      service_configs.zpool_ssds
+      [
+        "${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}"
+      ]
+    )
+    inputs.nix-minecraft.nixosModules.minecraft-servers
+    (lib.serviceFilePerms "minecraft-server-${service_configs.minecraft.server_name}" [
+      "Z ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name} 700 ${config.services.minecraft-servers.user} ${config.services.minecraft-servers.group}"
+      "Z ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web 750 ${config.services.minecraft-servers.user} ${config.services.minecraft-servers.group}"
+    ])
   ];
 
-  nixpkgs.config.allowUnfreePredicate =
-    pkg:
-    builtins.elem (lib.getName pkg) [
-      "minecraft-server"
-    ];
-
   services.minecraft-servers = {
     enable = true;
     eula = true;
@@ -30,18 +29,56 @@ in
 
     servers.${service_configs.minecraft.server_name} = {
       enable = true;
-      package = pkgs.fabricServers.fabric-1_21_4;
+      package = pkgs.fabricServers.fabric-1_21_11;
 
-      jvmOpts = "-Xmx${heap_size} -Xms${heap_size} -XX:+UseZGC -XX:+ZGenerational";
+      jvmOpts =
+        let
+          heap_size = "4000M";
+        in
+        lib.concatStringsSep " " [
+          # Memory
+          "-Xmx${heap_size}"
+          "-Xms${heap_size}"
+          # GC
+          "-XX:+UseZGC"
+          "-XX:+ZGenerational"
+          # Base JVM optimizations (brucethemoose/Minecraft-Performance-Flags-Benchmarks)
+          "-XX:+UnlockExperimentalVMOptions"
+          "-XX:+UnlockDiagnosticVMOptions"
+          "-XX:+AlwaysActAsServerClassMachine"
+          "-XX:+AlwaysPreTouch"
+          "-XX:+DisableExplicitGC"
+          "-XX:+UseNUMA"
+          "-XX:+PerfDisableSharedMem"
+          "-XX:+UseFastUnorderedTimeStamps"
+          "-XX:+UseCriticalJavaThreadPriority"
+          "-XX:ThreadPriorityPolicy=1"
+          "-XX:AllocatePrefetchStyle=3"
+          "-XX:-DontCompileHugeMethods"
+          "-XX:MaxNodeLimit=240000"
+          "-XX:NodeLimitFudgeFactor=8000"
+          "-XX:ReservedCodeCacheSize=400M"
+          "-XX:NonNMethodCodeHeapSize=12M"
+          "-XX:ProfiledCodeHeapSize=194M"
+          "-XX:NonProfiledCodeHeapSize=194M"
+          "-XX:NmethodSweepActivity=1"
+          "-XX:+UseVectorCmov"
+          # Large pages (requires vm.nr_hugepages sysctl)
+          "-XX:+UseLargePages"
+          "-XX:LargePageSizeInBytes=2m"
+        ];
 
       serverProperties = {
-        server-port = 25565;
+        server-port = service_configs.ports.minecraft;
         enforce-whitelist = true;
         gamemode = "survival";
         white-list = true;
         difficulty = "easy";
         motd = "A Minecraft Server";
-        view-distance = 12;
+        view-distance = 10;
+        simulation-distance = 6;
+        sync-chunk-writes = false;
+        spawn-protection = 0;
       };
 
       whitelist = import ../secrets/minecraft-whitelist.nix;
@@ -51,67 +88,102 @@ in
           with pkgs;
           builtins.attrValues {
             FabricApi = fetchurl {
-              url = "https://cdn.modrinth.com/data/P7dR8mSH/versions/HbTXYTBz/fabric-api-0.119.0%2B1.21.4.jar";
-              sha512 = "f2e44507dcf7c34ac5104bf78c0f0f0ab99840272d0c1afc51236b7f8a56541bd5c2024953a83599034e1b55191e38b3e437b6b80736137e2ee4d7d571f42c82";
+              url = "https://cdn.modrinth.com/data/P7dR8mSH/versions/i5tSkVBH/fabric-api-0.141.3%2B1.21.11.jar";
+              sha512 = "c20c017e23d6d2774690d0dd774cec84c16bfac5461da2d9345a1cd95eee495b1954333c421e3d1c66186284d24a433f6b0cced8021f62e0bfa617d2384d0471";
             };
 
             FerriteCore = fetchurl {
-              url = "https://cdn.modrinth.com/data/uXXizFIs/versions/IPM0JlHd/ferritecore-7.1.1-fabric.jar";
-              sha512 = "f41dc9e8b28327a1e29b14667cb42ae5e7e17bcfa4495260f6f851a80d4b08d98a30d5c52b110007ee325f02dac7431e3fad4560c6840af0bf347afad48c5aac";
+              url = "https://cdn.modrinth.com/data/uXXizFIs/versions/Ii0gP3D8/ferritecore-8.2.0-fabric.jar";
+              sha512 = "3210926a82eb32efd9bcebabe2f6c053daf5c4337eebc6d5bacba96d283510afbde646e7e195751de795ec70a2ea44fef77cb54bf22c8e57bb832d6217418869";
             };
 
             Lithium = fetchurl {
-              url = "https://cdn.modrinth.com/data/gvQqBUqZ/versions/kLc5Oxr4/lithium-fabric-0.14.8%2Bmc1.21.4.jar";
-              sha512 = "ea0d7a4aea29b32527245d933227c85d0606e17c88cc05ed9918a1b966f22011961bfa85e33ab318e729f1ac3e69217d37709413bf70d1dc5a3acc9fd75ef317";
+              url = "https://cdn.modrinth.com/data/gvQqBUqZ/versions/qvNsoO3l/lithium-fabric-0.21.3%2Bmc1.21.11.jar";
+              sha512 = "2883739303f0bb602d3797cc601ed86ce6833e5ec313ddce675f3d6af3ee6a40b9b0a06dafe39d308d919669325e95c0aafd08d78c97acd976efde899c7810fd";
             };
 
             NoChatReports = fetchurl {
-              url = "https://cdn.modrinth.com/data/qQyHxfxd/versions/9xt05630/NoChatReports-FABRIC-1.21.4-v2.11.0.jar";
-              sha512 = "d343b05c8e50f1de15791ff622ad44eeca6cdcb21e960a267a17d71506c61ca79b1c824167779e44d778ca18dcbdebe594ff234fbe355b68d25cdb5b6afd6e4f";
-            };
-
-            moonrise = fetchurl {
-              url = "https://cdn.modrinth.com/data/KOHu7RCS/versions/6Dgh9jQx/Moonrise-Fabric-0.2.0-beta.9%2Bac0c7de.jar";
-              sha512 = "c101f1a41db4095d651d32eae47bd7e6f7358f7390898610d1bf261ebfc7e0f4165fd551c08a99cca31a3308f1989a16b8c75c1ece60ef9cd475107ca4f4219e";
+              url = "https://cdn.modrinth.com/data/qQyHxfxd/versions/rhykGstm/NoChatReports-FABRIC-1.21.11-v2.18.0.jar";
+              sha512 = "d2c35cc8d624616f441665aff67c0e366e4101dba243bad25ed3518170942c1a3c1a477b28805cd1a36c44513693b1c55e76bea627d3fced13927a3d67022ccc";
             };
 
             squaremap = fetchurl {
-              url = "https://cdn.modrinth.com/data/PFb7ZqK6/versions/9i2KwI5R/squaremap-fabric-mc1.21.4-1.3.4.jar";
-              sha512 = "6eb44061f057d1bbd0bb6f9186d03d496479dcd953af8f09f70099c2e67e567e5dca626972d45af0315c2e2714c3dd74beef97575396e3bb90b7c670f5c80fef";
+              url = "https://cdn.modrinth.com/data/PFb7ZqK6/versions/BW8lMXBi/squaremap-fabric-mc1.21.11-1.3.12.jar";
+              sha512 = "f62eb791a3f5812eb174565d318f2e6925353f846ef8ac56b4e595f481494e0c281f26b9e9fcfdefa855093c96b735b12f67ee17c07c2477aa7a3439238670d9";
             };
 
-            modernfix = fetchurl {
-              url = "https://cdn.modrinth.com/data/nmDcB62a/versions/ZGxQddYr/modernfix-fabric-5.20.3%2Bmc1.21.4.jar";
-              sha512 = "ae49114c92a048c9ce79e197fc4df028e186cf13546e710f72247382fa8076f0b70d6aa3224951f4a36c886ca236f099a011f20b021a2b0d1a75c631da4d7d52";
+            scalablelux = fetchurl {
+              url = "https://cdn.modrinth.com/data/Ps1zyz6x/versions/PV9KcrYQ/ScalableLux-0.1.6%2Bfabric.c25518a-all.jar";
+              sha512 = "729515c1e75cf8d9cd704f12b3487ddb9664cf9928e7b85b12289c8fbbc7ed82d0211e1851375cbd5b385820b4fedbc3f617038fff5e30b302047b0937042ae7";
             };
 
-            alternatecurrent = fetchurl {
-              url = "https://cdn.modrinth.com/data/r0v8vy1s/versions/DwfiGUVU/alternate-current-mc1.21.2-1.9.1.jar";
-              sha512 = "8ed44291a8aed3e1c9750cfce85e0de679daeff7c3b1bc8f6329b41ba4570442750b8039d2d5c79c32655fc9372ea35843c60805438d33888b30e28731c39137";
+            c2me = fetchurl {
+              url = "https://cdn.modrinth.com/data/VSNURh3q/versions/QdLiMUjx/c2me-fabric-mc1.21.11-0.3.7%2Balpha.0.7.jar";
+              sha512 = "f9543febe2d649a82acd6d5b66189b6a3d820cf24aa503ba493fdb3bbd4e52e30912c4c763fe50006f9a46947ae8cd737d420838c61b93429542573ed67f958e";
             };
 
-            # fix `Error sending packet clientbound/minecraft:disconnect` error
-            disconnectpacketfix = fetchurl {
+            krypton = fetchurl {
+              url = "https://cdn.modrinth.com/data/fQEb0iXm/versions/O9LmWYR7/krypton-0.2.10.jar";
+              sha512 = "4dcd7228d1890ddfc78c99ff284b45f9cf40aae77ef6359308e26d06fa0d938365255696af4cc12d524c46c4886cdcd19268c165a2bf0a2835202fe857da5cab";
+            };
+
+            better-fabric-console = fetchurl {
+              url = "https://cdn.modrinth.com/data/Y8o1j1Sf/versions/6aIKl5wy/better-fabric-console-mc1.21.11-1.2.9.jar";
+              sha512 = "427247dafd99df202ee10b4bf60ffcbbecbabfadb01c167097ffb5b85670edb811f4d061c2551be816295cbbc6b8ec5ec464c14a6ff41912ef1f6c57b038d320";
+            };
+
+            disconnect-packet-fix = fetchurl {
               url = "https://cdn.modrinth.com/data/rd9rKuJT/versions/Gv74xveQ/disconnect-packet-fix-fabric-2.0.0.jar";
               sha512 = "1fd6f09a41ce36284e1a8e9def53f3f6834d7201e69e54e24933be56445ba569fbc26278f28300d36926ba92db6f4f9c0ae245d23576aaa790530345587316db";
             };
+
+            packet-fixer = fetchurl {
+              url = "https://cdn.modrinth.com/data/c7m1mi73/versions/CUh1DWeO/packetfixer-fabric-3.3.4-1.21.11.jar";
+              sha512 = "33331b16cb40c5e6fbaade3cacc26f3a0e8fa5805a7186f94d7366a0e14dbeee9de2d2e8c76fa71f5e9dd24eb1c261667c35447e32570ea965ca0f154fdfba0a";
+            };
+
+            # fork of Modernfix for 1.21.11 (upstream will support 26.1)
+            modernfix = fetchurl {
+              url = "https://cdn.modrinth.com/data/TjSm1wrD/versions/JwSO8JCN/modernfix-5.25.2-build.4.jar";
+              sha512 = "0d65c05ac0475408c58ef54215714e6301113101bf98bfe4bb2ba949fbfddd98225ac4e2093a5f9206a9e01ba80a931424b237bdfa3b6e178c741ca6f7f8c6a3";
+            };
+
+            debugify = fetchurl {
+              url = "https://cdn.modrinth.com/data/QwxR6Gcd/versions/8Q49lnaU/debugify-1.21.11%2B1.0.jar";
+              sha512 = "04d82dd33f44ced37045f1f9a54ad4eacd70861ff74a8800f2d2df358579e6cb0ea86a34b0086b3e87026b1a0691dd6594b4fdc49f89106466eea840518beb03";
+            };
+
           }
         );
       };
     };
   };
 
-  services.caddy.virtualHosts."map.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
-    root * ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web
-    file_server browse
-  '';
+  systemd.services.minecraft-server-main = {
+    serviceConfig = {
+      Nice = -5;
+      IOSchedulingPriority = 0;
+      LimitMEMLOCK = "infinity"; # Required for large pages
+    };
+  };
+
+  services.caddy.virtualHosts = lib.mkIf (config.services.caddy.enable) {
+    "map.${service_configs.https.domain}".extraConfig = ''
+      root * ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web
+      file_server browse
+    '';
+  };
+
+  users.users = lib.mkIf (config.services.caddy.enable) {
+    ${config.services.caddy.user}.extraGroups = [
+      # for `map.gardling.com`
+      config.services.minecraft-servers.group
+    ];
+  };
 
   systemd.tmpfiles.rules = [
-    "d ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name} 0770 minecraft minecraft"
-  ];
-
-  users.users.${username}.extraGroups = [
-    "minecraft"
+    # Allow caddy (in minecraft group) to traverse to squaremap/web for map.gardling.com
+    "z ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name} 710 ${config.services.minecraft-servers.user} ${config.services.minecraft-servers.group}"
+    "z ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap 710 ${config.services.minecraft-servers.user} ${config.services.minecraft-servers.group}"
   ];
 }

services/monero.nix

@@ -0,0 +1,23 @@
+{
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "monero" service_configs.zpool_hdds [
+      service_configs.monero.dataDir
+    ])
+    (lib.serviceFilePerms "monero" [
+      "Z ${service_configs.monero.dataDir} 0700 monero monero"
+    ])
+  ];
+
+  services.monero = {
+    enable = true;
+    dataDir = service_configs.monero.dataDir;
+    rpc = {
+      restricted = true;
+    };
+  };
+}

services/ntfy-alerts.nix

@@ -0,0 +1,10 @@
+{ config, service_configs, ... }:
+{
+  services.ntfyAlerts = {
+    enable = true;
+    serverUrl = "https://${service_configs.ntfy.domain}";
+    topicFile = config.age.secrets.ntfy-alerts-topic.path;
+
+    tokenFile = config.age.secrets.ntfy-alerts-token.path;
+  };
+}

services/ntfy.nix

@@ -0,0 +1,34 @@
+{
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "ntfy-sh" service_configs.zpool_ssds [
+      "/var/lib/private/ntfy-sh"
+    ])
+    (lib.serviceFilePerms "ntfy-sh" [
+      "Z /var/lib/private/ntfy-sh 0700 ${config.services.ntfy-sh.user} ${config.services.ntfy-sh.group}"
+    ])
+  ];
+
+  services.ntfy-sh = {
+    enable = true;
+
+    settings = {
+      base-url = "https://${service_configs.ntfy.domain}";
+      listen-http = "127.0.0.1:${builtins.toString service_configs.ports.ntfy}";
+      behind-proxy = true;
+      auth-default-access = "deny-all";
+      enable-login = true;
+      enable-signup = false;
+    };
+  };
+
+  services.caddy.virtualHosts."${service_configs.ntfy.domain}".extraConfig = ''
+    reverse_proxy :${builtins.toString service_configs.ports.ntfy}
+  '';
+
+}

services/owntracks.nix

@@ -1,48 +0,0 @@
-{
-  pkgs,
-  service_configs,
-  username,
-  ...
-}:
-let
-  owntracks_pkg = pkgs.owntracks-recorder.overrideAttrs (old: {
-    installPhase =
-      old.installPhase
-      + ''
-        mkdir -p $out/usr/share/ot-recorder
-        cp -R docroot/* $out/usr/share/ot-recorder'';
-  });
-in
-{
-  users.groups.owntracks = { };
-  users.users.owntracks = {
-    isNormalUser = true;
-    group = "owntracks";
-  };
-
-  systemd.services.owntracks = {
-    enable = true;
-    description = "Store and access data published by OwnTracks apps";
-    wantedBy = [ "multi-user.target" ];
-
-    serviceConfig = {
-      User = "owntracks";
-      Group = "owntracks";
-      WorkingDirectory = "${owntracks_pkg}";
-      ExecStart = "${owntracks_pkg}/bin/ot-recorder -S ${service_configs.owntracks.data_dir} --doc-root usr/share/ot-recorder --http-port ${builtins.toString service_configs.ports.owntracks} --port 0";
-    };
-  };
-
-  systemd.tmpfiles.rules = [
-    "d ${service_configs.owntracks.data_dir} 0770 owntracks owntracks"
-  ];
-
-  services.caddy.virtualHosts."owntracks.${service_configs.https.domain}".extraConfig = ''
-    ${builtins.readFile ../secrets/owntracks_caddy_auth}
-    reverse_proxy :${builtins.toString service_configs.ports.owntracks}
-  '';
-
-  users.users.${username}.extraGroups = [
-    "owntracks"
-  ];
-}

services/postgresql.nix

@@ -1,22 +1,24 @@
 {
   pkgs,
   config,
-  username,
+  service_configs,
+  lib,
   ...
 }:
 {
+  imports = [
+    (lib.serviceMountWithZpool "postgresql" service_configs.zpool_ssds [
+      config.services.postgresql.dataDir
+    ])
+    (lib.serviceFilePerms "postgresql" [
+      "Z ${config.services.postgresql.dataDir} 0700 postgres postgres"
+    ])
+  ];
+
   services.postgresql = {
     enable = true;
     package = pkgs.postgresql_16;
-    dataDir = "/tank/services/sql";
+    dataDir = service_configs.postgres.dataDir;
   };
 
-  systemd.tmpfiles.rules = [
-    # postgresql requires 0700
-    "d ${config.services.postgresql.dataDir} 0700 postgresql postgresql"
-  ];
-
-  users.users.${username}.extraGroups = [
-    "postgresql"
-  ];
 }

services/qbittorrent.nix

@@ -2,44 +2,46 @@
   pkgs,
   config,
   service_configs,
-  username,
   lib,
+  inputs,
   ...
 }:
 {
-  # network namespace that is proxied through mullvad
-  vpnNamespaces.wg = {
-    portMappings = [
-      {
-        from = config.services.qbittorrent.webuiPort;
-        to = config.services.qbittorrent.webuiPort;
-      }
-    ];
+  imports = [
+    (lib.serviceMountWithZpool "qbittorrent" service_configs.zpool_hdds [
+      service_configs.torrents_path
+      config.services.qbittorrent.serverConfig.Preferences.Downloads.TempPath
 
-    openVPNPorts = [
-      {
-        port = config.services.qbittorrent.webuiPort;
-        protocol = "both";
-      }
-    ];
-  };
+    ])
+    (lib.serviceMountWithZpool "qbittorrent" service_configs.zpool_ssds [
+      "${config.services.qbittorrent.profileDir}/qBittorrent"
+    ])
+    (lib.vpnNamespaceOpenPort config.services.qbittorrent.webuiPort "qbittorrent")
+    (lib.serviceFilePerms "qbittorrent" [
+      # 0770: group (media) needs write to delete files during upgrades —
+      # Radarr/Sonarr must unlink the old file before placing the new one.
+      "Z ${config.services.qbittorrent.serverConfig.Preferences.Downloads.SavePath} 0770 ${config.services.qbittorrent.user} ${service_configs.media_group}"
+      "Z ${config.services.qbittorrent.serverConfig.Preferences.Downloads.TempPath} 0700 ${config.services.qbittorrent.user} ${config.services.qbittorrent.group}"
+      "Z ${config.services.qbittorrent.profileDir} 0700 ${config.services.qbittorrent.user} ${config.services.qbittorrent.group}"
+    ])
+  ];
 
   services.qbittorrent = {
     enable = true;
-    package = pkgs.qbittorrent-nox;
     webuiPort = service_configs.ports.torrent;
+    profileDir = "/var/lib/qBittorrent";
+    # Set the service group to 'media' so the systemd unit runs with media as
+    # the primary GID. Linux assigns new file ownership from the process's GID
+    # (set by systemd's Group= directive), not from /etc/passwd. Without this,
+    # downloads land as qbittorrent:qbittorrent (0700), blocking Radarr/Sonarr.
+    group = service_configs.media_group;
 
     serverConfig.LegalNotice.Accepted = true;
 
     serverConfig.Preferences = {
       WebUI = {
         AlternativeUIEnabled = true;
-        RootFolder = builtins.toString (
-          pkgs.fetchzip {
-            url = "https://github.com/VueTorrent/VueTorrent/releases/download/v2.23.1/vuetorrent.zip";
-            sha256 = "yZmnRmYoinJ8uSuUpjGIRCQWBrK59hwyEkCq8aWiOvQ=";
-          }
-        );
+        RootFolder = "${pkgs.vuetorrent}/share/vuetorrent";
 
         # disable auth because we use caddy for auth
         AuthSubnetWhitelist = "0.0.0.0/0";
@@ -47,83 +49,77 @@
       };
 
       Downloads = {
-        SavePath = service_configs.torrent.SavePath;
-        TempPath = service_configs.torrent.TempPath;
+        inherit (service_configs.torrent) SavePath TempPath;
       };
     };
 
     serverConfig.BitTorrent = {
       Session = {
-        GlobalUPSpeedLimit = 1500; # 500 KiB/s
-        GlobalDLSpeedLimit = 5000; # 5 MiB/s
+        MaxConnectionsPerTorrent = 50;
+        MaxUploadsPerTorrent = 10;
+        MaxConnections = -1;
+        MaxUploads = -1;
 
-        IgnoreLimitsOnLAN = true;
+        MaxActiveCheckingTorrents = 5;
 
+        # queueing
+        QueueingSystemEnabled = true;
+        MaxActiveDownloads = 5; # keep focused: fewer torrents, each gets more bandwidth
+        MaxActiveUploads = -1;
+        MaxActiveTorrents = -1;
+        IgnoreSlowTorrentsForQueueing = true;
+
+        GlobalUPSpeedLimit = 0;
+        GlobalDLSpeedLimit = 0;
+
+        # Alternate speed limits for when Jellyfin is streaming
+        AlternativeGlobalUPSpeedLimit = 500; # 500 KB/s when throttled
+        AlternativeGlobalDLSpeedLimit = 800; # 800 KB/s when throttled
         IncludeOverheadInLimits = true;
 
-        GlobalMaxRatio = -1;
-        QueueingSystemEnabled = false; # seed all torrents all the time
+        GlobalMaxRatio = 7.0;
 
         AddTrackersEnabled = true;
-        AdditionalTrackers = (
-          lib.concatStringsSep "\\n" [
-            "udp://tracker.opentrackr.org:1337/announce"
-            "udp://open.stealth.si:80/announce"
-            "udp://open.demonii.com:1337"
-            "udp://exodus.desync.com:6969/announce"
-            "udp://tracker.dler.org:6969/announce"
-            "udp://tracker.bittor.pw:1337/announce"
-            "udp://tracker.torrent.eu.org:451/announce"
-            "udp://explodie.org:6969/announce"
-            "http://tracker.files.fm:6969/announce"
-            "udp://tracker.tiny-vps.com:6969/announce"
-            "udp://p4p.arenabg.com:1337/announce"
-            "udp://tracker.dler.com:6969/announce"
-            "udp://inferno.demonoid.is:3391/announce"
-
-            "udp://tracker.torrent.eu.org:451/announce"
-            "udp://tracker.ololosh.space:6969/announce"
-            "udp://ns-1.x-fins.com:6969/announce"
-            "udp://leet-tracker.moe:1337/announce"
-            "http://tracker.vanitycore.co:6969/announce"
-            "http://tracker.sbsub.com:2710/announce"
-            "http://tracker.moxing.party:6969/announce"
-            "http://tracker.ipv6tracker.org:80/announce"
-            "http://tracker.corpscorp.online:80/announce"
-            "http://shubt.net:2710/announce"
-            "http://share.hkg-fansub.info:80/announce.php"
-            "http://servandroidkino.ru:80/announce"
-            "http://bt.poletracker.org:2710/announce"
-            "http://0d.kebhana.mx:443/announce"
-          ]
+        AdditionalTrackers = lib.concatStringsSep "\\n" (
+          lib.lists.filter (x: x != "") (
+            lib.strings.splitString "\n" (builtins.readFile "${inputs.trackerlist}/trackers_all.txt")
+          )
         );
+        AnnounceToAllTrackers = true;
+
+        # idk why it also has to be specified here too?
+        inherit (config.services.qbittorrent.serverConfig.Preferences.Downloads) TempPath;
+        TempPathEnabled = true;
+
+        ConnectionSpeed = 200;
+
+        # Automatic Torrent Management: use category save paths for new torrents
+        DisableAutoTMMByDefault = false;
+        DisableAutoTMMTriggers.CategorySavePathChanged = false;
+        DisableAutoTMMTriggers.DefaultSavePathChanged = false;
+
+        ChokingAlgorithm = "RateBased";
+        PieceExtentAffinity = true;
+        SuggestMode = true;
+        CoalesceReadWrite = true;
+      };
+
+      Network = {
+        # traffic is routed through a vpn, we don't need
+        # port forwarding
+        PortForwardingEnabled = false;
       };
     };
   };
 
-  systemd.tmpfiles.rules = [
-    "d ${config.services.qbittorrent.serverConfig.Preferences.Downloads.SavePath} 0770 ${config.services.qbittorrent.user} ${service_configs.torrent_group}"
-    "d ${config.services.qbittorrent.serverConfig.Preferences.Downloads.TempPath} 0770 ${config.services.qbittorrent.user} ${service_configs.torrent_group}"
-  ];
-
-  # make qbittorrent use a vpn
-  systemd.services.qbittorrent.vpnConfinement = {
-    enable = true;
-    vpnNamespace = "wg";
-  };
+  systemd.services.qbittorrent.serviceConfig.TimeoutStopSec = lib.mkForce 10;
 
   services.caddy.virtualHosts."torrent.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
-    ${builtins.readFile ../secrets/caddy_auth}
-    reverse_proxy ${service_configs.https.wg_ip}:${builtins.toString config.services.qbittorrent.webuiPort}
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy ${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString config.services.qbittorrent.webuiPort}
   '';
 
   users.users.${config.services.qbittorrent.user}.extraGroups = [
-    service_configs.torrent_group
+    service_configs.media_group
   ];
-
-  users.users.${username}.extraGroups = [
-    config.services.qbittorrent.group
-  ];
-
 }

services/soulseek.nix

@@ -10,13 +10,27 @@ let
   slskd_env = "/etc/slskd_env";
 in
 {
+  imports = [
+    (lib.serviceMountWithZpool "slskd" "" [
+      service_configs.slskd.base
+      service_configs.slskd.downloads
+      service_configs.slskd.incomplete
+    ])
+    (lib.serviceFilePerms "slskd" [
+      "Z ${service_configs.music_dir} 0750 ${username} music"
+      "Z ${service_configs.slskd.base} 0750 ${config.services.slskd.user} ${config.services.slskd.group}"
+      "Z ${service_configs.slskd.downloads} 0750 ${config.services.slskd.user} music"
+      "Z ${service_configs.slskd.incomplete} 0750 ${config.services.slskd.user} music"
+    ])
+  ];
+
   users.groups."music" = { };
 
   system.activationScripts = {
-    "zfs-key".text = ''
+    "skskd_env".text = ''
       #!/bin/sh
       rm -fr ${slskd_env} || true
-      cp ${../secrets/slskd_env} ${slskd_env}
+      cp ${config.age.secrets.slskd_env.path} ${slskd_env}
       chmod 0500 ${slskd_env}
       chown ${config.services.slskd.user}:${config.services.slskd.group} ${slskd_env}
     '';
@@ -42,12 +56,12 @@ in
 
       global = {
         download = {
-          slots = 3;
-          speed_limit = 500;
+          slots = -1;
+          speed_limit = -1;
         };
         upload = {
           slots = 4;
-          speed_limit = 500;
+          speed_limit = 2000;
         };
       };
     };
@@ -55,11 +69,7 @@ in
 
   users.users.${config.services.slskd.user}.extraGroups = [ "music" ];
   users.users.${config.services.jellyfin.user}.extraGroups = [ "music" ];
-
-  systemd.tmpfiles.rules = [
-    "d ${service_configs.music_dir} 0750 ${username} music"
-    "d ${service_configs.music_dir} 0750 ${username} music"
-  ];
+  users.users.${username}.extraGroups = [ "music" ];
 
   # doesn't work with auth????
   services.caddy.virtualHosts."soulseek.${service_configs.https.domain}".extraConfig = ''

services/ssh.nix

@@ -0,0 +1,35 @@
+{
+  config,
+  lib,
+  pkgs,
+  username,
+  ...
+}:
+{
+  # Enable the OpenSSH daemon.
+  services.openssh = {
+    enable = true;
+    settings = {
+      AllowUsers = [
+        username
+        "root"
+      ];
+      PasswordAuthentication = false;
+      PermitRootLogin = "yes"; # for deploying configs
+    };
+  };
+
+  systemd.tmpfiles.rules = [
+    "Z /etc/ssh 755 root root"
+    "Z /etc/ssh/ssh_host_* 600 root root"
+  ];
+
+  users.users.${username}.openssh.authorizedKeys.keys = [
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO4jL6gYOunUlUtPvGdML0cpbKSsPNqQ1jit4E7U1RyH" # laptop
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBJjT5QZ3zRDb+V6Em20EYpSEgPW5e/U+06uQGJdraxi" # desktop
+  ];
+
+  # used for deploying configs to server
+  users.users.root.openssh.authorizedKeys.keys =
+    config.users.users.${username}.openssh.authorizedKeys.keys;
+}

services/syncthing.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  lib,
+  pkgs,
+  service_configs,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "syncthing" service_configs.zpool_ssds [
+      service_configs.syncthing.dataDir
+      service_configs.syncthing.signalBackupDir
+      service_configs.syncthing.grayjayBackupDir
+    ])
+    (lib.serviceFilePerms "syncthing" [
+      "Z ${service_configs.syncthing.dataDir} 0750 ${config.services.syncthing.user} ${config.services.syncthing.group}"
+      "Z ${service_configs.syncthing.signalBackupDir} 0750 ${config.services.syncthing.user} ${config.services.syncthing.group}"
+      "Z ${service_configs.syncthing.grayjayBackupDir} 0750 ${config.services.syncthing.user} ${config.services.syncthing.group}"
+    ])
+  ];
+
+  services.syncthing = {
+    enable = true;
+
+    dataDir = service_configs.syncthing.dataDir;
+
+    guiAddress = "127.0.0.1:${toString service_configs.ports.syncthing_gui}";
+
+    overrideDevices = false;
+    overrideFolders = false;
+
+    settings = {
+      gui = {
+        insecureSkipHostcheck = true; # Allow access via reverse proxy
+      };
+      options = {
+        urAccepted = 1; # enable usage reporting
+        relaysEnabled = true;
+      };
+    };
+  };
+
+  # Open firewall ports for syncthing protocol
+  networking.firewall = {
+    allowedTCPPorts = [ service_configs.ports.syncthing_protocol ];
+    allowedUDPPorts = [ service_configs.ports.syncthing_discovery ];
+  };
+
+  services.caddy.virtualHosts."syncthing.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${toString service_configs.ports.syncthing_gui}
+  '';
+
+}

services/ups.nix

@@ -0,0 +1,22 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  services.apcupsd = {
+    enable = true;
+    configText = ''
+      UPSTYPE usb
+      NISIP 127.0.0.1
+      BATTERYLEVEL 5 # shutdown after reaching 5% battery
+      MINUTES 5 # shutdown if estimated runtime on battery reaches 5 minutes
+    '';
+
+    hooks = {
+      # command to run when shutdown condition is met
+      doshutdown = "systemctl poweroff";
+    };
+  };
+}

services/wg.nix

@@ -1,9 +1,18 @@
-{ pkgs, service_configs, ... }:
 {
+  pkgs,
+  config,
+  inputs,
+  ...
+}:
+{
+  imports = [
+    inputs.vpn-confinement.nixosModules.default
+  ];
+
   # network namespace that is proxied through mullvad
   vpnNamespaces.wg = {
     enable = true;
-    wireguardConfigFile = ../secrets/wg0.conf;
+    wireguardConfigFile = config.age.secrets.wg0-conf.path;
     accessibleFrom = [
       # "192.168.0.0/24"
     ];

services/xmrig.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  lib,
+  pkgs,
+  hostname,
+  ...
+}:
+let
+  walletAddress = lib.strings.trim (builtins.readFile ../secrets/xmrig-wallet);
+  threadCount = 12;
+in
+{
+  services.xmrig = {
+    enable = true;
+    package = pkgs.xmrig;
+
+    settings = {
+      autosave = true;
+
+      cpu = {
+        enabled = true;
+        huge-pages = true;
+        hw-aes = true;
+        rx = lib.range 0 (threadCount - 1);
+      };
+
+      randomx = {
+        "1gb-pages" = true;
+      };
+
+      opencl = false;
+      cuda = false;
+
+      pools = [
+        {
+          url = "gulf.moneroocean.stream:20128";
+          user = walletAddress;
+          pass = hostname + "~rx/0";
+          keepalive = true;
+          tls = true;
+        }
+      ];
+    };
+  };
+
+  systemd.services.xmrig.serviceConfig = {
+    Nice = 19;
+    CPUSchedulingPolicy = "idle";
+    IOSchedulingClass = "idle";
+  };
+
+  # Stop mining on UPS battery to conserve power
+  services.apcupsd.hooks = lib.mkIf config.services.apcupsd.enable {
+    onbattery = "systemctl stop xmrig";
+    offbattery = "systemctl start xmrig";
+  };
+
+  # Reserve 1GB huge pages for RandomX (dataset is ~2GB)
+  boot.kernelParams = [
+    "hugepagesz=1G"
+    "hugepages=3"
+  ];
+}

tests/fail2ban-caddy.nix

@@ -0,0 +1,124 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-caddy";
+
+  nodes = {
+    server =
+      {
+        config,
+        pkgs,
+        lib,
+        ...
+      }:
+      {
+        imports = [
+          ../modules/security.nix
+        ];
+
+        # Set up Caddy with basic auth (minimal config, no production stuff)
+        # Using bcrypt hash generated with: caddy hash-password --plaintext testpass
+        services.caddy = {
+          enable = true;
+          virtualHosts.":80".extraConfig = ''
+            log {
+              output file /var/log/caddy/access-server.log
+              format json
+            }
+            basic_auth {
+              testuser $2a$14$XqaQlGTdmofswciqrLlMz.rv0/jiGQq8aU.fP6mh6gCGiLf6Cl3.a
+            }
+            respond "Authenticated!" 200
+          '';
+        };
+
+        # Add the fail2ban jail for caddy-auth (same as in services/caddy.nix)
+        services.fail2ban.jails.caddy-auth = {
+          enabled = true;
+          settings = {
+            backend = "auto";
+            port = "http,https";
+            logpath = "/var/log/caddy/access-*.log";
+            maxretry = 3; # Lower for testing
+          };
+          filter.Definition = {
+            # Only match 401s where an Authorization header was actually sent
+            failregex = ''^.*"remote_ip":"<HOST>".*"Authorization":\["REDACTED"\].*"status":401.*$'';
+            ignoreregex = "";
+            datepattern = ''"ts":{Epoch}\.'';
+          };
+        };
+
+        # Create log directory and initial log file so fail2ban can start
+        systemd.tmpfiles.rules = [
+          "d /var/log/caddy 755 caddy caddy"
+          "f /var/log/caddy/access-server.log 644 caddy caddy"
+        ];
+
+        networking.firewall.allowedTCPPorts = [ 80 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("caddy.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(80)
+    time.sleep(2)
+
+    with subtest("Verify caddy-auth jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "caddy-auth" in status, f"caddy-auth jail not found in: {status}"
+
+    with subtest("Verify correct password works"):
+        # Use -4 to force IPv4 for consistency
+        result = client.succeed("curl -4 -s -u testuser:testpass http://server/")
+        print(f"Curl result: {result}")
+        assert "Authenticated" in result, f"Auth should succeed: {result}"
+
+    with subtest("Unauthenticated requests (browser probes) should not trigger ban"):
+        # Simulate browser probe requests - no Authorization header sent
+        # This is the normal HTTP Basic Auth challenge-response flow:
+        # browser sends request without credentials, gets 401, then resends with credentials
+        for i in range(5):
+            client.execute("curl -4 -s http://server/ || true")
+            time.sleep(0.5)
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status caddy-auth")
+        print(f"caddy-auth jail status after unauthenticated requests: {status}")
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        banned = int(match.group(1)) if match else 0
+        assert banned == 0, f"Unauthenticated 401s should NOT trigger ban, but {banned} IPs were banned: {status}"
+
+    with subtest("Generate failed basic auth attempts (wrong password)"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        # These send an Authorization header with wrong credentials
+        for i in range(4):
+            client.execute("curl -4 -s -u testuser:wrongpass http://server/ || true")
+            time.sleep(1)
+
+    with subtest("Verify IP is banned after wrong password attempts"):
+        time.sleep(5)
+        status = server.succeed("fail2ban-client status caddy-auth")
+        print(f"caddy-auth jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-gitea.nix

@@ -0,0 +1,123 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    gitea = {
+      dir = "/var/lib/gitea";
+      domain = "git.test.local";
+    };
+    postgres = {
+      socket = "/run/postgresql";
+    };
+    ports = {
+      gitea = 3000;
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool =
+        serviceName: zpool: dirs:
+        { ... }:
+        { };
+      serviceFilePerms = serviceName: tmpfilesRules: { ... }: { };
+    }
+  );
+
+  giteaModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/gitea.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-gitea";
+
+  nodes = {
+    server =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        imports = [
+          ../modules/security.nix
+          giteaModule
+        ];
+
+        # Enable postgres for gitea
+        services.postgresql.enable = true;
+
+        # Disable ZFS mount dependency
+        systemd.services."gitea-mounts".enable = lib.mkForce false;
+        systemd.services.gitea = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ "postgresql.service" ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.gitea.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Gitea port, not via Caddy
+          port = lib.mkForce "3000";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 3000 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("postgresql.service")
+    server.wait_for_unit("gitea.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(3000)
+    time.sleep(3)
+
+    with subtest("Verify gitea jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "gitea" in status, f"gitea jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute(
+                "curl -4 -s -X POST http://server:3000/user/login -d 'user_name=baduser&password=badpass' || true"
+            )
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status gitea")
+        print(f"gitea jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:3000/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-immich.nix

@@ -0,0 +1,135 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      immich = 2283;
+    };
+    immich = {
+      dir = "/var/lib/immich";
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool =
+        serviceName: zpool: dirs:
+        { ... }:
+        { };
+      serviceFilePerms = serviceName: tmpfilesRules: { ... }: { };
+    }
+  );
+
+  immichModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/immich.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-immich";
+
+  nodes = {
+    server =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        imports = [
+          ../modules/security.nix
+          immichModule
+        ];
+
+        # Immich needs postgres
+        services.postgresql.enable = true;
+
+        # Let immich create its own DB for testing
+        services.immich.database.createDB = lib.mkForce true;
+
+        # Disable ZFS mount dependencies
+        systemd.services."immich-server-mounts".enable = lib.mkForce false;
+        systemd.services."immich-machine-learning-mounts".enable = lib.mkForce false;
+        systemd.services.immich-server = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ "postgresql.service" ];
+          requires = lib.mkForce [ ];
+        };
+        systemd.services.immich-machine-learning = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.immich.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Immich port, not via Caddy
+          port = lib.mkForce "2283";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 2283 ];
+
+        # Immich needs more resources
+        virtualisation.diskSize = 4 * 1024;
+        virtualisation.memorySize = 4 * 1024; # 4GB RAM for Immich
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("postgresql.service")
+    server.wait_for_unit("immich-server.service", timeout=120)
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(2283, timeout=60)
+    time.sleep(3)
+
+    with subtest("Verify immich jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "immich" in status, f"immich jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute(
+                "curl -4 -s -X POST http://server:2283/api/auth/login -H 'Content-Type: application/json' -d '{\"email\":\"bad@user.com\",\"password\":\"badpass\"}' || true"
+            )
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status immich")
+        print(f"immich jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:2283/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-jellyfin.nix

@@ -0,0 +1,147 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      jellyfin = 8096;
+    };
+    jellyfin = {
+      dataDir = "/var/lib/jellyfin";
+      cacheDir = "/var/cache/jellyfin";
+    };
+    media_group = "media";
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool =
+        serviceName: zpool: dirs:
+        { ... }:
+        { };
+      serviceFilePerms = serviceName: tmpfilesRules: { ... }: { };
+      optimizePackage = pkg: pkg; # No-op for testing
+    }
+  );
+
+  jellyfinModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/jellyfin.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-jellyfin";
+
+  nodes = {
+    server =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        imports = [
+          ../modules/security.nix
+          jellyfinModule
+        ];
+
+        # Create the media group
+        users.groups.media = { };
+
+        # Disable ZFS mount dependency
+        systemd.services."jellyfin-mounts".enable = lib.mkForce false;
+        systemd.services.jellyfin = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.jellyfin.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Jellyfin port, not via Caddy
+          port = lib.mkForce "8096";
+        };
+
+        # Create log directory and placeholder log file for fail2ban
+        # Jellyfin logs to files, not systemd journal
+        systemd.tmpfiles.rules = [
+          "d /var/lib/jellyfin/log 0755 jellyfin jellyfin"
+          "f /var/lib/jellyfin/log/log_placeholder.log 0644 jellyfin jellyfin"
+        ];
+
+        # Make fail2ban start after Jellyfin
+        systemd.services.fail2ban = {
+          wants = [ "jellyfin.service" ];
+          after = [ "jellyfin.service" ];
+        };
+
+        # Give jellyfin more disk space and memory
+        virtualisation.diskSize = 3 * 1024;
+        virtualisation.memorySize = 2 * 1024;
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("jellyfin.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(8096)
+    server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
+    time.sleep(2)
+
+    # Wait for Jellyfin to create real log files and reload fail2ban
+    server.wait_until_succeeds("ls /var/lib/jellyfin/log/log_2*.log", timeout=30)
+    server.succeed("fail2ban-client reload jellyfin")
+
+    with subtest("Verify jellyfin jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "jellyfin" in status, f"jellyfin jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute("""
+                curl -4 -s -X POST http://server:8096/Users/authenticatebyname \
+                  -H 'Content-Type: application/json' \
+                  -H 'X-Emby-Authorization: MediaBrowser Client="test", Device="test", DeviceId="test", Version="1.0"' \
+                  -d '{"Username":"baduser","Pw":"badpass"}' || true
+            """)
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status jellyfin")
+        print(f"jellyfin jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:8096/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-ssh.nix

@@ -0,0 +1,104 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    zpool_hdds = "";
+  };
+
+  securityModule = import ../modules/security.nix;
+
+  sshModule =
+    {
+      config,
+      lib,
+      pkgs,
+      ...
+    }:
+    {
+      imports = [
+        (import ../services/ssh.nix {
+          inherit config lib pkgs;
+          username = "testuser";
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-ssh";
+
+  nodes = {
+    server =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        imports = [
+          securityModule
+          sshModule
+        ];
+
+        # Override for testing - enable password auth
+        services.openssh.settings.PasswordAuthentication = lib.mkForce true;
+
+        users.users.testuser = {
+          isNormalUser = true;
+          password = "correctpassword";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 22 ];
+      };
+
+    client = {
+      environment.systemPackages = with pkgs; [
+        sshpass
+        openssh
+      ];
+    };
+  };
+
+  testScript = ''
+    import time
+
+    start_all()
+    server.wait_for_unit("sshd.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(22)
+    time.sleep(2)
+
+    with subtest("Verify sshd jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "sshd" in status, f"sshd jail not found in: {status}"
+
+    with subtest("Generate failed SSH login attempts"):
+        # Use -4 to force IPv4, timeout and NumberOfPasswordPrompts=1 to ensure quick failure
+        # maxRetry is 3 in our config, so 4 attempts should trigger a ban
+        for i in range(4):
+            client.execute(
+                "timeout 5 sshpass -p 'wrongpassword' ssh -4 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=3 -o NumberOfPasswordPrompts=1 testuser@server echo test 2>/dev/null || true"
+            )
+            time.sleep(1)
+
+    with subtest("Verify IP is banned"):
+        # Wait for fail2ban to process the logs and apply the ban
+        time.sleep(5)
+        status = server.succeed("fail2ban-client status sshd")
+        print(f"sshd jail status: {status}")
+        # Check that at least 1 IP is banned
+        import re
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("timeout 3 nc -4 -z -w 2 server 22")[0]
+        assert exit_code != 0, "Connection should be blocked for banned IP"
+  '';
+}

tests/fail2ban-vaultwarden.nix

@@ -0,0 +1,137 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      vaultwarden = 8222;
+    };
+    vaultwarden = {
+      path = "/var/lib/vaultwarden";
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool =
+        serviceName: zpool: dirs:
+        { ... }:
+        { };
+      serviceFilePerms = serviceName: tmpfilesRules: { ... }: { };
+    }
+  );
+
+  vaultwardenModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/bitwarden.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-vaultwarden";
+
+  nodes = {
+    server =
+      {
+        config,
+        lib,
+        pkgs,
+        ...
+      }:
+      {
+        imports = [
+          ../modules/security.nix
+          vaultwardenModule
+        ];
+
+        # Disable ZFS mount dependencies
+        systemd.services."vaultwarden-mounts".enable = lib.mkForce false;
+        systemd.services."backup-vaultwarden-mounts".enable = lib.mkForce false;
+        systemd.services.vaultwarden = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+        systemd.services.backup-vaultwarden = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override Vaultwarden settings for testing
+        # - Listen on all interfaces (not just localhost)
+        # - Enable logging at info level to capture failed login attempts
+        services.vaultwarden.config = {
+          ROCKET_ADDRESS = lib.mkForce "0.0.0.0";
+          ROCKET_LOG = lib.mkForce "info";
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.vaultwarden.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Vaultwarden port, not via Caddy
+          port = lib.mkForce "8222";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 8222 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("vaultwarden.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(8222)
+    time.sleep(2)
+
+    with subtest("Verify vaultwarden jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "vaultwarden" in status, f"vaultwarden jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute("""
+                curl -4 -s -X POST 'http://server:8222/identity/connect/token' \
+                  -H 'Content-Type: application/x-www-form-urlencoded' \
+                  -H 'Bitwarden-Client-Name: web' \
+                  -H 'Bitwarden-Client-Version: 2024.1.0' \
+                  -d 'grant_type=password&username=bad@user.com&password=badpass&scope=api+offline_access&client_id=web&deviceType=10&deviceIdentifier=test&deviceName=test' \
+                  || true
+            """)
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status vaultwarden")
+        print(f"vaultwarden jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:8222/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/file-perms.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
+in
+testPkgs.testers.runNixOSTest {
+  name = "file-perms test";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      imports = [
+        (lib.serviceFilePerms "test-service" [
+          "Z /tmp/test-perms-dir 0750 nobody nogroup"
+        ])
+      ];
+
+      systemd.services."test-service" = {
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = lib.getExe pkgs.bash;
+        };
+      };
+    };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("multi-user.target")
+
+    # Create test directory with wrong permissions
+    machine.succeed("mkdir -p /tmp/test-perms-dir")
+    machine.succeed("chown root:root /tmp/test-perms-dir")
+    machine.succeed("chmod 700 /tmp/test-perms-dir")
+
+    # Start service -- this should pull in test-service-file-perms
+    machine.succeed("systemctl start test-service")
+
+    # Verify file-perms service ran and is active
+    machine.succeed("systemctl is-active test-service-file-perms.service")
+
+    # Verify permissions were fixed by tmpfiles
+    result = machine.succeed("stat -c '%U:%G' /tmp/test-perms-dir").strip()
+    assert result == "nobody:nogroup", f"Expected nobody:nogroup, got {result}"
+
+    result = machine.succeed("stat -c '%a' /tmp/test-perms-dir").strip()
+    assert result == "750", f"Expected 750, got {result}"
+  '';
+}

tests/jellyfin-qbittorrent-monitor.nix

@@ -0,0 +1,583 @@
+{
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+let
+  payloads = {
+    auth = pkgs.writeText "auth.json" (builtins.toJSON { Username = "jellyfin"; });
+    empty = pkgs.writeText "empty.json" (builtins.toJSON { });
+  };
+in
+pkgs.testers.runNixOSTest {
+  name = "jellyfin-qbittorrent-monitor";
+
+  nodes = {
+    server =
+      { ... }:
+      {
+        imports = [
+          inputs.vpn-confinement.nixosModules.default
+        ];
+
+        services.jellyfin.enable = true;
+
+        # Real qBittorrent service
+        services.qbittorrent = {
+          enable = true;
+          webuiPort = 8080;
+          openFirewall = true;
+
+          serverConfig.LegalNotice.Accepted = true;
+
+          serverConfig.Preferences = {
+            WebUI = {
+              # Disable authentication for testing
+              AuthSubnetWhitelist = "0.0.0.0/0,::/0";
+              AuthSubnetWhitelistEnabled = true;
+              LocalHostAuth = false;
+            };
+
+            Downloads = {
+              SavePath = "/var/lib/qbittorrent/downloads";
+              TempPath = "/var/lib/qbittorrent/incomplete";
+            };
+          };
+
+          serverConfig.BitTorrent.Session = {
+            # Normal speed - unlimited
+            GlobalUPSpeedLimit = 0;
+            GlobalDLSpeedLimit = 0;
+
+            # Alternate speed limits for when Jellyfin is streaming
+            AlternativeGlobalUPSpeedLimit = 100;
+            AlternativeGlobalDLSpeedLimit = 100;
+          };
+        };
+
+        environment.systemPackages = with pkgs; [
+          curl
+          ffmpeg
+        ];
+        virtualisation.diskSize = 3 * 1024;
+        networking.firewall.allowedTCPPorts = [
+          8096
+          8080
+        ];
+        networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+          {
+            address = "192.168.1.1";
+            prefixLength = 24;
+          }
+        ];
+        networking.interfaces.eth1.ipv4.routes = [
+          {
+            address = "203.0.113.0";
+            prefixLength = 24;
+          }
+        ];
+
+        # Create directories for qBittorrent
+        systemd.tmpfiles.rules = [
+          "d /var/lib/qbittorrent/downloads 0755 qbittorrent qbittorrent"
+          "d /var/lib/qbittorrent/incomplete 0755 qbittorrent qbittorrent"
+        ];
+      };
+
+    # Public test IP (RFC 5737 TEST-NET-3) so Jellyfin sees it as external
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+      networking.interfaces.eth1.ipv4.addresses = lib.mkForce [
+        {
+          address = "203.0.113.10";
+          prefixLength = 24;
+        }
+      ];
+      networking.interfaces.eth1.ipv4.routes = [
+        {
+          address = "192.168.1.0";
+          prefixLength = 24;
+        }
+      ];
+    };
+  };
+
+  testScript = ''
+    import json
+    import time
+    from urllib.parse import urlencode
+
+    auth_header = 'MediaBrowser Client="NixOS Test", DeviceId="test-1337", Device="TestDevice", Version="1.0"'
+
+    def api_get(path, token=None):
+        header = auth_header + (f", Token={token}" if token else "")
+        return f"curl -sf 'http://server:8096{path}' -H 'X-Emby-Authorization:{header}'"
+
+    def api_post(path, json_file=None, token=None):
+        header = auth_header + (f", Token={token}" if token else "")
+        if json_file:
+            return f"curl -sf -X POST 'http://server:8096{path}' -d '@{json_file}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{header}'"
+        return f"curl -sf -X POST 'http://server:8096{path}' -H 'X-Emby-Authorization:{header}'"
+
+    def is_throttled():
+        return server.succeed("curl -s http://localhost:8080/api/v2/transfer/speedLimitsMode").strip() == "1"
+
+    def get_alt_dl_limit():
+        prefs = json.loads(server.succeed("curl -s http://localhost:8080/api/v2/app/preferences"))
+        return prefs["alt_dl_limit"]
+
+    def get_alt_up_limit():
+        prefs = json.loads(server.succeed("curl -s http://localhost:8080/api/v2/app/preferences"))
+        return prefs["alt_up_limit"]
+
+    def are_torrents_paused():
+        torrents = json.loads(server.succeed("curl -s 'http://localhost:8080/api/v2/torrents/info'"))
+        if not torrents:
+            return False
+        return all(t["state"].startswith("stopped") for t in torrents)
+
+    movie_id: str = ""
+    media_source_id: str = ""
+
+    start_all()
+    server.wait_for_unit("jellyfin.service")
+    server.wait_for_open_port(8096)
+    server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
+    server.wait_for_unit("qbittorrent.service")
+    server.wait_for_open_port(8080)
+
+    # Wait for qBittorrent WebUI to be responsive
+    server.wait_until_succeeds("curl -sf http://localhost:8080/api/v2/app/version", timeout=30)
+
+    with subtest("Complete Jellyfin setup wizard"):
+        server.wait_until_succeeds(api_get("/Startup/Configuration"))
+        server.succeed(api_get("/Startup/FirstUser"))
+        server.succeed(api_post("/Startup/Complete"))
+
+    with subtest("Authenticate and get token"):
+        auth_result = json.loads(server.succeed(api_post("/Users/AuthenticateByName", "${payloads.auth}")))
+        token = auth_result["AccessToken"]
+        user_id = auth_result["User"]["Id"]
+
+    with subtest("Create test video library"):
+        tempdir = server.succeed("mktemp -d -p /var/lib/jellyfin").strip()
+        server.succeed(f"chmod 755 '{tempdir}'")
+        server.succeed(f"ffmpeg -f lavfi -i testsrc2=duration=5 '{tempdir}/Test Movie (2024) [1080p].mkv'")
+
+        add_folder_query = urlencode({
+            "name": "Test Library",
+            "collectionType": "Movies",
+            "paths": tempdir,
+            "refreshLibrary": "true",
+        })
+        server.succeed(api_post(f"/Library/VirtualFolders?{add_folder_query}", "${payloads.empty}", token))
+
+        def is_library_ready(_):
+            folders = json.loads(server.succeed(api_get("/Library/VirtualFolders", token)))
+            return all(f.get("RefreshStatus") == "Idle" for f in folders)
+        retry(is_library_ready, timeout=60)
+
+        def get_movie(_):
+            global movie_id, media_source_id
+            items = json.loads(server.succeed(api_get(f"/Users/{user_id}/Items?IncludeItemTypes=Movie&Recursive=true", token)))
+            if items["TotalRecordCount"] > 0:
+                movie_id = items["Items"][0]["Id"]
+                item_info = json.loads(server.succeed(api_get(f"/Users/{user_id}/Items/{movie_id}", token)))
+                media_source_id = item_info["MediaSources"][0]["Id"]
+                return True
+            return False
+        retry(get_movie, timeout=60)
+
+    with subtest("Start monitor service"):
+        python = "${pkgs.python3.withPackages (ps: [ ps.requests ])}/bin/python"
+        monitor = "${../services/jellyfin-qbittorrent-monitor.py}"
+        server.succeed(f"""
+          systemd-run --unit=monitor-test \
+            --setenv=JELLYFIN_URL=http://localhost:8096 \
+            --setenv=JELLYFIN_API_KEY={token} \
+            --setenv=QBITTORRENT_URL=http://localhost:8080 \
+            --setenv=CHECK_INTERVAL=1 \
+            --setenv=STREAMING_START_DELAY=1 \
+            --setenv=STREAMING_STOP_DELAY=1 \
+            --setenv=TOTAL_BANDWIDTH_BUDGET=50000000 \
+            --setenv=SERVICE_BUFFER=2000000 \
+            --setenv=DEFAULT_STREAM_BITRATE=10000000 \
+            --setenv=MIN_TORRENT_SPEED=100 \
+            {python} {monitor}
+        """)
+        time.sleep(2)
+        assert not is_throttled(), "Should start unthrottled"
+
+    client_auth = 'MediaBrowser Client="External Client", DeviceId="external-9999", Device="ExternalDevice", Version="1.0"'
+    client_auth2 = 'MediaBrowser Client="External Client 2", DeviceId="external-8888", Device="ExternalDevice2", Version="1.0"'
+    server_ip = "192.168.1.1"
+
+    with subtest("Client authenticates from external network"):
+        auth_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+        client_auth_result = json.loads(client.succeed(auth_cmd))
+        client_token = client_auth_result["AccessToken"]
+
+    with subtest("Second client authenticates from external network"):
+        auth_cmd2 = f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+        client_auth_result2 = json.loads(client.succeed(auth_cmd2))
+        client_token2 = client_auth_result2["AccessToken"]
+
+    with subtest("External video playback triggers throttling"):
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-1",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(2)
+        assert is_throttled(), "Should throttle for external video playback"
+
+    with subtest("Pausing disables throttling"):
+        playback_progress = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-1",
+            "IsPaused": True,
+            "PositionTicks": 10000000,
+        }
+        progress_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Progress' -d '{json.dumps(playback_progress)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(progress_cmd)
+        time.sleep(2)
+
+        assert not is_throttled(), "Should unthrottle when paused"
+
+    with subtest("Resuming re-enables throttling"):
+        playback_progress["IsPaused"] = False
+        playback_progress["PositionTicks"] = 20000000
+        progress_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Progress' -d '{json.dumps(playback_progress)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(progress_cmd)
+        time.sleep(2)
+
+        assert is_throttled(), "Should re-throttle when resumed"
+
+    with subtest("Stopping playback disables throttling"):
+        playback_stop = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-1",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(playback_stop)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(stop_cmd)
+        time.sleep(2)
+
+        assert not is_throttled(), "Should unthrottle when playback stops"
+
+    with subtest("Single stream sets proportional alt speed limits"):
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-proportional",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(3)
+
+        assert is_throttled(), "Should be in alt speed mode during streaming"
+        dl_limit = get_alt_dl_limit()
+        ul_limit = get_alt_up_limit()
+        # Both upload and download should get remaining bandwidth (proportional)
+        assert dl_limit > 0, f"Download limit should be > 0, got {dl_limit}"
+        assert ul_limit == dl_limit, f"Upload limit ({ul_limit}) should equal download limit ({dl_limit})"
+
+        # Stop playback
+        playback_stop = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-proportional",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(playback_stop)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(stop_cmd)
+        time.sleep(3)
+
+    with subtest("Multiple streams reduce available bandwidth"):
+        # Start first stream
+        playback1 = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-multi-1",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd1 = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback1)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd1)
+        time.sleep(3)
+
+        single_dl_limit = get_alt_dl_limit()
+
+        # Start second stream with different client identity
+        playback2 = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-multi-2",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd2 = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback2)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}, Token={client_token2}'"
+        client.succeed(start_cmd2)
+        time.sleep(3)
+
+        dual_dl_limit = get_alt_dl_limit()
+        # Two streams should leave less bandwidth than one stream
+        assert dual_dl_limit < single_dl_limit, f"Two streams ({dual_dl_limit}) should have lower limit than one ({single_dl_limit})"
+
+        # Stop both streams
+        stop1 = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-multi-1",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd1 = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(stop1)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(stop_cmd1)
+
+        stop2 = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-multi-2",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd2 = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(stop2)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}, Token={client_token2}'"
+        client.succeed(stop_cmd2)
+        time.sleep(3)
+
+    with subtest("Budget exhaustion pauses all torrents"):
+        # Stop current monitor
+        server.succeed("systemctl stop monitor-test || true")
+        time.sleep(1)
+
+        # Add a dummy torrent so we can check pause state
+        server.succeed("curl -sf -X POST 'http://localhost:8080/api/v2/torrents/add' -d 'urls=magnet:?xt=urn:btih:0000000000000000000000000000000000000001%26dn=test-torrent'")
+        time.sleep(2)
+
+        # Start monitor with impossibly low budget
+        server.succeed(f"""
+          systemd-run --unit=monitor-exhaust \
+            --setenv=JELLYFIN_URL=http://localhost:8096 \
+            --setenv=JELLYFIN_API_KEY={token} \
+            --setenv=QBITTORRENT_URL=http://localhost:8080 \
+            --setenv=CHECK_INTERVAL=1 \
+            --setenv=STREAMING_START_DELAY=1 \
+            --setenv=STREAMING_STOP_DELAY=1 \
+            --setenv=TOTAL_BANDWIDTH_BUDGET=1000 \
+            --setenv=SERVICE_BUFFER=500 \
+            --setenv=DEFAULT_STREAM_BITRATE=10000000 \
+            --setenv=MIN_TORRENT_SPEED=100 \
+            {python} {monitor}
+        """)
+        time.sleep(2)
+
+        # Start a stream - this will exceed the tiny budget
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-exhaust",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(3)
+
+        assert are_torrents_paused(), "Torrents should be paused when budget is exhausted"
+
+    with subtest("Recovery from pause restores unlimited"):
+        # Stop the stream
+        playback_stop = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-exhaust",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(playback_stop)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(stop_cmd)
+        time.sleep(3)
+
+        assert not is_throttled(), "Should return to unlimited after streams stop"
+        assert not are_torrents_paused(), "Torrents should be resumed after streams stop"
+
+        # Clean up: stop exhaust monitor, restart normal monitor
+        server.succeed("systemctl stop monitor-exhaust || true")
+        time.sleep(1)
+        server.succeed(f"""
+          systemd-run --unit=monitor-test \
+            --setenv=JELLYFIN_URL=http://localhost:8096 \
+            --setenv=JELLYFIN_API_KEY={token} \
+            --setenv=QBITTORRENT_URL=http://localhost:8080 \
+            --setenv=CHECK_INTERVAL=1 \
+            --setenv=STREAMING_START_DELAY=1 \
+            --setenv=STREAMING_STOP_DELAY=1 \
+            --setenv=TOTAL_BANDWIDTH_BUDGET=50000000 \
+            --setenv=SERVICE_BUFFER=2000000 \
+            --setenv=DEFAULT_STREAM_BITRATE=10000000 \
+            --setenv=MIN_TORRENT_SPEED=100 \
+            {python} {monitor}
+        """)
+        time.sleep(2)
+
+    with subtest("Local playback does NOT trigger throttling"):
+        local_auth = 'MediaBrowser Client="Local Client", DeviceId="local-1111", Device="LocalDevice", Version="1.0"'
+        local_auth_result = json.loads(server.succeed(
+            f"curl -sf -X POST 'http://localhost:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{local_auth}'"
+        ))
+        local_token = local_auth_result["AccessToken"]
+
+        local_playback = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-local",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        server.succeed(f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing' -d '{json.dumps(local_playback)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{local_auth}, Token={local_token}'")
+        time.sleep(2)
+        assert not is_throttled(), "Should NOT throttle for local playback"
+
+        local_playback["PositionTicks"] = 50000000
+        server.succeed(f"curl -sf -X POST 'http://localhost:8096/Sessions/Playing/Stopped' -d '{json.dumps(local_playback)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{local_auth}, Token={local_token}'")
+
+    # === SERVICE RESTART TESTS ===
+
+    with subtest("qBittorrent restart during throttled state re-applies throttling"):
+        # Start external playback to trigger throttling
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-restart-1",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(2)
+        assert is_throttled(), "Should be throttled before qBittorrent restart"
+
+        # Restart qBittorrent (this resets alt_speed to its config default - disabled)
+        server.succeed("systemctl restart qbittorrent.service")
+        server.wait_for_unit("qbittorrent.service")
+        server.wait_for_open_port(8080)
+        server.wait_until_succeeds("curl -sf http://localhost:8080/api/v2/app/version", timeout=30)
+
+        # qBittorrent restarted - alt_speed is now False (default on startup)
+        # The monitor should detect this and re-apply throttling
+        time.sleep(3)  # Give monitor time to detect and re-apply
+        assert is_throttled(), "Monitor should re-apply throttling after qBittorrent restart"
+
+        # Stop playback to clean up
+        playback_stop = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-restart-1",
+            "PositionTicks": 50000000,
+        }
+        stop_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing/Stopped' -d '{json.dumps(playback_stop)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(stop_cmd)
+        time.sleep(2)
+
+    with subtest("qBittorrent restart during unthrottled state stays unthrottled"):
+        # Verify we're unthrottled (no active streams)
+        assert not is_throttled(), "Should be unthrottled before test"
+
+        # Restart qBittorrent
+        server.succeed("systemctl restart qbittorrent.service")
+        server.wait_for_unit("qbittorrent.service")
+        server.wait_for_open_port(8080)
+        server.wait_until_succeeds("curl -sf http://localhost:8080/api/v2/app/version", timeout=30)
+
+        # Give monitor time to check state
+        time.sleep(3)
+        assert not is_throttled(), "Should remain unthrottled after qBittorrent restart with no streams"
+
+    with subtest("Jellyfin restart during throttled state maintains throttling"):
+        # Start external playback to trigger throttling
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-restart-2",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(2)
+        assert is_throttled(), "Should be throttled before Jellyfin restart"
+
+        # Restart Jellyfin
+        server.succeed("systemctl restart jellyfin.service")
+        server.wait_for_unit("jellyfin.service")
+        server.wait_for_open_port(8096)
+        server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
+
+        # During Jellyfin restart, monitor can't reach Jellyfin
+        # After restart, sessions are cleared - monitor should eventually unthrottle
+        # But during the unavailability window, throttling should be maintained (fail-safe)
+        time.sleep(3)
+
+        # Re-authenticate (old token invalid after restart)
+        client_auth_result = json.loads(client.succeed(
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+        ))
+        client_token = client_auth_result["AccessToken"]
+        client_auth_result2 = json.loads(client.succeed(
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+        ))
+        client_token2 = client_auth_result2["AccessToken"]
+
+        # No active streams after Jellyfin restart, should eventually unthrottle
+        time.sleep(3)
+        assert not is_throttled(), "Should unthrottle after Jellyfin restart clears sessions"
+
+    with subtest("Monitor recovers after Jellyfin temporary unavailability"):
+        # Re-authenticate with fresh token
+        client_auth_result = json.loads(client.succeed(
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}'"
+        ))
+        client_token = client_auth_result["AccessToken"]
+        client_auth_result2 = json.loads(client.succeed(
+            f"curl -sf -X POST 'http://{server_ip}:8096/Users/AuthenticateByName' -d '@${payloads.auth}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth2}'"
+        ))
+        client_token2 = client_auth_result2["AccessToken"]
+
+        # Start playback
+        playback_start = {
+            "ItemId": movie_id,
+            "MediaSourceId": media_source_id,
+            "PlaySessionId": "test-play-session-restart-3",
+            "CanSeek": True,
+            "IsPaused": False,
+        }
+        start_cmd = f"curl -sf -X POST 'http://{server_ip}:8096/Sessions/Playing' -d '{json.dumps(playback_start)}' -H 'Content-Type:application/json' -H 'X-Emby-Authorization:{client_auth}, Token={client_token}'"
+        client.succeed(start_cmd)
+        time.sleep(2)
+        assert is_throttled(), "Should be throttled"
+
+        # Stop Jellyfin briefly (simulating temporary unavailability)
+        server.succeed("systemctl stop jellyfin.service")
+        time.sleep(2)
+
+        # During unavailability, throttle state should be maintained (fail-safe)
+        assert is_throttled(), "Should maintain throttle during Jellyfin unavailability"
+
+        # Bring Jellyfin back
+        server.succeed("systemctl start jellyfin.service")
+        server.wait_for_unit("jellyfin.service")
+        server.wait_for_open_port(8096)
+        server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
+
+        # After Jellyfin comes back, sessions are gone - should unthrottle
+        time.sleep(3)
+        assert not is_throttled(), "Should unthrottle after Jellyfin returns with no sessions"
+  '';
+}

tests/minecraft.nix

@@ -0,0 +1,98 @@
+{
+  config,
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    minecraft = {
+      server_name = "main";
+      parent_dir = "/var/lib/minecraft";
+    };
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      minecraft = 25565;
+    };
+    zpool_ssds = "";
+  };
+
+  # Create pkgs with nix-minecraft overlay and unfree packages allowed
+  testPkgs = import inputs.nixpkgs {
+    system = pkgs.stdenv.targetPlatform.system;
+    config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "minecraft-server" ];
+    overlays = [
+      inputs.nix-minecraft.overlay
+      (import ../modules/overlays.nix)
+    ];
+  };
+in
+testPkgs.testers.runNixOSTest {
+  name = "minecraft server startup test";
+
+  node.specialArgs = {
+    inherit inputs lib;
+    service_configs = testServiceConfigs;
+    username = "testuser";
+  };
+
+  nodes.machine =
+    { lib, ... }:
+    {
+      imports = [
+        ../services/minecraft.nix
+      ];
+
+      # Enable caddy service (required by minecraft service)
+      services.caddy.enable = true;
+
+      # Enable networking for the test (needed for minecraft mods to download mappings)
+      networking.dhcpcd.enable = true;
+
+      # Disable the ZFS mount dependency service in test environment
+      systemd.services."minecraft-server-main_mounts".enable = lib.mkForce false;
+
+      # Remove service dependencies that require ZFS
+      systemd.services.minecraft-server-main = {
+        wants = lib.mkForce [ ];
+        after = lib.mkForce [ ];
+        requires = lib.mkForce [ ];
+        serviceConfig = {
+          Nice = lib.mkForce 0;
+          LimitMEMLOCK = lib.mkForce "infinity";
+        };
+      };
+
+      # Test-specific overrides only - reduce memory for testing
+      services.minecraft-servers.servers.main.jvmOpts = lib.mkForce "-Xmx1G -Xms1G";
+
+      # Create test user
+      users.users.testuser = {
+        isNormalUser = true;
+        uid = 1000;
+        extraGroups = [ "minecraft" ];
+      };
+    };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("multi-user.target")
+
+    # Wait for minecraft service to be available
+    machine.wait_for_unit("minecraft-server-main.service")
+
+    # Wait up to 60 seconds for the server to complete startup
+    with machine.nested("Waiting for minecraft server startup completion"):
+        try:
+            machine.wait_until_succeeds(
+                "grep -Eq '\\[[0-9]+:[0-9]+:[0-9]+\\] \\[Server thread/INFO\\]: Done \\([0-9]+\\.[0-9]+s\\)! For help, type \"help\"' /var/lib/minecraft/main/logs/latest.log",
+                timeout=120
+            )
+        except Exception:
+            print(machine.succeed("cat /var/lib/minecraft/main/logs/latest.log"))
+            raise
+  '';
+}

tests/ntfy-alerts.nix

@@ -0,0 +1,174 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
+in
+testPkgs.testers.runNixOSTest {
+  name = "ntfy-alerts";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      imports = [
+        ../modules/ntfy-alerts.nix
+      ];
+
+      system.stateVersion = config.system.stateVersion;
+
+      virtualisation.memorySize = 2048;
+
+      environment.systemPackages = with pkgs; [
+        curl
+        jq
+      ];
+
+      # Create test topic file
+      systemd.tmpfiles.rules = [
+        "f /run/ntfy-test-topic 0644 root root - test-alerts"
+      ];
+
+      # Mock ntfy server that records POST requests
+      systemd.services.mock-ntfy =
+        let
+          mockNtfyScript = pkgs.writeScript "mock-ntfy.py" ''
+            import json
+            import os
+            from http.server import HTTPServer, BaseHTTPRequestHandler
+            from datetime import datetime
+
+            REQUESTS_FILE = "/tmp/ntfy-requests.json"
+
+            class MockNtfy(BaseHTTPRequestHandler):
+                def _respond(self, code=200, body=b"Ok"):
+                    self.send_response(code)
+                    self.send_header("Content-Type", "application/json")
+                    self.end_headers()
+                    self.wfile.write(body if isinstance(body, bytes) else body.encode())
+
+                def do_GET(self):
+                    self._respond()
+
+                def do_POST(self):
+                    content_length = int(self.headers.get("Content-Length", 0))
+                    body = self.rfile.read(content_length).decode() if content_length > 0 else ""
+
+                    request_data = {
+                        "timestamp": datetime.now().isoformat(),
+                        "path": self.path,
+                        "headers": dict(self.headers),
+                        "body": body,
+                    }
+
+                    # Load existing requests or start new list
+                    requests = []
+                    if os.path.exists(REQUESTS_FILE):
+                        try:
+                            with open(REQUESTS_FILE, "r") as f:
+                                requests = json.load(f)
+                        except:
+                            requests = []
+
+                    requests.append(request_data)
+
+                    with open(REQUESTS_FILE, "w") as f:
+                        json.dump(requests, f, indent=2)
+
+                    self._respond()
+
+                def log_message(self, format, *args):
+                    pass
+
+            HTTPServer(("0.0.0.0", 8080), MockNtfy).serve_forever()
+          '';
+        in
+        {
+          description = "Mock ntfy server";
+          wantedBy = [ "multi-user.target" ];
+          before = [ "ntfy-alert@test-fail.service" ];
+          serviceConfig = {
+            ExecStart = "${pkgs.python3}/bin/python3 ${mockNtfyScript}";
+            Type = "simple";
+          };
+        };
+
+      # Test service that will fail
+      systemd.services.test-fail = {
+        description = "Test service that fails";
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.coreutils}/bin/false";
+        };
+      };
+
+      # Configure ntfy-alerts to use mock server
+      services.ntfyAlerts = {
+        enable = true;
+        serverUrl = "http://localhost:8080";
+        topicFile = "/run/ntfy-test-topic";
+
+      };
+    };
+
+  testScript = ''
+    import json
+    import time
+
+    start_all()
+
+    # Wait for mock ntfy server to be ready
+    machine.wait_for_unit("mock-ntfy.service")
+    machine.wait_until_succeeds("curl -sf http://localhost:8080/", timeout=30)
+
+    # Verify the ntfy-alert@ template service exists
+    machine.succeed("systemctl list-unit-files | grep ntfy-alert@")
+
+    # Verify the global OnFailure drop-in is configured
+    machine.succeed("cat /etc/systemd/system/service.d/onfailure.conf | grep -q 'OnFailure=ntfy-alert@%p.service'")
+
+    # Trigger the test-fail service
+    machine.succeed("systemctl start test-fail.service || true")
+
+    # Wait a moment for the failure notification to be sent
+    time.sleep(2)
+
+    # Verify the ntfy-alert@test-fail service ran
+    machine.succeed("systemctl is-active ntfy-alert@test-fail.service || systemctl is-failed ntfy-alert@test-fail.service || true")
+
+    # Check that the mock server received a POST request
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+
+    # Verify the request content
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+
+    assert len(requests) >= 1, f"Expected at least 1 request, got {len(requests)}"
+
+    # Check the first request
+    req = requests[0]
+    assert "/test-alerts" in req["path"], f"Expected path to contain /test-alerts, got {req['path']}"
+    assert "Title" in req["headers"], "Expected Title header"
+    assert "test-fail" in req["headers"]["Title"], f"Expected Title to contain 'test-fail', got {req['headers']['Title']}"
+    assert req["headers"]["Priority"] == "high", f"Expected Priority 'high', got {req['headers'].get('Priority')}"
+    assert req["headers"]["Tags"] == "warning", f"Expected Tags 'warning', got {req['headers'].get('Tags')}"
+
+    print(f"Received notification: Title={req['headers']['Title']}, Body={req['body'][:100]}...")
+
+    # Idempotency test: trigger failure again
+    machine.succeed("rm /tmp/ntfy-requests.json")
+    machine.succeed("systemctl reset-failed test-fail.service || true")
+    machine.succeed("systemctl start test-fail.service || true")
+    time.sleep(2)
+
+    # Verify another notification was sent
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+    assert len(requests) >= 1, f"Expected at least 1 request after second failure, got {len(requests)}"
+
+    print("All tests passed!")
+  '';
+}

tests/testTest.nix

@@ -0,0 +1,20 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+pkgs.testers.runNixOSTest {
+  name = "test of tests";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+    };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("multi-user.target")
+    machine.succeed("echo hello!")
+  '';
+}

tests/tests.nix

@@ -0,0 +1,27 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}@args:
+let
+  handleTest = file: import file (args);
+in
+{
+  zfsTest = handleTest ./zfs.nix;
+  testTest = handleTest ./testTest.nix;
+  minecraftTest = handleTest ./minecraft.nix;
+  jellyfinQbittorrentMonitorTest = handleTest ./jellyfin-qbittorrent-monitor.nix;
+  filePermsTest = handleTest ./file-perms.nix;
+
+  # fail2ban tests
+  fail2banSshTest = handleTest ./fail2ban-ssh.nix;
+  fail2banCaddyTest = handleTest ./fail2ban-caddy.nix;
+  fail2banGiteaTest = handleTest ./fail2ban-gitea.nix;
+  fail2banVaultwardenTest = handleTest ./fail2ban-vaultwarden.nix;
+  fail2banImmichTest = handleTest ./fail2ban-immich.nix;
+  fail2banJellyfinTest = handleTest ./fail2ban-jellyfin.nix;
+
+  # ntfy alerts test
+  ntfyAlertsTest = handleTest ./ntfy-alerts.nix;
+}

tests/zfs.nix

@@ -0,0 +1,153 @@
+{
+  config,
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+let
+  # Create pkgs with ensureZfsMounts overlay
+  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
+in
+testPkgs.testers.runNixOSTest {
+  name = "zfs test";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      imports = [
+        # Test valid paths within zpool
+        (lib.serviceMountWithZpool "test-service" "rpool" [ "/mnt/rpool_data" ])
+
+        # Test service with paths outside zpool (should fail assertion)
+        (lib.serviceMountWithZpool "invalid-service" "rpool2" [ "/mnt/rpool_data" ])
+
+        # Test multi-command logic: service with multiple serviceMountWithZpool calls
+        (lib.serviceMountWithZpool "multi-service" "rpool" [ "/mnt/rpool_data" ])
+        (lib.serviceMountWithZpool "multi-service" "rpool2" [ "/mnt/rpool2_data" ])
+
+        # Test multi-command logic: service with multiple serviceMountWithZpool calls
+        # BUT this one should fail as `/mnt/rpool_moar_data` is not on rpool2
+        (lib.serviceMountWithZpool "multi-service-fail" "rpool" [ "/mnt/rpool_data" ])
+        (lib.serviceMountWithZpool "multi-service-fail" "rpool2" [ "/mnt/rpool_moar_data" ])
+      ];
+
+      virtualisation = {
+        emptyDiskImages = [
+          4096
+          4096
+        ];
+        # Add this to avoid ZFS hanging issues
+        additionalPaths = [ pkgs.zfs ];
+      };
+      networking.hostId = "deadbeef";
+      boot.kernelPackages = config.boot.kernelPackages;
+      boot.zfs.package = config.boot.zfs.package;
+      boot.supportedFilesystems = [ "zfs" ];
+
+      environment.systemPackages = with pkgs; [
+        parted
+        ensureZfsMounts
+      ];
+
+      systemd.services."test-service" = {
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = lib.getExe pkgs.bash;
+        };
+      };
+
+      systemd.services."invalid-service" = {
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = lib.getExe pkgs.bash;
+        };
+      };
+
+      systemd.services."multi-service" = {
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = lib.getExe pkgs.bash;
+        };
+      };
+
+      systemd.services."multi-service-fail" = {
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = lib.getExe pkgs.bash;
+        };
+      };
+    };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("multi-user.target")
+
+    # Setup ZFS pool
+    machine.succeed(
+        "parted --script /dev/vdb mklabel msdos",
+        "parted --script /dev/vdb -- mkpart primary 1024M -1s",
+        "zpool create rpool /dev/vdb1"
+    )
+
+    # Setup ZFS pool 2
+    machine.succeed(
+        "parted --script /dev/vdc mklabel msdos",
+        "parted --script /dev/vdc -- mkpart primary 1024M -1s",
+        "zpool create rpool2 /dev/vdc1"
+    )
+
+    machine.succeed("zfs create -o mountpoint=/mnt/rpool_data rpool/data")
+
+    machine.succeed("zfs create -o mountpoint=/mnt/rpool2_data rpool2/data")
+
+    machine.succeed("zfs create -o mountpoint=/mnt/rpool_moar_data rpool/moar_data")
+
+    # Test that valid service starts successfully
+    machine.succeed("systemctl start test-service")
+
+    # Manually test our validation logic by checking the debug output
+    zfs_output = machine.succeed("zfs list -H -o name,mountpoint")
+    print("ZFS LIST OUTPUT:")
+    print(zfs_output)
+
+    dataset = machine.succeed("zfs list -H -o name,mountpoint | awk '/\\/mnt\\/rpool_data/ { print $1 }'")
+    print("DATASET FOR /mnt/rpool_data:")
+    print(dataset)
+
+    # Test that invalid-service mount service fails validation
+    machine.fail("systemctl start invalid-service.service")
+
+    # Check the journal for our detailed validation error message
+    journal_output = machine.succeed("journalctl -u invalid-service-mounts.service --no-pager")
+    print("JOURNAL OUTPUT:")
+    print(journal_output)
+
+    # Verify our validation error is in the journal using Python string matching
+    assert "ERROR: ZFS pool mismatch for /mnt/rpool_data" in journal_output
+    assert "Expected pool: rpool2" in journal_output
+    assert "Actual pool:   rpool" in journal_output
+
+
+    # Test that invalid-service mount service fails validation
+    machine.fail("systemctl start multi-service-fail.service")
+
+    # Check the journal for our detailed validation error message
+    journal_output = machine.succeed("journalctl -u multi-service-fail-mounts.service --no-pager")
+    print("JOURNAL OUTPUT:")
+    print(journal_output)
+
+    # Verify our validation error is in the journal using Python string matching
+    assert "ERROR: ZFS pool mismatch for /mnt/rpool_moar_data" in journal_output, "no zfs pool mismatch found (1)"
+    assert "Expected pool: rpool2" in journal_output, "no zfs pool mismatch found (2)"
+    assert "Actual pool:   rpool" in journal_output, "no zfs pool mismatch found (3)"
+
+
+    machine.succeed("systemctl start multi-service")
+    machine.succeed("systemctl is-active multi-service-mounts.service")
+  '';
+}

usb-secrets/setup-usb.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p parted dosfstools
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "$(realpath "$0")")"
+USB_DEVICE="$1"
+if [[ -z "${USB_DEVICE:-}" ]]; then
+    echo "Usage: $0 <usb_device>"
+    echo "Example: $0 /dev/sdb"
+    exit 1
+fi
+
+if [[ ! -b "$USB_DEVICE" ]]; then
+    echo "Error: $USB_DEVICE is not a block device"
+    exit 1
+fi
+
+if [[ ! -f "$SCRIPT_DIR/usb-secrets/usb-secrets-key" ]]; then
+    echo "Error: usb-secrets-key not found at $SCRIPT_DIR/usb-secrets/usb-secrets-key"
+    exit 1
+fi
+
+echo "WARNING: This will completely wipe $USB_DEVICE"
+echo "Press Ctrl+C to abort, or Enter to continue..."
+read
+
+echo "Creating partition and formatting as FAT32..."
+parted -s "$USB_DEVICE" mklabel msdos
+parted -s "$USB_DEVICE" mkpart primary fat32 0% 100%
+parted -s "$USB_DEVICE" set 1 boot on
+
+USB_PARTITION="${USB_DEVICE}1"
+mkfs.fat -F 32 -n "SECRETS" "$USB_PARTITION"
+
+echo "Copying key to USB..."
+MOUNT_POINT=$(mktemp -d)
+trap "umount $MOUNT_POINT 2>/dev/null || true; rmdir $MOUNT_POINT" EXIT
+
+mount "$USB_PARTITION" "$MOUNT_POINT"
+cp "$SCRIPT_DIR/usb-secrets/usb-secrets-key" "$MOUNT_POINT/"
+umount "$MOUNT_POINT"
+
+echo "USB setup complete! Label: SECRETS"
+echo "Create multiple backup USB keys for redundancy."