minecraft: update mods + add modernfix + debugify

configuration.nix

@@ -19,6 +19,7 @@
     ./modules/secureboot.nix
     ./modules/no-rgb.nix
     ./modules/security.nix
+    ./modules/ntfy-alerts.nix
 
     ./services/postgresql.nix
     ./services/jellyfin.nix
@@ -32,6 +33,14 @@
     ./services/jellyfin-qbittorrent-monitor.nix
     ./services/bitmagnet.nix
 
+    ./services/arr/prowlarr.nix
+    ./services/arr/sonarr.nix
+    ./services/arr/radarr.nix
+    ./services/arr/bazarr.nix
+    ./services/arr/jellyseerr.nix
+    ./services/arr/recyclarr.nix
+    ./services/arr/init.nix
+
     ./services/soulseek.nix
 
     ./services/ups.nix
@@ -55,6 +64,7 @@
     ./services/syncthing.nix
 
     ./services/ntfy.nix
+    ./services/ntfy-alerts.nix
   ];
 
   services.kmscon.enable = true;
@@ -123,6 +133,37 @@
       compressor = "zstd";
       supportedFilesystems = [ "f2fs" ];
     };
+
+    # BBR congestion control handles variable-latency VPN connections much
+    # better than CUBIC by probing bandwidth continuously rather than
+    # reacting to packet loss.
+    kernelModules = [ "tcp_bbr" ];
+
+    kernel.sysctl = {
+      # Use BBR + fair queuing for smooth throughput through the WireGuard VPN
+      "net.core.default_qdisc" = "fq";
+      "net.ipv4.tcp_congestion_control" = "bbr";
+
+      # Disable slow-start after idle: prevents TCP from resetting window
+      # size on each burst cycle (the primary cause of the 0 -> 40 MB/s spikes)
+      "net.ipv4.tcp_slow_start_after_idle" = 0;
+
+      # Larger socket buffers to accommodate the VPN bandwidth-delay product
+      # (22ms RTT * target throughput). Current 2.5MB max is too small.
+      "net.core.rmem_max" = 16777216;
+      "net.core.wmem_max" = 16777216;
+      "net.ipv4.tcp_rmem" = "4096 87380 16777216";
+      "net.ipv4.tcp_wmem" = "4096 65536 16777216";
+
+      # Higher backlog for the large number of concurrent torrent connections
+      "net.core.netdev_max_backlog" = 5000;
+
+      # Minecraft server optimizations
+      # Disable autogroup for better scheduling of game server threads
+      "kernel.sched_autogroup_enabled" = 0;
+      # Huge pages for Minecraft JVM (4000MB heap / 2MB per page + ~200 overhead)
+      "vm.nr_hugepages" = 2200;
+    };
   };
 
   environment.etc = {
@@ -192,6 +233,7 @@
     hostName = hostname;
     hostId = "0f712d56";
     firewall.enable = true;
+    firewall.trustedInterfaces = [ "wg-br" ];
     useDHCP = false;
     enableIPv6 = false;
 

flake.lock

@@ -25,13 +25,33 @@
         "type": "github"
       }
     },
+    "arr-init": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772249948,
+        "narHash": "sha256-v68tO12mTCET68eZG583U+OlBL4f6kAoHS9iKA/xLzQ=",
+        "ref": "refs/heads/main",
+        "rev": "d21eb9f5b0a30bb487de7c0afbbbaf19324eaa49",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      }
+    },
     "crane": {
       "locked": {
-        "lastModified": 1770419512,
-        "narHash": "sha256-o8Vcdz6B6bkiGUYkZqFwH3Pv1JwZyXht3dMtS7RchIo=",
+        "lastModified": 1771796463,
+        "narHash": "sha256-9bCDuUzpwJXcHMQYMS1yNuzYMmKO/CCwCexpjWOl62I=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "2510f2cbc3ccd237f700bb213756a8f35c32d8d7",
+        "rev": "3d3de3313e263e04894f284ac18177bd26169bad",
         "type": "github"
       },
       "original": {
@@ -69,11 +89,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771271879,
-        "narHash": "sha256-Vn32sMuvV35ChjVGZE4d8NNmCq3E/6HjaK2uVUUp2JI=",
+        "lastModified": 1771881364,
+        "narHash": "sha256-A5uE/hMium5of/QGC6JwF5TGoDAfpNtW00T0s9u/PN8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "e963ed5aea88ad0c093adde7c1c2abd4e1b48beb",
+        "rev": "a4cb7bf73f264d40560ba527f9280469f1f081c6",
         "type": "github"
       },
       "original": {
@@ -177,11 +197,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770260404,
-        "narHash": "sha256-3iVX1+7YUIt23hBx1WZsUllhbmP2EnXrV8tCRbLxHc8=",
+        "lastModified": 1772020340,
+        "narHash": "sha256-aqBl3GNpCadMoJ/hVkWTijM1Aeilc278MjM+LA3jK6g=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "0d782ee42c86b196acff08acfbf41bb7d13eed5b",
+        "rev": "36e38ca0d9afe4c55405fdf22179a5212243eecc",
         "type": "github"
       },
       "original": {
@@ -243,11 +263,11 @@
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1770734117,
-        "narHash": "sha256-PNXSnK507MRj+hYMgnUR7InNJzVCmOfsjHV4YXZgpwQ=",
+        "lastModified": 1772216104,
+        "narHash": "sha256-1TnGN26vnCEQk5m4AavJZxGZTb/6aZyphemRPRwFUfs=",
         "owner": "nix-community",
         "repo": "lanzaboote",
-        "rev": "2038a9a19adb886eccba775321b055fdbdc5029d",
+        "rev": "dbe5112de965bbbbff9f0729a9789c20a65ab047",
         "type": "github"
       },
       "original": {
@@ -265,11 +285,11 @@
         "systems": "systems_3"
       },
       "locked": {
-        "lastModified": 1771296409,
-        "narHash": "sha256-p0fEFcqNnhYBKsHTint5pwkcnQk1b68OeQJh95B9Adg=",
+        "lastModified": 1772160153,
+        "narHash": "sha256-lk5IxQzY9ZeeEyjKNT7P6dFnlRpQgkus4Ekc/+slypY=",
         "owner": "Infinidoge",
         "repo": "nix-minecraft",
-        "rev": "22cb60087e549a90f6b0347e84ac178c0c9085ad",
+        "rev": "deca3fb710b502ba10cd5cdc8f66c2cc184b92df",
         "type": "github"
       },
       "original": {
@@ -280,11 +300,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1771257191,
-        "narHash": "sha256-H1l+zHq+ZinWH7F1IidpJ2farmbfHXjaxAm1RKWE1KI=",
+        "lastModified": 1771969195,
+        "narHash": "sha256-qwcDBtrRvJbrrnv1lf/pREQi8t2hWZxVAyeMo7/E9sw=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "66e1a090ded57a0f88e2b381a7d4daf4a5722c3f",
+        "rev": "41c6b421bdc301b2624486e11905c9af7b8ec68e",
         "type": "github"
       },
       "original": {
@@ -296,11 +316,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1771208521,
-        "narHash": "sha256-X01Q3DgSpjeBpapoGA4rzKOn25qdKxbPnxHeMLNoHTU=",
+        "lastModified": 1772047000,
+        "narHash": "sha256-7DaQVv4R97cii/Qdfy4tmDZMB2xxtyIvNGSwXBBhSmo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "fa56d7d6de78f5a7f997b0ea2bc6efd5868ad9e8",
+        "rev": "1267bb4920d0fc06ea916734c11b0bf004bbe17e",
         "type": "github"
       },
       "original": {
@@ -336,11 +356,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1769939035,
-        "narHash": "sha256-Fok2AmefgVA0+eprw2NDwqKkPGEI5wvR+twiZagBvrg=",
+        "lastModified": 1771858127,
+        "narHash": "sha256-Gtre9YoYl3n25tJH2AoSdjuwcqij5CPxL3U3xysYD08=",
         "owner": "cachix",
         "repo": "pre-commit-hooks.nix",
-        "rev": "a8ca480175326551d6c4121498316261cbb5b260",
+        "rev": "49bbbfc218bf3856dfa631cead3b052d78248b83",
         "type": "github"
       },
       "original": {
@@ -352,6 +372,7 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "arr-init": "arr-init",
         "deploy-rs": "deploy-rs",
         "disko": "disko",
         "home-manager": "home-manager",
@@ -376,11 +397,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770520253,
-        "narHash": "sha256-6rWuHgSENXKnC6HGGAdRolQrnp/8IzscDn7FQEo1uEQ=",
+        "lastModified": 1771988922,
+        "narHash": "sha256-Fc6FHXtfEkLtuVJzd0B6tFYMhmcPLuxr90rWfb/2jtQ=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ebb8a141f60bb0ec33836333e0ca7928a072217f",
+        "rev": "f4443dc3f0b6c5e6b77d923156943ce816d1fcb9",
         "type": "github"
       },
       "original": {
@@ -413,11 +434,11 @@
     "senior_project-website": {
       "flake": false,
       "locked": {
-        "lastModified": 1769471280,
-        "narHash": "sha256-6BADVRSHHwO3NcAua44hagAJTqPNDxEhPjBMehURiHQ=",
+        "lastModified": 1771869552,
+        "narHash": "sha256-veaVrRWCSy7HYAAjUFLw8HASKcj+3f0W+sCwS3QiaM4=",
         "owner": "Titaniumtown",
         "repo": "senior-project-website",
-        "rev": "d6f443ede6c90a049085b4598e438849e19e74f4",
+        "rev": "28a2b93492dac877dce0b38f078eacf74fce26e7",
         "type": "github"
       },
       "original": {
@@ -433,11 +454,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771207491,
-        "narHash": "sha256-08s9LKq9Et4y9r6FSJLJUnRCyJHZMauAIok45ulQo0k=",
+        "lastModified": 1772071250,
+        "narHash": "sha256-LDWvJDR1J8xE8TBJjzWnOA0oVP/l9xBFC4npQPJDHN4=",
         "owner": "nix-community",
         "repo": "srvos",
-        "rev": "434ed3900e9a7b23638da97ebe16ab0e0be7fef5",
+        "rev": "5cd73bcf984b72d8046e1175d13753de255adfb9",
         "type": "github"
       },
       "original": {
@@ -509,11 +530,11 @@
     "trackerlist": {
       "flake": false,
       "locked": {
-        "lastModified": 1771283390,
-        "narHash": "sha256-rkSYYntpKP/OD1vXWw/W+GGRBSaC5OoHLR/yqJhlq/M=",
+        "lastModified": 1772233783,
+        "narHash": "sha256-2jPUBKpPuT4dCXwVFuZvTH3QyURixsfJZD7Zqs0atPY=",
         "owner": "ngosang",
         "repo": "trackerslist",
-        "rev": "a7c87dd33cacc627b67447bdef591bd9a8b7d878",
+        "rev": "85c4f103f130b070a192343c334f50c2f56b61a9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -68,6 +68,11 @@
     ytbn-graphing-software = {
       url = "git+https://git.gardling.com/titaniumtown/YTBN-Graphing-Software";
     };
+
+    arr-init = {
+      url = "git+ssh://gitea@git.gardling.com/titaniumtown/arr-init";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =
@@ -83,7 +88,7 @@
       srvos,
       deploy-rs,
       impermanence,
-      agenix,
+      arr-init,
       ...
     }@inputs:
     let
@@ -125,6 +130,11 @@
           ntfy = 2586;
           livekit = 7880;
           lk_jwt = 8081;
+          prowlarr = 9696;
+          sonarr = 8989;
+          radarr = 7878;
+          bazarr = 6767;
+          jellyseerr = 5055;
         };
 
         https = {
@@ -193,6 +203,35 @@
           signalBackupDir = "/${zpool_ssds}/bak/signal";
           grayjayBackupDir = "/${zpool_ssds}/bak/grayjay";
         };
+
+        prowlarr = {
+          dataDir = services_dir + "/prowlarr";
+        };
+
+        sonarr = {
+          dataDir = services_dir + "/sonarr";
+        };
+
+        radarr = {
+          dataDir = services_dir + "/radarr";
+        };
+
+        bazarr = {
+          dataDir = services_dir + "/bazarr";
+        };
+
+        jellyseerr = {
+          configDir = services_dir + "/jellyseerr";
+        };
+
+        recyclarr = {
+          dataDir = services_dir + "/recyclarr";
+        };
+
+        media = {
+          moviesDir = torrents_path + "/media/movies";
+          tvDir = torrents_path + "/media/tv";
+        };
       };
 
       pkgs = import nixpkgs {
@@ -201,6 +240,10 @@
         buildPlatform = builtins.currentSystem;
       };
       lib = import ./modules/lib.nix { inherit inputs pkgs service_configs; };
+      testSuite = import ./tests/tests.nix {
+        inherit pkgs lib inputs;
+        config = self.nixosConfigurations.muffin.config;
+      };
     in
     {
       formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-tree;
@@ -257,6 +300,8 @@
 
           lanzaboote.nixosModules.lanzaboote
 
+          arr-init.nixosModules.default
+
           home-manager.nixosModules.home-manager
           (
             {
@@ -285,24 +330,19 @@
         };
       };
 
-      packages.${system} =
-        let
-          testSuite = import ./tests/tests.nix {
-            inherit pkgs lib inputs;
-            config = self.nixosConfigurations.muffin.config;
-          };
-        in
-        {
-          tests = pkgs.linkFarm "all-tests" (
-            pkgs.lib.mapAttrsToList (name: test: {
-              name = name;
-              path = test;
-            }) testSuite
-          );
-        }
-        // (pkgs.lib.mapAttrs' (name: test: {
-          name = "test-${name}";
-          value = test;
-        }) testSuite);
+      checks.${system} = testSuite;
+
+      packages.${system} = {
+        tests = pkgs.linkFarm "all-tests" (
+          pkgs.lib.mapAttrsToList (name: test: {
+            name = name;
+            path = test;
+          }) testSuite
+        );
+      }
+      // (pkgs.lib.mapAttrs' (name: test: {
+        name = "test-${name}";
+        value = test;
+      }) testSuite);
     };
 }

modules/age-secrets.nix

@@ -65,5 +65,20 @@
       owner = "root";
       group = "root";
     };
+
+    # ntfy-alerts secrets
+    ntfy-alerts-topic = {
+      file = ../secrets/ntfy-alerts-topic.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    ntfy-alerts-token = {
+      file = ../secrets/ntfy-alerts-token.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
   };
 }

modules/ntfy-alerts.nix

@@ -0,0 +1,132 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.ntfyAlerts;
+
+  curl = "${pkgs.curl}/bin/curl";
+  hostname = config.networking.hostName;
+
+  # Build the curl auth args as a proper bash array fragment
+  authCurlArgs =
+    if cfg.tokenFile != null then
+      ''
+        if [ -f "${cfg.tokenFile}" ]; then
+          TOKEN=$(cat "${cfg.tokenFile}" 2>/dev/null || echo "")
+          if [ -n "$TOKEN" ]; then
+            AUTH_ARGS=(-H "Authorization: Bearer $TOKEN")
+          fi
+        fi
+      ''
+    else
+      "";
+
+  # Systemd failure alert script
+  systemdAlertScript = pkgs.writeShellScript "ntfy-systemd-alert" ''
+    set -euo pipefail
+
+    UNIT_NAME="$1"
+    SERVER_URL="${cfg.serverUrl}"
+    TOPIC=$(cat "${cfg.topicFile}" 2>/dev/null | tr -d '[:space:]')
+    if [ -z "$TOPIC" ]; then
+      echo "ERROR: Could not read topic from ${cfg.topicFile}"
+      exit 1
+    fi
+
+    # Get journal output for context
+    JOURNAL_OUTPUT=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 15 --no-pager 2>/dev/null || echo "No journal output available")
+
+    # Build auth args
+    AUTH_ARGS=()
+    ${authCurlArgs}
+
+    # Send notification
+    ${curl} -sf --max-time 15 -X POST \
+      "$SERVER_URL/$TOPIC" \
+      -H "Title: [${hostname}] Service failed: $UNIT_NAME" \
+      -H "Priority: high" \
+      -H "Tags: warning" \
+      "''${AUTH_ARGS[@]}" \
+      -d "$JOURNAL_OUTPUT" || true
+  '';
+
+in
+{
+  options.services.ntfyAlerts = {
+    enable = lib.mkEnableOption "ntfy push notifications for system alerts";
+
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "The ntfy server URL (e.g. https://ntfy.example.com)";
+      example = "https://ntfy.example.com";
+    };
+
+    topicFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to a file containing the ntfy topic name to publish alerts to.";
+      example = "/run/agenix/ntfy-alerts-topic";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Path to a file containing the ntfy auth token.
+        If set, uses Authorization: Bearer header for authentication.
+      '';
+      example = "/run/secrets/ntfy-token";
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    # Per-service OnFailure for monitored services
+    systemd.services = {
+      "ntfy-alert@" = {
+        description = "Send ntfy notification for failed service %i";
+
+        unitConfig.OnFailure = lib.mkForce "";
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${systemdAlertScript} %i";
+          TimeoutSec = 30;
+        };
+      };
+
+      # TODO: sanoid's ExecStartPre runs `zfs allow` which blocks on TXG sync;
+      # on the hdds pool (slow spinning disks + large async frees) this causes
+      # 30+ minute hangs and guaranteed timeouts. Suppress until we fix sanoid
+      # to run as root without `zfs allow`. See: nixpkgs#72060, openzfs/zfs#14180
+      "sanoid".unitConfig.OnFailure = lib.mkForce "";
+    };
+
+    # Global OnFailure drop-in for all services
+    systemd.packages = [
+      (pkgs.writeTextDir "etc/systemd/system/service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=ntfy-alert@%p.service
+      '')
+
+      # Sanoid-specific drop-in to override the global OnFailure (see TODO above)
+      (pkgs.writeTextDir "etc/systemd/system/sanoid.service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=
+      '')
+    ];
+    # ZED (ZFS Event Daemon) ntfy notification settings
+    services.zfs.zed = {
+      enableMail = false;
+      settings = {
+        ZED_NTFY_URL = cfg.serverUrl;
+        ZED_NTFY_TOPIC = "$(cat ${cfg.topicFile} | tr -d '[:space:]')";
+        ZED_NTFY_ACCESS_TOKEN = lib.mkIf (cfg.tokenFile != null) "$(cat ${cfg.tokenFile})";
+        ZED_NOTIFY_VERBOSE = true;
+      };
+    };
+
+  };
+}

modules/zfs.nix

@@ -64,7 +64,7 @@ in
       yearly = 0;
     };
 
-    datasets."${service_configs.zpool_ssds}/services/jellyfin_cache" = {
+    datasets."${service_configs.zpool_ssds}/services/jellyfin/cache" = {
       recursive = true;
       autoprune = true;
       autosnap = true;

services/arr/bazarr.nix

@@ -0,0 +1,34 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "bazarr" service_configs.zpool_ssds [
+      service_configs.bazarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "bazarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "bazarr" [
+      "Z ${service_configs.bazarr.dataDir} 0700 ${config.services.bazarr.user} ${config.services.bazarr.group}"
+    ])
+  ];
+
+  services.bazarr = {
+    enable = true;
+    listenPort = service_configs.ports.bazarr;
+  };
+
+  services.caddy.virtualHosts."bazarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.bazarr}
+  '';
+
+  users.users.${config.services.bazarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/arr/init.nix

@@ -0,0 +1,115 @@
+{ config, service_configs, ... }:
+{
+  services.arrInit = {
+    prowlarr = {
+      enable = true;
+      serviceName = "prowlarr";
+      port = service_configs.ports.prowlarr;
+      dataDir = service_configs.prowlarr.dataDir;
+      apiVersion = "v1";
+      networkNamespacePath = "/run/netns/wg";
+      syncedApps = [
+        {
+          name = "Sonarr";
+          implementation = "Sonarr";
+          configContract = "SonarrSettings";
+          prowlarrUrl = "http://localhost:${builtins.toString service_configs.ports.prowlarr}";
+          baseUrl = "http://${config.vpnNamespaces.wg.bridgeAddress}:${builtins.toString service_configs.ports.sonarr}";
+          apiKeyFrom = "${service_configs.sonarr.dataDir}/config.xml";
+          syncCategories = [
+            5000
+            5010
+            5020
+            5030
+            5040
+            5045
+            5050
+            5090
+          ];
+          serviceName = "sonarr";
+        }
+        {
+          name = "Radarr";
+          implementation = "Radarr";
+          configContract = "RadarrSettings";
+          prowlarrUrl = "http://localhost:${builtins.toString service_configs.ports.prowlarr}";
+          baseUrl = "http://${config.vpnNamespaces.wg.bridgeAddress}:${builtins.toString service_configs.ports.radarr}";
+          apiKeyFrom = "${service_configs.radarr.dataDir}/config.xml";
+          syncCategories = [
+            2000
+            2010
+            2020
+            2030
+            2040
+            2045
+            2050
+            2060
+            2070
+            2080
+          ];
+          serviceName = "radarr";
+        }
+      ];
+    };
+
+    sonarr = {
+      enable = true;
+      serviceName = "sonarr";
+      port = service_configs.ports.sonarr;
+      dataDir = service_configs.sonarr.dataDir;
+      rootFolders = [ service_configs.media.tvDir ];
+      downloadClients = [
+        {
+          name = "qBittorrent";
+          implementation = "QBittorrent";
+          configContract = "QBittorrentSettings";
+          fields = {
+            host = config.vpnNamespaces.wg.namespaceAddress;
+            port = service_configs.ports.torrent;
+            useSsl = false;
+            tvCategory = "tvshows";
+          };
+        }
+      ];
+    };
+
+    radarr = {
+      enable = true;
+      serviceName = "radarr";
+      port = service_configs.ports.radarr;
+      dataDir = service_configs.radarr.dataDir;
+      rootFolders = [ service_configs.media.moviesDir ];
+      downloadClients = [
+        {
+          name = "qBittorrent";
+          implementation = "QBittorrent";
+          configContract = "QBittorrentSettings";
+          fields = {
+            host = config.vpnNamespaces.wg.namespaceAddress;
+            port = service_configs.ports.torrent;
+            useSsl = false;
+            movieCategory = "movies";
+          };
+        }
+      ];
+    };
+  };
+
+  services.bazarrInit = {
+    enable = true;
+    dataDir = "/var/lib/bazarr";
+    port = service_configs.ports.bazarr;
+    sonarr = {
+      enable = true;
+      dataDir = service_configs.sonarr.dataDir;
+      port = service_configs.ports.sonarr;
+      serviceName = "sonarr";
+    };
+    radarr = {
+      enable = true;
+      dataDir = service_configs.radarr.dataDir;
+      port = service_configs.ports.radarr;
+      serviceName = "radarr";
+    };
+  };
+}

services/arr/jellyseerr.nix

@@ -0,0 +1,43 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "jellyseerr" service_configs.zpool_ssds [
+      service_configs.jellyseerr.configDir
+    ])
+    (lib.serviceFilePerms "jellyseerr" [
+      "Z ${service_configs.jellyseerr.configDir} 0700 jellyseerr jellyseerr"
+    ])
+  ];
+
+  services.jellyseerr = {
+    enable = true;
+    port = service_configs.ports.jellyseerr;
+    configDir = service_configs.jellyseerr.configDir;
+  };
+
+  systemd.services.jellyseerr.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "jellyseerr";
+    Group = "jellyseerr";
+    ReadWritePaths = [ service_configs.jellyseerr.configDir ];
+  };
+
+  users.users.jellyseerr = {
+    isSystemUser = true;
+    group = "jellyseerr";
+    home = service_configs.jellyseerr.configDir;
+  };
+
+  users.groups.jellyseerr = { };
+
+  services.caddy.virtualHosts."jellyseerr.${service_configs.https.domain}".extraConfig = ''
+    # import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.jellyseerr}
+  '';
+}

services/arr/prowlarr.nix

@@ -0,0 +1,30 @@
+{
+  pkgs,
+  service_configs,
+  config,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "prowlarr" service_configs.zpool_ssds [
+      service_configs.prowlarr.dataDir
+    ])
+    (lib.vpnNamespaceOpenPort service_configs.ports.prowlarr "prowlarr")
+  ];
+
+  services.prowlarr = {
+    enable = true;
+    dataDir = service_configs.prowlarr.dataDir;
+    settings.server.port = service_configs.ports.prowlarr;
+  };
+
+  systemd.services.prowlarr.serviceConfig = {
+    ExecStartPre = "+${pkgs.coreutils}/bin/chown -R prowlarr /var/lib/prowlarr";
+  };
+
+  services.caddy.virtualHosts."prowlarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy ${config.vpnNamespaces.wg.namespaceAddress}:${builtins.toString service_configs.ports.prowlarr}
+  '';
+}

services/arr/radarr.nix

@@ -0,0 +1,36 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "radarr" service_configs.zpool_ssds [
+      service_configs.radarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "radarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "radarr" [
+      "Z ${service_configs.radarr.dataDir} 0700 ${config.services.radarr.user} ${config.services.radarr.group}"
+    ])
+  ];
+
+  services.radarr = {
+    enable = true;
+    dataDir = service_configs.radarr.dataDir;
+    settings.server.port = service_configs.ports.radarr;
+    settings.update.mechanism = "external";
+  };
+
+  services.caddy.virtualHosts."radarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.radarr}
+  '';
+
+  users.users.${config.services.radarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/arr/recyclarr.nix

@@ -0,0 +1,202 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+let
+  radarrConfig = "${service_configs.radarr.dataDir}/config.xml";
+  sonarrConfig = "${service_configs.sonarr.dataDir}/config.xml";
+  appDataDir = "${service_configs.recyclarr.dataDir}/data";
+
+  # Runs as root (via + prefix) to read API keys, writes secrets.yml for recyclarr
+  generateSecrets = pkgs.writeShellScript "recyclarr-generate-secrets" ''
+    RADARR_KEY=$(${pkgs.gnugrep}/bin/grep -oP '(?<=<ApiKey>)[^<]+' ${radarrConfig})
+    SONARR_KEY=$(${pkgs.gnugrep}/bin/grep -oP '(?<=<ApiKey>)[^<]+' ${sonarrConfig})
+    cat > ${appDataDir}/secrets.yml <<EOF
+    movies_api_key: $RADARR_KEY
+    series_api_key: $SONARR_KEY
+    EOF
+    chown recyclarr:recyclarr ${appDataDir}/secrets.yml
+    chmod 600 ${appDataDir}/secrets.yml
+  '';
+in
+{
+  imports = [
+    (lib.serviceMountWithZpool "recyclarr" service_configs.zpool_ssds [
+      service_configs.recyclarr.dataDir
+    ])
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d ${service_configs.recyclarr.dataDir} 0755 recyclarr recyclarr -"
+    "d ${appDataDir} 0755 recyclarr recyclarr -"
+  ];
+
+  services.recyclarr = {
+    enable = true;
+    command = "sync";
+    schedule = "daily";
+    user = "recyclarr";
+    group = "recyclarr";
+
+    configuration = {
+      radarr.movies = {
+        base_url = "http://localhost:${builtins.toString service_configs.ports.radarr}";
+
+        include = [
+          { template = "radarr-quality-definition-movie"; }
+          { template = "radarr-quality-profile-remux-web-2160p"; }
+          { template = "radarr-custom-formats-remux-web-2160p"; }
+        ];
+
+        quality_profiles = [
+          {
+            name = "Remux + WEB 2160p";
+            upgrade = {
+              allowed = true;
+              until_quality = "Remux-2160p";
+            };
+            qualities = [
+              { name = "Remux-2160p"; }
+              {
+                name = "WEB 2160p";
+                qualities = [
+                  "WEBDL-2160p"
+                  "WEBRip-2160p"
+                ];
+              }
+              { name = "Remux-1080p"; }
+              { name = "Bluray-1080p"; }
+              {
+                name = "WEB 1080p";
+                qualities = [
+                  "WEBDL-1080p"
+                  "WEBRip-1080p"
+                ];
+              }
+              { name = "HDTV-1080p"; }
+            ];
+          }
+        ];
+
+        custom_formats = [
+          # Upscaled
+          {
+            trash_ids = [ "bfd8eb01832d646a0a89c4deb46f8564" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = -10000;
+              }
+            ];
+          }
+          # x265 (HD) - override template -10000 penalty
+          {
+            trash_ids = [ "dc98083864ea246d05a42df0d05f81cc" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = 0;
+              }
+            ];
+          }
+          # x265 (no HDR/DV) - override template -10000 penalty
+          {
+            trash_ids = [ "839bea857ed2c0a8e084f3cbdbd65ecb" ];
+            assign_scores_to = [
+              {
+                name = "Remux + WEB 2160p";
+                score = 0;
+              }
+            ];
+          }
+        ];
+      };
+
+      sonarr.series = {
+        base_url = "http://localhost:${builtins.toString service_configs.ports.sonarr}";
+
+        include = [
+          { template = "sonarr-quality-definition-series"; }
+          { template = "sonarr-v4-quality-profile-web-2160p"; }
+          { template = "sonarr-v4-custom-formats-web-2160p"; }
+        ];
+
+        quality_profiles = [
+          {
+            name = "WEB-2160p";
+            upgrade = {
+              allowed = true;
+              until_quality = "WEB 2160p";
+            };
+            qualities = [
+              {
+                name = "WEB 2160p";
+                qualities = [
+                  "WEBDL-2160p"
+                  "WEBRip-2160p"
+                ];
+              }
+              { name = "Bluray-1080p Remux"; }
+              { name = "Bluray-1080p"; }
+              {
+                name = "WEB 1080p";
+                qualities = [
+                  "WEBDL-1080p"
+                  "WEBRip-1080p"
+                ];
+              }
+              { name = "HDTV-1080p"; }
+            ];
+          }
+        ];
+
+        custom_formats = [
+          # Upscaled
+          {
+            trash_ids = [ "23297a736ca77c0fc8e70f8edd7ee56c" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = -10000;
+              }
+            ];
+          }
+          # x265 (HD) - override template -10000 penalty
+          {
+            trash_ids = [ "47435ece6b99a0b477caf360e79ba0bb" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = 0;
+              }
+            ];
+          }
+          # x265 (no HDR/DV) - override template -10000 penalty
+          {
+            trash_ids = [ "9b64dff695c2115facf1b6ea59c9bd07" ];
+            assign_scores_to = [
+              {
+                name = "WEB-2160p";
+                score = 0;
+              }
+            ];
+          }
+        ];
+      };
+    };
+  };
+
+  # Add secrets generation before recyclarr runs
+  systemd.services.recyclarr = {
+    after = [
+      "network-online.target"
+      "radarr.service"
+      "sonarr.service"
+    ];
+    wants = [ "network-online.target" ];
+    serviceConfig.ExecStartPre = "+${generateSecrets}";
+  };
+}

services/arr/sonarr.nix

@@ -0,0 +1,42 @@
+{
+  pkgs,
+  config,
+  service_configs,
+  lib,
+  ...
+}:
+{
+  imports = [
+    (lib.serviceMountWithZpool "sonarr" service_configs.zpool_ssds [
+      service_configs.sonarr.dataDir
+    ])
+    (lib.serviceMountWithZpool "sonarr" service_configs.zpool_hdds [
+      service_configs.torrents_path
+    ])
+    (lib.serviceFilePerms "sonarr" [
+      "Z ${service_configs.sonarr.dataDir} 0700 ${config.services.sonarr.user} ${config.services.sonarr.group}"
+    ])
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d /torrents/media 2775 root ${service_configs.media_group} -"
+    "d ${service_configs.media.tvDir} 2775 root ${service_configs.media_group} -"
+    "d ${service_configs.media.moviesDir} 2775 root ${service_configs.media_group} -"
+  ];
+
+  services.sonarr = {
+    enable = true;
+    dataDir = service_configs.sonarr.dataDir;
+    settings.server.port = service_configs.ports.sonarr;
+    settings.update.mechanism = "external";
+  };
+
+  services.caddy.virtualHosts."sonarr.${service_configs.https.domain}".extraConfig = ''
+    import ${config.age.secrets.caddy_auth.path}
+    reverse_proxy :${builtins.toString service_configs.ports.sonarr}
+  '';
+
+  users.users.${config.services.sonarr.user}.extraGroups = [
+    service_configs.media_group
+  ];
+}

services/matrix.nix

@@ -12,8 +12,8 @@ let
         domain = "forgejo.ellis.link";
         owner = "continuwuation";
         repo = "continuwuity";
-        rev = "082c44f3556e4e939c31cb66dda261af4f70bea8";
-        hash = "sha256-v7W6ZqSYB2TSkRj6Hte/UxBTCad94b+uzpROQ9jlwdQ=";
+        rev = "052c4dfa2165fdc4839fed95b71446120273cf23";
+        hash = "sha256-kQV4glRrKczoJpn9QIMgB5ac+saZQjSZPel+9K9Ykcs=";
       };
     in
     pkgs.matrix-continuwuity.overrideAttrs (old: {
@@ -21,7 +21,7 @@ let
       cargoDeps = pkgs.rustPlatform.fetchCargoVendor {
         inherit src;
         name = "${old.pname}-vendor";
-        hash = "sha256-Ib4yAT0Ncch8QT8CioF9s3fN34E50ZhbcX7m0lgwJkI=";
+        hash = "sha256-vlOXQL8wwEGFX+w0G/eIeHW3J1UDzhJ501kYhAghDV8=";
       };
 
       patches = (old.patches or [ ]) ++ [

services/minecraft.nix

@@ -35,7 +35,38 @@
         let
           heap_size = "4000M";
         in
-        "-Xmx${heap_size} -Xms${heap_size} -XX:+UseZGC -XX:+ZGenerational";
+        lib.concatStringsSep " " [
+          # Memory
+          "-Xmx${heap_size}"
+          "-Xms${heap_size}"
+          # GC
+          "-XX:+UseZGC"
+          "-XX:+ZGenerational"
+          # Base JVM optimizations (brucethemoose/Minecraft-Performance-Flags-Benchmarks)
+          "-XX:+UnlockExperimentalVMOptions"
+          "-XX:+UnlockDiagnosticVMOptions"
+          "-XX:+AlwaysActAsServerClassMachine"
+          "-XX:+AlwaysPreTouch"
+          "-XX:+DisableExplicitGC"
+          "-XX:+UseNUMA"
+          "-XX:+PerfDisableSharedMem"
+          "-XX:+UseFastUnorderedTimeStamps"
+          "-XX:+UseCriticalJavaThreadPriority"
+          "-XX:ThreadPriorityPolicy=1"
+          "-XX:AllocatePrefetchStyle=3"
+          "-XX:-DontCompileHugeMethods"
+          "-XX:MaxNodeLimit=240000"
+          "-XX:NodeLimitFudgeFactor=8000"
+          "-XX:ReservedCodeCacheSize=400M"
+          "-XX:NonNMethodCodeHeapSize=12M"
+          "-XX:ProfiledCodeHeapSize=194M"
+          "-XX:NonProfiledCodeHeapSize=194M"
+          "-XX:NmethodSweepActivity=1"
+          "-XX:+UseVectorCmov"
+          # Large pages (requires vm.nr_hugepages sysctl)
+          "-XX:+UseLargePages"
+          "-XX:LargePageSizeInBytes=2m"
+        ];
 
       serverProperties = {
         server-port = service_configs.ports.minecraft;
@@ -87,8 +118,8 @@
             };
 
             c2me = fetchurl {
-              url = "https://cdn.modrinth.com/data/VSNURh3q/versions/DLKF3HZk/c2me-fabric-mc1.21.11-0.3.6%2Bbeta.1.0.jar";
-              sha512 = "d4f983aeb5083033b525522e623a9a9ba86b6fc9c83db008cc0575d0077e736ac9bee0b6b0e03b8d1c89ae27a4e5cdc269041f61eb0d1a10757de4c30b065467";
+              url = "https://cdn.modrinth.com/data/VSNURh3q/versions/QdLiMUjx/c2me-fabric-mc1.21.11-0.3.7%2Balpha.0.7.jar";
+              sha512 = "f9543febe2d649a82acd6d5b66189b6a3d820cf24aa503ba493fdb3bbd4e52e30912c4c763fe50006f9a46947ae8cd737d420838c61b93429542573ed67f958e";
             };
 
             krypton = fetchurl {
@@ -110,12 +141,32 @@
               url = "https://cdn.modrinth.com/data/c7m1mi73/versions/CUh1DWeO/packetfixer-fabric-3.3.4-1.21.11.jar";
               sha512 = "33331b16cb40c5e6fbaade3cacc26f3a0e8fa5805a7186f94d7366a0e14dbeee9de2d2e8c76fa71f5e9dd24eb1c261667c35447e32570ea965ca0f154fdfba0a";
             };
+
+            # fork of Modernfix for 1.21.11 (upstream will support 26.1)
+            modernfix = fetchurl {
+              url = "https://cdn.modrinth.com/data/TjSm1wrD/versions/JwSO8JCN/modernfix-5.25.2-build.4.jar";
+              sha512 = "0d65c05ac0475408c58ef54215714e6301113101bf98bfe4bb2ba949fbfddd98225ac4e2093a5f9206a9e01ba80a931424b237bdfa3b6e178c741ca6f7f8c6a3";
+            };
+
+            debugify = fetchurl {
+              url = "https://cdn.modrinth.com/data/QwxR6Gcd/versions/8Q49lnaU/debugify-1.21.11%2B1.0.jar";
+              sha512 = "04d82dd33f44ced37045f1f9a54ad4eacd70861ff74a8800f2d2df358579e6cb0ea86a34b0086b3e87026b1a0691dd6594b4fdc49f89106466eea840518beb03";
+            };
+
           }
         );
       };
     };
   };
 
+  systemd.services.minecraft-server-main = {
+    serviceConfig = {
+      Nice = -5;
+      IOSchedulingPriority = 0;
+      LimitMEMLOCK = "infinity"; # Required for large pages
+    };
+  };
+
   services.caddy.virtualHosts = lib.mkIf (config.services.caddy.enable) {
     "map.${service_configs.https.domain}".extraConfig = ''
       root * ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web

services/monero.nix

@@ -20,5 +20,4 @@
       restricted = true;
     };
   };
-
 }

services/ntfy-alerts.nix

@@ -0,0 +1,10 @@
+{ config, service_configs, ... }:
+{
+  services.ntfyAlerts = {
+    enable = true;
+    serverUrl = "https://${service_configs.ntfy.domain}";
+    topicFile = config.age.secrets.ntfy-alerts-topic.path;
+
+    tokenFile = config.age.secrets.ntfy-alerts-token.path;
+  };
+}

services/qbittorrent.nix

@@ -18,7 +18,9 @@
     ])
     (lib.vpnNamespaceOpenPort config.services.qbittorrent.webuiPort "qbittorrent")
     (lib.serviceFilePerms "qbittorrent" [
-      "Z ${config.services.qbittorrent.serverConfig.Preferences.Downloads.SavePath} 0750 ${config.services.qbittorrent.user} ${service_configs.media_group}"
+      # 0770: group (media) needs write to delete files during upgrades —
+      # Radarr/Sonarr must unlink the old file before placing the new one.
+      "Z ${config.services.qbittorrent.serverConfig.Preferences.Downloads.SavePath} 0770 ${config.services.qbittorrent.user} ${service_configs.media_group}"
       "Z ${config.services.qbittorrent.serverConfig.Preferences.Downloads.TempPath} 0700 ${config.services.qbittorrent.user} ${config.services.qbittorrent.group}"
       "Z ${config.services.qbittorrent.profileDir} 0700 ${config.services.qbittorrent.user} ${config.services.qbittorrent.group}"
     ])
@@ -28,6 +30,11 @@
     enable = true;
     webuiPort = service_configs.ports.torrent;
     profileDir = "/var/lib/qBittorrent";
+    # Set the service group to 'media' so the systemd unit runs with media as
+    # the primary GID. Linux assigns new file ownership from the process's GID
+    # (set by systemd's Group= directive), not from /etc/passwd. Without this,
+    # downloads land as qbittorrent:qbittorrent (0700), blocking Radarr/Sonarr.
+    group = service_configs.media_group;
 
     serverConfig.LegalNotice.Accepted = true;
 
@@ -48,7 +55,7 @@
 
     serverConfig.BitTorrent = {
       Session = {
-        MaxConnectionsPerTorrent = 10;
+        MaxConnectionsPerTorrent = 50;
         MaxUploadsPerTorrent = 10;
         MaxConnections = -1;
         MaxUploads = -1;
@@ -56,9 +63,10 @@
         MaxActiveCheckingTorrents = 5;
 
         # queueing
-        QueueingSystemEnabled = false;
-        MaxActiveDownloads = 2; # num of torrents that can download at the same time
-        MaxActiveUploads = 20;
+        QueueingSystemEnabled = true;
+        MaxActiveDownloads = 5; # keep focused: fewer torrents, each gets more bandwidth
+        MaxActiveUploads = -1;
+        MaxActiveTorrents = -1;
         IgnoreSlowTorrentsForQueueing = true;
 
         GlobalUPSpeedLimit = 0;
@@ -83,12 +91,17 @@
         inherit (config.services.qbittorrent.serverConfig.Preferences.Downloads) TempPath;
         TempPathEnabled = true;
 
-        # how many connections per sec
-        ConnectionSpeed = 300;
+        ConnectionSpeed = 200;
+
+        # Automatic Torrent Management: use category save paths for new torrents
+        DisableAutoTMMByDefault = false;
+        DisableAutoTMMTriggers.CategorySavePathChanged = false;
+        DisableAutoTMMTriggers.DefaultSavePathChanged = false;
 
         ChokingAlgorithm = "RateBased";
         PieceExtentAffinity = true;
         SuggestMode = true;
+        CoalesceReadWrite = true;
       };
 
       Network = {

tests/minecraft.nix

@@ -60,6 +60,10 @@ testPkgs.testers.runNixOSTest {
         wants = lib.mkForce [ ];
         after = lib.mkForce [ ];
         requires = lib.mkForce [ ];
+        serviceConfig = {
+          Nice = lib.mkForce 0;
+          LimitMEMLOCK = lib.mkForce "infinity";
+        };
       };
 
       # Test-specific overrides only - reduce memory for testing

tests/ntfy-alerts.nix

@@ -0,0 +1,174 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
+in
+testPkgs.testers.runNixOSTest {
+  name = "ntfy-alerts";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      imports = [
+        ../modules/ntfy-alerts.nix
+      ];
+
+      system.stateVersion = config.system.stateVersion;
+
+      virtualisation.memorySize = 2048;
+
+      environment.systemPackages = with pkgs; [
+        curl
+        jq
+      ];
+
+      # Create test topic file
+      systemd.tmpfiles.rules = [
+        "f /run/ntfy-test-topic 0644 root root - test-alerts"
+      ];
+
+      # Mock ntfy server that records POST requests
+      systemd.services.mock-ntfy =
+        let
+          mockNtfyScript = pkgs.writeScript "mock-ntfy.py" ''
+            import json
+            import os
+            from http.server import HTTPServer, BaseHTTPRequestHandler
+            from datetime import datetime
+
+            REQUESTS_FILE = "/tmp/ntfy-requests.json"
+
+            class MockNtfy(BaseHTTPRequestHandler):
+                def _respond(self, code=200, body=b"Ok"):
+                    self.send_response(code)
+                    self.send_header("Content-Type", "application/json")
+                    self.end_headers()
+                    self.wfile.write(body if isinstance(body, bytes) else body.encode())
+
+                def do_GET(self):
+                    self._respond()
+
+                def do_POST(self):
+                    content_length = int(self.headers.get("Content-Length", 0))
+                    body = self.rfile.read(content_length).decode() if content_length > 0 else ""
+
+                    request_data = {
+                        "timestamp": datetime.now().isoformat(),
+                        "path": self.path,
+                        "headers": dict(self.headers),
+                        "body": body,
+                    }
+
+                    # Load existing requests or start new list
+                    requests = []
+                    if os.path.exists(REQUESTS_FILE):
+                        try:
+                            with open(REQUESTS_FILE, "r") as f:
+                                requests = json.load(f)
+                        except:
+                            requests = []
+
+                    requests.append(request_data)
+
+                    with open(REQUESTS_FILE, "w") as f:
+                        json.dump(requests, f, indent=2)
+
+                    self._respond()
+
+                def log_message(self, format, *args):
+                    pass
+
+            HTTPServer(("0.0.0.0", 8080), MockNtfy).serve_forever()
+          '';
+        in
+        {
+          description = "Mock ntfy server";
+          wantedBy = [ "multi-user.target" ];
+          before = [ "ntfy-alert@test-fail.service" ];
+          serviceConfig = {
+            ExecStart = "${pkgs.python3}/bin/python3 ${mockNtfyScript}";
+            Type = "simple";
+          };
+        };
+
+      # Test service that will fail
+      systemd.services.test-fail = {
+        description = "Test service that fails";
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.coreutils}/bin/false";
+        };
+      };
+
+      # Configure ntfy-alerts to use mock server
+      services.ntfyAlerts = {
+        enable = true;
+        serverUrl = "http://localhost:8080";
+        topicFile = "/run/ntfy-test-topic";
+
+      };
+    };
+
+  testScript = ''
+    import json
+    import time
+
+    start_all()
+
+    # Wait for mock ntfy server to be ready
+    machine.wait_for_unit("mock-ntfy.service")
+    machine.wait_until_succeeds("curl -sf http://localhost:8080/", timeout=30)
+
+    # Verify the ntfy-alert@ template service exists
+    machine.succeed("systemctl list-unit-files | grep ntfy-alert@")
+
+    # Verify the global OnFailure drop-in is configured
+    machine.succeed("cat /etc/systemd/system/service.d/onfailure.conf | grep -q 'OnFailure=ntfy-alert@%p.service'")
+
+    # Trigger the test-fail service
+    machine.succeed("systemctl start test-fail.service || true")
+
+    # Wait a moment for the failure notification to be sent
+    time.sleep(2)
+
+    # Verify the ntfy-alert@test-fail service ran
+    machine.succeed("systemctl is-active ntfy-alert@test-fail.service || systemctl is-failed ntfy-alert@test-fail.service || true")
+
+    # Check that the mock server received a POST request
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+
+    # Verify the request content
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+
+    assert len(requests) >= 1, f"Expected at least 1 request, got {len(requests)}"
+
+    # Check the first request
+    req = requests[0]
+    assert "/test-alerts" in req["path"], f"Expected path to contain /test-alerts, got {req['path']}"
+    assert "Title" in req["headers"], "Expected Title header"
+    assert "test-fail" in req["headers"]["Title"], f"Expected Title to contain 'test-fail', got {req['headers']['Title']}"
+    assert req["headers"]["Priority"] == "high", f"Expected Priority 'high', got {req['headers'].get('Priority')}"
+    assert req["headers"]["Tags"] == "warning", f"Expected Tags 'warning', got {req['headers'].get('Tags')}"
+
+    print(f"Received notification: Title={req['headers']['Title']}, Body={req['body'][:100]}...")
+
+    # Idempotency test: trigger failure again
+    machine.succeed("rm /tmp/ntfy-requests.json")
+    machine.succeed("systemctl reset-failed test-fail.service || true")
+    machine.succeed("systemctl start test-fail.service || true")
+    time.sleep(2)
+
+    # Verify another notification was sent
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+    assert len(requests) >= 1, f"Expected at least 1 request after second failure, got {len(requests)}"
+
+    print("All tests passed!")
+  '';
+}

tests/tests.nix

@@ -21,4 +21,7 @@ in
   fail2banVaultwardenTest = handleTest ./fail2ban-vaultwarden.nix;
   fail2banImmichTest = handleTest ./fail2ban-immich.nix;
   fail2banJellyfinTest = handleTest ./fail2ban-jellyfin.nix;
+
+  # ntfy alerts test
+  ntfyAlertsTest = handleTest ./ntfy-alerts.nix;
 }