minecraft: update mods + add modernfix + debugify

configuration.nix

@@ -19,7 +19,7 @@
     ./modules/secureboot.nix
     ./modules/no-rgb.nix
     ./modules/security.nix
-    ./modules/arr-init.nix
+    ./modules/ntfy-alerts.nix
 
     ./services/postgresql.nix
     ./services/jellyfin.nix
@@ -64,6 +64,7 @@
     ./services/syncthing.nix
 
     ./services/ntfy.nix
+    ./services/ntfy-alerts.nix
   ];
 
   services.kmscon.enable = true;
@@ -156,6 +157,12 @@
 
       # Higher backlog for the large number of concurrent torrent connections
       "net.core.netdev_max_backlog" = 5000;
+
+      # Minecraft server optimizations
+      # Disable autogroup for better scheduling of game server threads
+      "kernel.sched_autogroup_enabled" = 0;
+      # Huge pages for Minecraft JVM (4000MB heap / 2MB per page + ~200 overhead)
+      "vm.nr_hugepages" = 2200;
     };
   };
 

flake.lock

@@ -25,13 +25,33 @@
         "type": "github"
       }
     },
+    "arr-init": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772249948,
+        "narHash": "sha256-v68tO12mTCET68eZG583U+OlBL4f6kAoHS9iKA/xLzQ=",
+        "ref": "refs/heads/main",
+        "rev": "d21eb9f5b0a30bb487de7c0afbbbaf19324eaa49",
+        "revCount": 1,
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      },
+      "original": {
+        "type": "git",
+        "url": "ssh://gitea@git.gardling.com/titaniumtown/arr-init"
+      }
+    },
     "crane": {
       "locked": {
-        "lastModified": 1771121070,
-        "narHash": "sha256-aIlv7FRXF9q70DNJPI237dEDAznSKaXmL5lfK/Id/bI=",
+        "lastModified": 1771796463,
+        "narHash": "sha256-9bCDuUzpwJXcHMQYMS1yNuzYMmKO/CCwCexpjWOl62I=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "a2812c19f1ed2e5ed5ce2ef7109798b575c180e1",
+        "rev": "3d3de3313e263e04894f284ac18177bd26169bad",
         "type": "github"
       },
       "original": {
@@ -69,11 +89,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771469470,
-        "narHash": "sha256-GnqdqhrguKNN3HtVfl6z+zbV9R9jhHFm3Z8nu7R6ml0=",
+        "lastModified": 1771881364,
+        "narHash": "sha256-A5uE/hMium5of/QGC6JwF5TGoDAfpNtW00T0s9u/PN8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "4707eec8d1d2db5182ea06ed48c820a86a42dc13",
+        "rev": "a4cb7bf73f264d40560ba527f9280469f1f081c6",
         "type": "github"
       },
       "original": {
@@ -177,11 +197,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771744638,
-        "narHash": "sha256-EDLi+YAsEEAmMeZe1v6GccuGRbCkpSZp/+A6g+pivR8=",
+        "lastModified": 1772020340,
+        "narHash": "sha256-aqBl3GNpCadMoJ/hVkWTijM1Aeilc278MjM+LA3jK6g=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "cb6c151f5c9db4df0b69d06894dc8484de1f16a0",
+        "rev": "36e38ca0d9afe4c55405fdf22179a5212243eecc",
         "type": "github"
       },
       "original": {
@@ -243,11 +263,11 @@
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1771834715,
-        "narHash": "sha256-5VI2KiMifx3Dca7nDJzctO3HpnS6zrvesdkLoZBrQRY=",
+        "lastModified": 1772216104,
+        "narHash": "sha256-1TnGN26vnCEQk5m4AavJZxGZTb/6aZyphemRPRwFUfs=",
         "owner": "nix-community",
         "repo": "lanzaboote",
-        "rev": "b798c53da0f7e521317a5413335096a21070cf0b",
+        "rev": "dbe5112de965bbbbff9f0729a9789c20a65ab047",
         "type": "github"
       },
       "original": {
@@ -265,11 +285,11 @@
         "systems": "systems_3"
       },
       "locked": {
-        "lastModified": 1771641457,
-        "narHash": "sha256-TIekRGfeCwuEmYcWex40RTx0Gd46pqmyUtxdFKb5juI=",
+        "lastModified": 1772160153,
+        "narHash": "sha256-lk5IxQzY9ZeeEyjKNT7P6dFnlRpQgkus4Ekc/+slypY=",
         "owner": "Infinidoge",
         "repo": "nix-minecraft",
-        "rev": "c4e2b8969e09067da9d44b6b5762e1e896418f40",
+        "rev": "deca3fb710b502ba10cd5cdc8f66c2cc184b92df",
         "type": "github"
       },
       "original": {
@@ -280,11 +300,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1771423359,
-        "narHash": "sha256-yRKJ7gpVmXbX2ZcA8nFi6CMPkJXZGjie2unsiMzj3Ig=",
+        "lastModified": 1771969195,
+        "narHash": "sha256-qwcDBtrRvJbrrnv1lf/pREQi8t2hWZxVAyeMo7/E9sw=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "740a22363033e9f1bb6270fbfb5a9574067af15b",
+        "rev": "41c6b421bdc301b2624486e11905c9af7b8ec68e",
         "type": "github"
       },
       "original": {
@@ -296,11 +316,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1771714954,
-        "narHash": "sha256-nhZJPnBavtu40/L2aqpljrfUNb2rxmWTmSjK2c9UKds=",
+        "lastModified": 1772047000,
+        "narHash": "sha256-7DaQVv4R97cii/Qdfy4tmDZMB2xxtyIvNGSwXBBhSmo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "afbbf774e2087c3d734266c22f96fca2e78d3620",
+        "rev": "1267bb4920d0fc06ea916734c11b0bf004bbe17e",
         "type": "github"
       },
       "original": {
@@ -336,11 +356,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1770726378,
-        "narHash": "sha256-kck+vIbGOaM/dHea7aTBxdFYpeUl/jHOy5W3eyRvVx8=",
+        "lastModified": 1771858127,
+        "narHash": "sha256-Gtre9YoYl3n25tJH2AoSdjuwcqij5CPxL3U3xysYD08=",
         "owner": "cachix",
         "repo": "pre-commit-hooks.nix",
-        "rev": "5eaaedde414f6eb1aea8b8525c466dc37bba95ae",
+        "rev": "49bbbfc218bf3856dfa631cead3b052d78248b83",
         "type": "github"
       },
       "original": {
@@ -352,6 +372,7 @@
     "root": {
       "inputs": {
         "agenix": "agenix",
+        "arr-init": "arr-init",
         "deploy-rs": "deploy-rs",
         "disko": "disko",
         "home-manager": "home-manager",
@@ -376,11 +397,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771125043,
-        "narHash": "sha256-ldf/s49n6rOAxl7pYLJGGS1N/assoHkCOWdEdLyNZkc=",
+        "lastModified": 1771988922,
+        "narHash": "sha256-Fc6FHXtfEkLtuVJzd0B6tFYMhmcPLuxr90rWfb/2jtQ=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "4912f951a26dc8142b176be2c2ad834319dc06e8",
+        "rev": "f4443dc3f0b6c5e6b77d923156943ce816d1fcb9",
         "type": "github"
       },
       "original": {
@@ -433,11 +454,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1771812348,
-        "narHash": "sha256-d8LL7nSpFueYtZhK29t7j3JiaKLA4lqW8neJv/uZGQc=",
+        "lastModified": 1772071250,
+        "narHash": "sha256-LDWvJDR1J8xE8TBJjzWnOA0oVP/l9xBFC4npQPJDHN4=",
         "owner": "nix-community",
         "repo": "srvos",
-        "rev": "ffc8fceb1e3cad06b5074cda30f88132b4fb4869",
+        "rev": "5cd73bcf984b72d8046e1175d13753de255adfb9",
         "type": "github"
       },
       "original": {
@@ -509,11 +530,11 @@
     "trackerlist": {
       "flake": false,
       "locked": {
-        "lastModified": 1771801782,
-        "narHash": "sha256-bwNSnudbgtknL3qhWw42JmYlLPiVPx37krjH+m8jsrA=",
+        "lastModified": 1772233783,
+        "narHash": "sha256-2jPUBKpPuT4dCXwVFuZvTH3QyURixsfJZD7Zqs0atPY=",
         "owner": "ngosang",
         "repo": "trackerslist",
-        "rev": "2e524b3ddb3919427c9d264cfa442b3c2b8d7eb0",
+        "rev": "85c4f103f130b070a192343c334f50c2f56b61a9",
         "type": "github"
       },
       "original": {

flake.nix

@@ -68,6 +68,11 @@
     ytbn-graphing-software = {
       url = "git+https://git.gardling.com/titaniumtown/YTBN-Graphing-Software";
     };
+
+    arr-init = {
+      url = "git+ssh://gitea@git.gardling.com/titaniumtown/arr-init";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =
@@ -83,7 +88,7 @@
       srvos,
       deploy-rs,
       impermanence,
-      agenix,
+      arr-init,
       ...
     }@inputs:
     let
@@ -235,6 +240,10 @@
         buildPlatform = builtins.currentSystem;
       };
       lib = import ./modules/lib.nix { inherit inputs pkgs service_configs; };
+      testSuite = import ./tests/tests.nix {
+        inherit pkgs lib inputs;
+        config = self.nixosConfigurations.muffin.config;
+      };
     in
     {
       formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt-tree;
@@ -291,6 +300,8 @@
 
           lanzaboote.nixosModules.lanzaboote
 
+          arr-init.nixosModules.default
+
           home-manager.nixosModules.home-manager
           (
             {
@@ -319,24 +330,19 @@
         };
       };
 
-      packages.${system} =
-        let
-          testSuite = import ./tests/tests.nix {
-            inherit pkgs lib inputs;
-            config = self.nixosConfigurations.muffin.config;
-          };
-        in
-        {
-          tests = pkgs.linkFarm "all-tests" (
-            pkgs.lib.mapAttrsToList (name: test: {
-              name = name;
-              path = test;
-            }) testSuite
-          );
-        }
-        // (pkgs.lib.mapAttrs' (name: test: {
-          name = "test-${name}";
-          value = test;
-        }) testSuite);
+      checks.${system} = testSuite;
+
+      packages.${system} = {
+        tests = pkgs.linkFarm "all-tests" (
+          pkgs.lib.mapAttrsToList (name: test: {
+            name = name;
+            path = test;
+          }) testSuite
+        );
+      }
+      // (pkgs.lib.mapAttrs' (name: test: {
+        name = "test-${name}";
+        value = test;
+      }) testSuite);
     };
 }

modules/age-secrets.nix

@@ -65,5 +65,20 @@
       owner = "root";
       group = "root";
     };
+
+    # ntfy-alerts secrets
+    ntfy-alerts-topic = {
+      file = ../secrets/ntfy-alerts-topic.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
+
+    ntfy-alerts-token = {
+      file = ../secrets/ntfy-alerts-token.age;
+      mode = "0400";
+      owner = "root";
+      group = "root";
+    };
   };
 }

modules/arr-init.nix

@@ -1,497 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.services.arrInit;
-  bazarrCfg = config.services.bazarrInit;
-
-  downloadClientModule = lib.types.submodule {
-    options = {
-      name = lib.mkOption {
-        type = lib.types.str;
-        description = "Display name of the download client (e.g. \"qBittorrent\").";
-        example = "qBittorrent";
-      };
-
-      implementation = lib.mkOption {
-        type = lib.types.str;
-        description = "Implementation identifier for the Servarr API.";
-        example = "QBittorrent";
-      };
-
-      configContract = lib.mkOption {
-        type = lib.types.str;
-        description = "Config contract identifier for the Servarr API.";
-        example = "QBittorrentSettings";
-      };
-
-      protocol = lib.mkOption {
-        type = lib.types.enum [
-          "torrent"
-          "usenet"
-        ];
-        default = "torrent";
-        description = "Download protocol type.";
-      };
-
-      fields = lib.mkOption {
-        type = lib.types.attrsOf lib.types.anything;
-        default = { };
-        description = ''
-          Flat key/value pairs for the download client configuration.
-          These are converted to the API's [{name, value}] array format.
-        '';
-        example = {
-          host = "192.168.15.1";
-          port = 6011;
-          useSsl = false;
-          tvCategory = "tvshows";
-        };
-      };
-    };
-  };
-
-  syncedAppModule = lib.types.submodule {
-    options = {
-      name = lib.mkOption {
-        type = lib.types.str;
-        description = "Display name of the application to sync (e.g. \"Sonarr\").";
-        example = "Sonarr";
-      };
-
-      implementation = lib.mkOption {
-        type = lib.types.str;
-        description = "Implementation identifier for the Prowlarr application API.";
-        example = "Sonarr";
-      };
-
-      configContract = lib.mkOption {
-        type = lib.types.str;
-        description = "Config contract identifier for the Prowlarr application API.";
-        example = "SonarrSettings";
-      };
-
-      syncLevel = lib.mkOption {
-        type = lib.types.str;
-        default = "fullSync";
-        description = "Sync level for the application.";
-      };
-
-      prowlarrUrl = lib.mkOption {
-        type = lib.types.str;
-        description = "URL of the Prowlarr instance.";
-        example = "http://localhost:9696";
-      };
-
-      baseUrl = lib.mkOption {
-        type = lib.types.str;
-        description = "URL of the target application.";
-        example = "http://localhost:8989";
-      };
-
-      apiKeyFrom = lib.mkOption {
-        type = lib.types.str;
-        description = "Path to the config.xml file to read the API key from at runtime.";
-        example = "/services/sonarr/config.xml";
-      };
-
-      syncCategories = lib.mkOption {
-        type = lib.types.listOf lib.types.int;
-        default = [ ];
-        description = "List of sync category IDs for the application.";
-        example = [
-          5000
-          5010
-          5020
-        ];
-      };
-
-      serviceName = lib.mkOption {
-        type = lib.types.str;
-        description = "Name of the systemd service to depend on for reading the API key.";
-        example = "sonarr";
-      };
-    };
-  };
-
-  instanceModule = lib.types.submodule {
-    options = {
-      enable = lib.mkEnableOption "Servarr application API initialization";
-
-      serviceName = lib.mkOption {
-        type = lib.types.str;
-        description = "Name of the systemd service this init depends on.";
-        example = "sonarr";
-      };
-
-      dataDir = lib.mkOption {
-        type = lib.types.str;
-        description = "Path to the application data directory containing config.xml.";
-        example = "/var/lib/sonarr";
-      };
-
-      port = lib.mkOption {
-        type = lib.types.port;
-        description = "API port of the Servarr application.";
-        example = 8989;
-      };
-
-      apiVersion = lib.mkOption {
-        type = lib.types.str;
-        default = "v3";
-        description = "API version string used in the base URL.";
-      };
-
-      networkNamespacePath = lib.mkOption {
-        type = lib.types.nullOr lib.types.str;
-        default = null;
-        description = "If set, run this init service inside the given network namespace path (e.g. /run/netns/wg).";
-      };
-
-      downloadClients = lib.mkOption {
-        type = lib.types.listOf downloadClientModule;
-        default = [ ];
-        description = "List of download clients to configure via the API.";
-      };
-
-      rootFolders = lib.mkOption {
-        type = lib.types.listOf lib.types.str;
-        default = [ ];
-        description = "List of root folder paths to configure via the API.";
-        example = [
-          "/media/tv"
-          "/media/movies"
-        ];
-      };
-
-      syncedApps = lib.mkOption {
-        type = lib.types.listOf syncedAppModule;
-        default = [ ];
-        description = "Applications to register for indexer sync (Prowlarr only).";
-      };
-    };
-  };
-
-  bazarrProviderModule = lib.types.submodule {
-    options = {
-      enable = lib.mkEnableOption "provider connection";
-
-      dataDir = lib.mkOption {
-        type = lib.types.str;
-        description = "Path to the provider's data directory containing config.xml.";
-        example = "/services/sonarr";
-      };
-
-      port = lib.mkOption {
-        type = lib.types.port;
-        description = "API port of the provider.";
-        example = 8989;
-      };
-
-      serviceName = lib.mkOption {
-        type = lib.types.str;
-        description = "Name of the systemd service to depend on.";
-        example = "sonarr";
-      };
-    };
-  };
-
-  bazarrInitModule = lib.types.submodule {
-    options = {
-      enable = lib.mkEnableOption "Bazarr API initialization";
-
-      dataDir = lib.mkOption {
-        type = lib.types.str;
-        description = "Path to Bazarr's data directory containing config/config.ini.";
-        example = "/services/bazarr";
-      };
-
-      port = lib.mkOption {
-        type = lib.types.port;
-        default = 6767;
-        description = "API port of Bazarr.";
-      };
-
-      sonarr = lib.mkOption {
-        type = bazarrProviderModule;
-        default = {
-          enable = false;
-        };
-        description = "Sonarr provider configuration.";
-      };
-
-      radarr = lib.mkOption {
-        type = bazarrProviderModule;
-        default = {
-          enable = false;
-        };
-        description = "Radarr provider configuration.";
-      };
-    };
-  };
-
-  curl = "${pkgs.curl}/bin/curl";
-  jq = "${pkgs.jq}/bin/jq";
-  grep = "${pkgs.gnugrep}/bin/grep";
-  awk = "${pkgs.gawk}/bin/awk";
-
-  mkDownloadClientPayload =
-    dc:
-    builtins.toJSON {
-      enable = true;
-      protocol = dc.protocol;
-      priority = 1;
-      name = dc.name;
-      implementation = dc.implementation;
-      configContract = dc.configContract;
-      fields = lib.mapAttrsToList (n: v: {
-        name = n;
-        value = v;
-      }) dc.fields;
-      tags = [ ];
-    };
-
-  mkDownloadClientSection = dc: ''
-    # Download client: ${dc.name}
-    echo "Checking download client '${dc.name}'..."
-    EXISTING_DC=$(${curl} -sf "$BASE_URL/downloadclient" -H "X-Api-Key: $API_KEY")
-    if echo "$EXISTING_DC" | ${jq} -e --arg name ${lib.escapeShellArg dc.name} '.[] | select(.name == $name)' > /dev/null 2>&1; then
-      echo "Download client '${dc.name}' already exists, skipping"
-    else
-      echo "Adding download client '${dc.name}'..."
-      ${curl} -sf -X POST "$BASE_URL/downloadclient?forceSave=true" \
-        -H "X-Api-Key: $API_KEY" \
-        -H "Content-Type: application/json" \
-        -d ${lib.escapeShellArg (mkDownloadClientPayload dc)}
-      echo "Download client '${dc.name}' added"
-    fi
-  '';
-
-  mkRootFolderSection = path: ''
-    # Root folder: ${path}
-    echo "Checking root folder '${path}'..."
-    EXISTING_RF=$(${curl} -sf "$BASE_URL/rootfolder" -H "X-Api-Key: $API_KEY")
-    if echo "$EXISTING_RF" | ${jq} -e --arg path ${lib.escapeShellArg path} '.[] | select(.path == $path)' > /dev/null 2>&1; then
-      echo "Root folder '${path}' already exists, skipping"
-    else
-      echo "Adding root folder '${path}'..."
-      ${curl} -sf -X POST "$BASE_URL/rootfolder" \
-        -H "X-Api-Key: $API_KEY" \
-        -H "Content-Type: application/json" \
-        -d ${lib.escapeShellArg (builtins.toJSON { inherit path; })}
-      echo "Root folder '${path}' added"
-    fi
-  '';
-
-  mkSyncedAppSection = app: ''
-    # Synced app: ${app.name}
-    echo "Checking synced app '${app.name}'..."
-    TARGET_API_KEY=$(${grep} -oP '(?<=<ApiKey>)[^<]+' ${lib.escapeShellArg app.apiKeyFrom})
-    EXISTING_APPS=$(${curl} -sf "$BASE_URL/applications" -H "X-Api-Key: $API_KEY")
-    if echo "$EXISTING_APPS" | ${jq} -e --arg name ${lib.escapeShellArg app.name} '.[] | select(.name == $name)' > /dev/null 2>&1; then
-      echo "Synced app '${app.name}' already exists, skipping"
-    else
-      echo "Adding synced app '${app.name}'..."
-      PAYLOAD=$(${jq} -n \
-        --arg name ${lib.escapeShellArg app.name} \
-        --arg implementation ${lib.escapeShellArg app.implementation} \
-        --arg configContract ${lib.escapeShellArg app.configContract} \
-        --arg syncLevel ${lib.escapeShellArg app.syncLevel} \
-        --arg prowlarrUrl ${lib.escapeShellArg app.prowlarrUrl} \
-        --arg baseUrl ${lib.escapeShellArg app.baseUrl} \
-        --arg apiKey "$TARGET_API_KEY" \
-        --argjson syncCategories ${builtins.toJSON app.syncCategories} \
-        '{
-          name: $name,
-          implementation: $implementation,
-          configContract: $configContract,
-          syncLevel: $syncLevel,
-          fields: [
-            {name: "prowlarrUrl", value: $prowlarrUrl},
-            {name: "baseUrl", value: $baseUrl},
-            {name: "apiKey", value: $apiKey},
-            {name: "syncCategories", value: $syncCategories}
-          ],
-          tags: []
-        }')
-      ${curl} -sf -X POST "$BASE_URL/applications?forceSave=true" \
-        -H "X-Api-Key: $API_KEY" \
-        -H "Content-Type: application/json" \
-        -d "$PAYLOAD"
-      echo "Synced app '${app.name}' added"
-    fi
-  '';
-
-  mkInitScript =
-    name: inst:
-    pkgs.writeShellScript "${name}-init" ''
-      set -euo pipefail
-
-      CONFIG_XML="${inst.dataDir}/config.xml"
-
-      if [ ! -f "$CONFIG_XML" ]; then
-        echo "Config file $CONFIG_XML not found, skipping ${name} init"
-        exit 0
-      fi
-
-      API_KEY=$(${grep} -oP '(?<=<ApiKey>)[^<]+' "$CONFIG_XML")
-      BASE_URL="http://localhost:${builtins.toString inst.port}/api/${inst.apiVersion}"
-
-      # Wait for API to become available
-      echo "Waiting for ${name} API..."
-      for i in $(seq 1 90); do
-        if ${curl} -sf "$BASE_URL/system/status" -H "X-Api-Key: $API_KEY" > /dev/null 2>&1; then
-          echo "${name} API is ready"
-          break
-        fi
-        if [ "$i" -eq 90 ]; then
-          echo "${name} API not available after 90 seconds" >&2
-          exit 1
-        fi
-        sleep 1
-      done
-
-      ${lib.concatMapStringsSep "\n" mkDownloadClientSection inst.downloadClients}
-      ${lib.concatMapStringsSep "\n" mkRootFolderSection inst.rootFolders}
-      ${lib.concatMapStringsSep "\n" mkSyncedAppSection inst.syncedApps}
-
-      echo "${name} init complete"
-    '';
-
-  # Get list of service names that syncedApps depend on
-  getSyncedAppDeps = inst: map (app: "${app.serviceName}.service") inst.syncedApps;
-
-  enabledInstances = lib.filterAttrs (_: inst: inst.enable) cfg;
-
-  mkBazarrProviderSection =
-    type: provider:
-    let
-      ltype = lib.toLower type;
-    in
-    ''
-      # ${type} provider
-      echo "Checking ${type} provider..."
-      PROVIDER_API_KEY=$(${grep} -oP '(?<=<ApiKey>)[^<]+' ${lib.escapeShellArg "${provider.dataDir}/config.xml"})
-      EXISTING=$(${curl} -sf "$BASE_URL/api/system/settings" -H "X-API-KEY: $API_KEY")
-      USE_FLAG=$(echo "$EXISTING" | ${jq} -r '.general.use_${ltype}')
-      EXISTING_KEY=$(echo "$EXISTING" | ${jq} -r '.${ltype}.apikey // ""')
-      if [ "$USE_FLAG" = "true" ] && [ -n "$EXISTING_KEY" ]; then
-        echo "${type} provider already configured, skipping"
-      else
-        echo "Adding ${type} provider..."
-        ${curl} -sf -X POST "$BASE_URL/api/system/settings" \
-          -H "X-API-KEY: $API_KEY" \
-          -d "settings-general-use_${ltype}=true" \
-          -d "settings-${ltype}-ip=localhost" \
-          -d "settings-${ltype}-port=${builtins.toString provider.port}" \
-          -d "settings-${ltype}-apikey=$PROVIDER_API_KEY" \
-          -d "settings-${ltype}-ssl=false" \
-          -d "settings-${ltype}-base_url=/"
-        echo "${type} provider added"
-      fi
-    '';
-
-  mkBazarrInitScript = pkgs.writeShellScript "bazarr-init" ''
-    set -euo pipefail
-
-    CONFIG_YAML="${bazarrCfg.dataDir}/config/config.yaml"
-
-    if [ ! -f "$CONFIG_YAML" ]; then
-      echo "Config file $CONFIG_YAML not found, skipping bazarr init"
-      exit 0
-    fi
-
-    API_KEY=$(${awk} '/^auth:/{f=1} f && /apikey:/{gsub(/.*apikey: /, ""); print; exit}' "$CONFIG_YAML")
-    BASE_URL="http://localhost:${builtins.toString bazarrCfg.port}"
-
-    # Wait for API to become available
-    echo "Waiting for Bazarr API..."
-    for i in $(seq 1 90); do
-      if ${curl} -sf "$BASE_URL/api/system/status" -H "X-API-KEY: $API_KEY" > /dev/null 2>&1; then
-        echo "Bazarr API is ready"
-        break
-      fi
-      if [ "$i" -eq 90 ]; then
-        echo "Bazarr API not available after 90 seconds" >&2
-        exit 1
-      fi
-      sleep 1
-    done
-
-    ${lib.optionalString bazarrCfg.sonarr.enable (mkBazarrProviderSection "Sonarr" bazarrCfg.sonarr)}
-    ${lib.optionalString bazarrCfg.radarr.enable (mkBazarrProviderSection "Radarr" bazarrCfg.radarr)}
-
-    echo "Bazarr init complete"
-  '';
-
-  bazarrDeps = [
-    "bazarr.service"
-  ]
-  ++ (lib.optional bazarrCfg.sonarr.enable "${bazarrCfg.sonarr.serviceName}.service")
-  ++ (lib.optional bazarrCfg.radarr.enable "${bazarrCfg.radarr.serviceName}.service");
-in
-{
-  options.services.arrInit = lib.mkOption {
-    type = lib.types.attrsOf instanceModule;
-    default = { };
-    description = ''
-      Attribute set of Servarr application instances to initialize via their APIs.
-      Each instance generates a systemd oneshot service that idempotently configures
-      download clients, root folders, and synced applications.
-    '';
-  };
-
-  options.services.bazarrInit = lib.mkOption {
-    type = bazarrInitModule;
-    default = {
-      enable = false;
-    };
-    description = ''
-      Bazarr API initialization for connecting Sonarr and Radarr providers.
-      Bazarr uses a different API than Servarr applications, so it has its own module.
-    '';
-  };
-
-  config = lib.mkMerge [
-    (lib.mkIf (enabledInstances != { }) {
-      systemd.services = lib.mapAttrs' (
-        name: inst:
-        lib.nameValuePair "${inst.serviceName}-init" {
-          description = "Initialize ${name} API connections";
-          after = [
-            "${inst.serviceName}.service"
-          ]
-          ++ (getSyncedAppDeps inst)
-          ++ (lib.optional (inst.networkNamespacePath != null) "wg.service");
-          requires = [ "${inst.serviceName}.service" ];
-          wantedBy = [ "multi-user.target" ];
-          serviceConfig = {
-            Type = "oneshot";
-            RemainAfterExit = true;
-            ExecStart = "${mkInitScript name inst}";
-          }
-          // lib.optionalAttrs (inst.networkNamespacePath != null) {
-            NetworkNamespacePath = inst.networkNamespacePath;
-          };
-        }
-      ) enabledInstances;
-    })
-
-    (lib.mkIf bazarrCfg.enable {
-      systemd.services.bazarr-init = {
-        description = "Initialize Bazarr API connections";
-        after = bazarrDeps;
-        requires = bazarrDeps;
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig = {
-          Type = "oneshot";
-          RemainAfterExit = true;
-          ExecStart = "${mkBazarrInitScript}";
-        };
-      };
-    })
-  ];
-}

modules/ntfy-alerts.nix

@@ -0,0 +1,132 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.ntfyAlerts;
+
+  curl = "${pkgs.curl}/bin/curl";
+  hostname = config.networking.hostName;
+
+  # Build the curl auth args as a proper bash array fragment
+  authCurlArgs =
+    if cfg.tokenFile != null then
+      ''
+        if [ -f "${cfg.tokenFile}" ]; then
+          TOKEN=$(cat "${cfg.tokenFile}" 2>/dev/null || echo "")
+          if [ -n "$TOKEN" ]; then
+            AUTH_ARGS=(-H "Authorization: Bearer $TOKEN")
+          fi
+        fi
+      ''
+    else
+      "";
+
+  # Systemd failure alert script
+  systemdAlertScript = pkgs.writeShellScript "ntfy-systemd-alert" ''
+    set -euo pipefail
+
+    UNIT_NAME="$1"
+    SERVER_URL="${cfg.serverUrl}"
+    TOPIC=$(cat "${cfg.topicFile}" 2>/dev/null | tr -d '[:space:]')
+    if [ -z "$TOPIC" ]; then
+      echo "ERROR: Could not read topic from ${cfg.topicFile}"
+      exit 1
+    fi
+
+    # Get journal output for context
+    JOURNAL_OUTPUT=$(${pkgs.systemd}/bin/journalctl -u "$UNIT_NAME" -n 15 --no-pager 2>/dev/null || echo "No journal output available")
+
+    # Build auth args
+    AUTH_ARGS=()
+    ${authCurlArgs}
+
+    # Send notification
+    ${curl} -sf --max-time 15 -X POST \
+      "$SERVER_URL/$TOPIC" \
+      -H "Title: [${hostname}] Service failed: $UNIT_NAME" \
+      -H "Priority: high" \
+      -H "Tags: warning" \
+      "''${AUTH_ARGS[@]}" \
+      -d "$JOURNAL_OUTPUT" || true
+  '';
+
+in
+{
+  options.services.ntfyAlerts = {
+    enable = lib.mkEnableOption "ntfy push notifications for system alerts";
+
+    serverUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "The ntfy server URL (e.g. https://ntfy.example.com)";
+      example = "https://ntfy.example.com";
+    };
+
+    topicFile = lib.mkOption {
+      type = lib.types.path;
+      description = "Path to a file containing the ntfy topic name to publish alerts to.";
+      example = "/run/agenix/ntfy-alerts-topic";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Path to a file containing the ntfy auth token.
+        If set, uses Authorization: Bearer header for authentication.
+      '';
+      example = "/run/secrets/ntfy-token";
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    # Per-service OnFailure for monitored services
+    systemd.services = {
+      "ntfy-alert@" = {
+        description = "Send ntfy notification for failed service %i";
+
+        unitConfig.OnFailure = lib.mkForce "";
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${systemdAlertScript} %i";
+          TimeoutSec = 30;
+        };
+      };
+
+      # TODO: sanoid's ExecStartPre runs `zfs allow` which blocks on TXG sync;
+      # on the hdds pool (slow spinning disks + large async frees) this causes
+      # 30+ minute hangs and guaranteed timeouts. Suppress until we fix sanoid
+      # to run as root without `zfs allow`. See: nixpkgs#72060, openzfs/zfs#14180
+      "sanoid".unitConfig.OnFailure = lib.mkForce "";
+    };
+
+    # Global OnFailure drop-in for all services
+    systemd.packages = [
+      (pkgs.writeTextDir "etc/systemd/system/service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=ntfy-alert@%p.service
+      '')
+
+      # Sanoid-specific drop-in to override the global OnFailure (see TODO above)
+      (pkgs.writeTextDir "etc/systemd/system/sanoid.service.d/onfailure.conf" ''
+        [Unit]
+        OnFailure=
+      '')
+    ];
+    # ZED (ZFS Event Daemon) ntfy notification settings
+    services.zfs.zed = {
+      enableMail = false;
+      settings = {
+        ZED_NTFY_URL = cfg.serverUrl;
+        ZED_NTFY_TOPIC = "$(cat ${cfg.topicFile} | tr -d '[:space:]')";
+        ZED_NTFY_ACCESS_TOKEN = lib.mkIf (cfg.tokenFile != null) "$(cat ${cfg.tokenFile})";
+        ZED_NOTIFY_VERBOSE = true;
+      };
+    };
+
+  };
+}

modules/zfs.nix

@@ -64,7 +64,7 @@ in
       yearly = 0;
     };
 
-    datasets."${service_configs.zpool_ssds}/services/jellyfin_cache" = {
+    datasets."${service_configs.zpool_ssds}/services/jellyfin/cache" = {
       recursive = true;
       autoprune = true;
       autosnap = true;

services/matrix.nix

@@ -12,8 +12,8 @@ let
         domain = "forgejo.ellis.link";
         owner = "continuwuation";
         repo = "continuwuity";
-        rev = "cb9786466bee299c1260171cee2a5ca407bda108";
-        hash = "sha256-uKntvmaXyeuqpeFTv0k4IjMu/fCdYlKiKJQ6JC/z7Yk=";
+        rev = "052c4dfa2165fdc4839fed95b71446120273cf23";
+        hash = "sha256-kQV4glRrKczoJpn9QIMgB5ac+saZQjSZPel+9K9Ykcs=";
       };
     in
     pkgs.matrix-continuwuity.overrideAttrs (old: {
@@ -21,7 +21,7 @@ let
       cargoDeps = pkgs.rustPlatform.fetchCargoVendor {
         inherit src;
         name = "${old.pname}-vendor";
-        hash = "sha256-V7OEvZxRe4Hg/XNp4PtQWxxQS8a5ONUw5X+6n7DIaCI=";
+        hash = "sha256-vlOXQL8wwEGFX+w0G/eIeHW3J1UDzhJ501kYhAghDV8=";
       };
 
       patches = (old.patches or [ ]) ++ [

services/minecraft.nix

@@ -35,7 +35,38 @@
         let
           heap_size = "4000M";
         in
-        "-Xmx${heap_size} -Xms${heap_size} -XX:+UseZGC -XX:+ZGenerational";
+        lib.concatStringsSep " " [
+          # Memory
+          "-Xmx${heap_size}"
+          "-Xms${heap_size}"
+          # GC
+          "-XX:+UseZGC"
+          "-XX:+ZGenerational"
+          # Base JVM optimizations (brucethemoose/Minecraft-Performance-Flags-Benchmarks)
+          "-XX:+UnlockExperimentalVMOptions"
+          "-XX:+UnlockDiagnosticVMOptions"
+          "-XX:+AlwaysActAsServerClassMachine"
+          "-XX:+AlwaysPreTouch"
+          "-XX:+DisableExplicitGC"
+          "-XX:+UseNUMA"
+          "-XX:+PerfDisableSharedMem"
+          "-XX:+UseFastUnorderedTimeStamps"
+          "-XX:+UseCriticalJavaThreadPriority"
+          "-XX:ThreadPriorityPolicy=1"
+          "-XX:AllocatePrefetchStyle=3"
+          "-XX:-DontCompileHugeMethods"
+          "-XX:MaxNodeLimit=240000"
+          "-XX:NodeLimitFudgeFactor=8000"
+          "-XX:ReservedCodeCacheSize=400M"
+          "-XX:NonNMethodCodeHeapSize=12M"
+          "-XX:ProfiledCodeHeapSize=194M"
+          "-XX:NonProfiledCodeHeapSize=194M"
+          "-XX:NmethodSweepActivity=1"
+          "-XX:+UseVectorCmov"
+          # Large pages (requires vm.nr_hugepages sysctl)
+          "-XX:+UseLargePages"
+          "-XX:LargePageSizeInBytes=2m"
+        ];
 
       serverProperties = {
         server-port = service_configs.ports.minecraft;
@@ -87,8 +118,8 @@
             };
 
             c2me = fetchurl {
-              url = "https://cdn.modrinth.com/data/VSNURh3q/versions/DLKF3HZk/c2me-fabric-mc1.21.11-0.3.6%2Bbeta.1.0.jar";
-              sha512 = "d4f983aeb5083033b525522e623a9a9ba86b6fc9c83db008cc0575d0077e736ac9bee0b6b0e03b8d1c89ae27a4e5cdc269041f61eb0d1a10757de4c30b065467";
+              url = "https://cdn.modrinth.com/data/VSNURh3q/versions/QdLiMUjx/c2me-fabric-mc1.21.11-0.3.7%2Balpha.0.7.jar";
+              sha512 = "f9543febe2d649a82acd6d5b66189b6a3d820cf24aa503ba493fdb3bbd4e52e30912c4c763fe50006f9a46947ae8cd737d420838c61b93429542573ed67f958e";
             };
 
             krypton = fetchurl {
@@ -110,12 +141,32 @@
               url = "https://cdn.modrinth.com/data/c7m1mi73/versions/CUh1DWeO/packetfixer-fabric-3.3.4-1.21.11.jar";
               sha512 = "33331b16cb40c5e6fbaade3cacc26f3a0e8fa5805a7186f94d7366a0e14dbeee9de2d2e8c76fa71f5e9dd24eb1c261667c35447e32570ea965ca0f154fdfba0a";
             };
+
+            # fork of Modernfix for 1.21.11 (upstream will support 26.1)
+            modernfix = fetchurl {
+              url = "https://cdn.modrinth.com/data/TjSm1wrD/versions/JwSO8JCN/modernfix-5.25.2-build.4.jar";
+              sha512 = "0d65c05ac0475408c58ef54215714e6301113101bf98bfe4bb2ba949fbfddd98225ac4e2093a5f9206a9e01ba80a931424b237bdfa3b6e178c741ca6f7f8c6a3";
+            };
+
+            debugify = fetchurl {
+              url = "https://cdn.modrinth.com/data/QwxR6Gcd/versions/8Q49lnaU/debugify-1.21.11%2B1.0.jar";
+              sha512 = "04d82dd33f44ced37045f1f9a54ad4eacd70861ff74a8800f2d2df358579e6cb0ea86a34b0086b3e87026b1a0691dd6594b4fdc49f89106466eea840518beb03";
+            };
+
           }
         );
       };
     };
   };
 
+  systemd.services.minecraft-server-main = {
+    serviceConfig = {
+      Nice = -5;
+      IOSchedulingPriority = 0;
+      LimitMEMLOCK = "infinity"; # Required for large pages
+    };
+  };
+
   services.caddy.virtualHosts = lib.mkIf (config.services.caddy.enable) {
     "map.${service_configs.https.domain}".extraConfig = ''
       root * ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web

services/ntfy-alerts.nix

@@ -0,0 +1,10 @@
+{ config, service_configs, ... }:
+{
+  services.ntfyAlerts = {
+    enable = true;
+    serverUrl = "https://${service_configs.ntfy.domain}";
+    topicFile = config.age.secrets.ntfy-alerts-topic.path;
+
+    tokenFile = config.age.secrets.ntfy-alerts-token.path;
+  };
+}

tests/arr-init.nix

@@ -1,419 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
-in
-testPkgs.testers.runNixOSTest {
-  name = "arr-init";
-
-  nodes.machine =
-    { pkgs, ... }:
-    {
-      imports = [
-        ../modules/arr-init.nix
-      ];
-
-      system.stateVersion = config.system.stateVersion;
-
-      virtualisation.memorySize = 4096;
-
-      environment.systemPackages = with pkgs; [
-        curl
-        jq
-        gnugrep
-      ];
-
-      systemd.services.mock-qbittorrent =
-        let
-          mockQbitScript = pkgs.writeScript "mock-qbittorrent.py" ''
-            import json
-            from http.server import HTTPServer, BaseHTTPRequestHandler
-            from urllib.parse import parse_qs, urlparse
-
-
-            CATEGORIES = {
-                "tv": {"name": "tv", "savePath": "/downloads"},
-                "movies": {"name": "movies", "savePath": "/downloads"},
-            }
-
-
-            class QBitMock(BaseHTTPRequestHandler):
-                def _respond(self, code=200, body=b"Ok.", content_type="text/plain"):
-                    self.send_response(code)
-                    self.send_header("Content-Type", content_type)
-                    self.send_header("Set-Cookie", "SID=mock_session_id; Path=/")
-                    self.end_headers()
-                    self.wfile.write(body if isinstance(body, bytes) else body.encode())
-
-                def do_GET(self):
-                    path = self.path.split("?")[0]
-                    if path == "/api/v2/app/webapiVersion":
-                        self._respond(body=b"2.9.3")
-                    elif path == "/api/v2/app/version":
-                        self._respond(body=b"v5.0.0")
-                    elif path == "/api/v2/torrents/info":
-                        self._respond(body=b"[]", content_type="application/json")
-                    elif path == "/api/v2/torrents/categories":
-                        body = json.dumps(CATEGORIES).encode()
-                        self._respond(body=body, content_type="application/json")
-                    elif path == "/api/v2/app/preferences":
-                        body = json.dumps({"save_path": "/tmp"}).encode()
-                        self._respond(body=body, content_type="application/json")
-                    else:
-                        self._respond()
-
-                def do_POST(self):
-                    content_length = int(self.headers.get("Content-Length", 0))
-                    body = self.rfile.read(content_length).decode()
-                    path = urlparse(self.path).path
-                    query = parse_qs(urlparse(self.path).query)
-                    form = parse_qs(body)
-                    params = {**query, **form}
-                    if path == "/api/v2/torrents/createCategory":
-                        name = params.get("category", [""])[0]
-                        save_path = params.get("savePath", params.get("save_path", [""]))[0] or "/downloads"
-                        if name:
-                            CATEGORIES[name] = {"name": name, "savePath": save_path}
-                    if path in ["/api/v2/torrents/editCategory", "/api/v2/torrents/removeCategory"]:
-                        self._respond()
-                        return
-                    self._respond()
-
-                def log_message(self, format, *args):
-                    pass
-
-
-            HTTPServer(("0.0.0.0", 6011), QBitMock).serve_forever()
-          '';
-        in
-        {
-          description = "Mock qBittorrent API";
-          wantedBy = [ "multi-user.target" ];
-          before = [
-            "sonarr-init.service"
-            "radarr-init.service"
-          ];
-          serviceConfig = {
-            ExecStart = "${pkgs.python3}/bin/python3 ${mockQbitScript}";
-            Type = "simple";
-          };
-        };
-
-      systemd.tmpfiles.rules = [
-        "d /media/tv 0755 sonarr sonarr -"
-        "d /media/movies 0755 radarr radarr -"
-      ];
-
-      services.sonarr = {
-        enable = true;
-        dataDir = "/var/lib/sonarr/.config/NzbDrone";
-        settings.server.port = lib.mkDefault 8989;
-      };
-
-      services.radarr = {
-        enable = true;
-        dataDir = "/var/lib/radarr/.config/Radarr";
-        settings.server.port = lib.mkDefault 7878;
-      };
-
-      services.prowlarr = {
-        enable = true;
-      };
-
-      services.bazarr = {
-        enable = true;
-        listenPort = 6767;
-      };
-
-      services.arrInit.sonarr = {
-        enable = true;
-        serviceName = "sonarr";
-        dataDir = "/var/lib/sonarr/.config/NzbDrone";
-        port = 8989;
-        downloadClients = [
-          {
-            name = "qBittorrent";
-            implementation = "QBittorrent";
-            configContract = "QBittorrentSettings";
-            protocol = "torrent";
-            fields = {
-              host = "127.0.0.1";
-              port = 6011;
-              useSsl = false;
-              tvCategory = "tv";
-            };
-          }
-        ];
-        rootFolders = [ "/media/tv" ];
-      };
-
-      services.arrInit.radarr = {
-        enable = true;
-        serviceName = "radarr";
-        dataDir = "/var/lib/radarr/.config/Radarr";
-        port = 7878;
-        downloadClients = [
-          {
-            name = "qBittorrent";
-            implementation = "QBittorrent";
-            configContract = "QBittorrentSettings";
-            protocol = "torrent";
-            fields = {
-              host = "127.0.0.1";
-              port = 6011;
-              useSsl = false;
-              movieCategory = "movies";
-            };
-          }
-        ];
-        rootFolders = [ "/media/movies" ];
-      };
-
-      services.arrInit.prowlarr = {
-        enable = true;
-        serviceName = "prowlarr";
-        dataDir = "/var/lib/prowlarr";
-        port = 9696;
-        apiVersion = "v1";
-        syncedApps = [
-          {
-            name = "Sonarr";
-            implementation = "Sonarr";
-            configContract = "SonarrSettings";
-            prowlarrUrl = "http://localhost:9696";
-            baseUrl = "http://localhost:8989";
-            apiKeyFrom = "/var/lib/sonarr/.config/NzbDrone/config.xml";
-            syncCategories = [
-              5000
-              5010
-              5020
-              5030
-              5040
-              5045
-              5050
-              5090
-            ];
-            serviceName = "sonarr";
-          }
-          {
-            name = "Radarr";
-            implementation = "Radarr";
-            configContract = "RadarrSettings";
-            prowlarrUrl = "http://localhost:9696";
-            baseUrl = "http://localhost:7878";
-            apiKeyFrom = "/var/lib/radarr/.config/Radarr/config.xml";
-            syncCategories = [
-              2000
-              2010
-              2020
-              2030
-              2040
-              2045
-              2050
-              2060
-              2070
-              2080
-            ];
-            serviceName = "radarr";
-          }
-        ];
-      };
-
-      services.bazarrInit = {
-        enable = true;
-        dataDir = "/var/lib/bazarr";
-        port = 6767;
-        sonarr = {
-          enable = true;
-          dataDir = "/var/lib/sonarr/.config/NzbDrone";
-          port = 8989;
-          serviceName = "sonarr";
-        };
-        radarr = {
-          enable = true;
-          dataDir = "/var/lib/radarr/.config/Radarr";
-          port = 7878;
-          serviceName = "radarr";
-        };
-      };
-    };
-
-  testScript = ''
-    start_all()
-
-    # Wait for services to start
-    machine.wait_for_unit("mock-qbittorrent.service")
-    machine.wait_until_succeeds("curl -sf http://localhost:6011/api/v2/app/version", timeout=30)
-    machine.wait_for_unit("sonarr.service")
-    machine.wait_for_unit("radarr.service")
-    machine.wait_for_unit("prowlarr.service")
-    machine.wait_for_unit("bazarr.service")
-
-    # Wait for Sonarr API to be ready (config.xml is auto-generated on first start)
-    machine.wait_until_succeeds(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
-        "curl -sf http://localhost:8989/api/v3/system/status -H \"X-Api-Key: $API_KEY\"",
-        timeout=120,
-    )
-
-    # Wait for Radarr API to be ready
-    machine.wait_until_succeeds(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
-        "curl -sf http://localhost:7878/api/v3/system/status -H \"X-Api-Key: $API_KEY\"",
-        timeout=120,
-    )
-
-    # Wait for Prowlarr API to be ready
-    machine.wait_until_succeeds(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/prowlarr/config.xml) && "
-        "curl -sf http://localhost:9696/api/v1/system/status -H \"X-Api-Key: $API_KEY\"",
-        timeout=180,
-    )
-
-    # Ensure init services run after config.xml exists
-    machine.succeed("systemctl restart sonarr-init.service")
-    machine.succeed("systemctl restart radarr-init.service")
-    machine.wait_for_unit("sonarr-init.service")
-    machine.wait_for_unit("radarr-init.service")
-
-    # Wait for init services to complete
-    machine.wait_for_unit("sonarr-init.service")
-    machine.wait_for_unit("radarr-init.service")
-
-    # Verify Sonarr download clients
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
-        "curl -sf http://localhost:8989/api/v3/downloadclient -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.name == \"qBittorrent\")'"
-    )
-
-    # Verify Sonarr root folders
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
-        "curl -sf http://localhost:8989/api/v3/rootfolder -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.path == \"/media/tv\")'"
-    )
-
-    # Verify Radarr download clients
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
-        "curl -sf http://localhost:7878/api/v3/downloadclient -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.name == \"qBittorrent\")'"
-    )
-
-    # Verify Radarr root folders
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
-        "curl -sf http://localhost:7878/api/v3/rootfolder -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.path == \"/media/movies\")'"
-    )
-
-    # Restart prowlarr-init now that all config.xml files exist
-    machine.succeed("systemctl restart prowlarr-init.service")
-    machine.wait_for_unit("prowlarr-init.service")
-
-    # Verify Sonarr registered as synced app in Prowlarr
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/prowlarr/config.xml) && "
-        "curl -sf http://localhost:9696/api/v1/applications -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.name == \"Sonarr\")'"
-    )
-
-    # Verify Radarr registered as synced app in Prowlarr
-    machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/prowlarr/config.xml) && "
-        "curl -sf http://localhost:9696/api/v1/applications -H \"X-Api-Key: $API_KEY\" | "
-        "jq -e '.[] | select(.name == \"Radarr\")'"
-    )
-
-    # Idempotency test: restart init services and verify no duplicate entries
-    machine.succeed("systemctl restart sonarr-init.service")
-    machine.succeed("systemctl restart radarr-init.service")
-    machine.succeed("systemctl restart prowlarr-init.service")
-
-    # Verify Sonarr still has exactly 1 download client
-    result = machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
-        "curl -sf http://localhost:8989/api/v3/downloadclient -H \"X-Api-Key: $API_KEY\" | "
-        "jq '. | length'"
-    ).strip()
-    assert result == "1", f"Expected 1 Sonarr download client, got {result}"
-
-    # Verify Sonarr still has exactly 1 root folder
-    result = machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/sonarr/.config/NzbDrone/config.xml) && "
-        "curl -sf http://localhost:8989/api/v3/rootfolder -H \"X-Api-Key: $API_KEY\" | "
-        "jq '. | length'"
-    ).strip()
-    assert result == "1", f"Expected 1 Sonarr root folder, got {result}"
-
-    # Verify Radarr still has exactly 1 download client
-    result = machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
-        "curl -sf http://localhost:7878/api/v3/downloadclient -H \"X-Api-Key: $API_KEY\" | "
-        "jq '. | length'"
-    ).strip()
-    assert result == "1", f"Expected 1 Radarr download client, got {result}"
-
-    # Verify Radarr still has exactly 1 root folder
-    result = machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/radarr/.config/Radarr/config.xml) && "
-        "curl -sf http://localhost:7878/api/v3/rootfolder -H \"X-Api-Key: $API_KEY\" | "
-        "jq '. | length'"
-    ).strip()
-    assert result == "1", f"Expected 1 Radarr root folder, got {result}"
-
-    # Verify Prowlarr still has exactly 2 synced apps
-    result = machine.succeed(
-        "API_KEY=$(grep -oP '(?<=<ApiKey>)[^<]+' /var/lib/prowlarr/config.xml) && "
-        "curl -sf http://localhost:9696/api/v1/applications -H \"X-Api-Key: $API_KEY\" | "
-        "jq '. | length'"
-    ).strip()
-    assert result == "2", f"Expected 2 Prowlarr synced apps, got {result}"
-
-    # Wait for Bazarr to generate config.yaml
-    machine.wait_until_succeeds(
-        "test -f /var/lib/bazarr/config/config.yaml",
-        timeout=120,
-    )
-
-    # Wait for Bazarr API to be ready
-    machine.wait_until_succeeds(
-        "API_KEY=$(awk '/^auth:/{f=1} f && /apikey:/{gsub(/.*apikey: /, \"\"); print; exit}' /var/lib/bazarr/config/config.yaml) && "
-        "curl -sf http://localhost:6767/api/system/status -H \"X-API-KEY: $API_KEY\"",
-        timeout=120,
-    )
-
-    # Restart bazarr-init now that config.yaml exists
-    machine.succeed("systemctl restart bazarr-init.service")
-    machine.wait_for_unit("bazarr-init.service")
-
-    # Verify Sonarr provider configured in Bazarr
-    machine.succeed(
-        "API_KEY=$(awk '/^auth:/{f=1} f && /apikey:/{gsub(/.*apikey: /, \"\"); print; exit}' /var/lib/bazarr/config/config.yaml) && "
-        "curl -sf http://localhost:6767/api/system/settings -H \"X-API-KEY: $API_KEY\" | "
-        "jq -e '.general.use_sonarr == true and (.sonarr.apikey // \"\") != \"\"'"
-    )
-
-    # Verify Radarr provider configured in Bazarr
-    machine.succeed(
-        "API_KEY=$(awk '/^auth:/{f=1} f && /apikey:/{gsub(/.*apikey: /, \"\"); print; exit}' /var/lib/bazarr/config/config.yaml) && "
-        "curl -sf http://localhost:6767/api/system/settings -H \"X-API-KEY: $API_KEY\" | "
-        "jq -e '.general.use_radarr == true and (.radarr.apikey // \"\") != \"\"'"
-    )
-
-    # Idempotency: restart bazarr-init and verify no duplicate config
-    machine.succeed("systemctl restart bazarr-init.service")
-    machine.wait_for_unit("bazarr-init.service")
-
-    machine.succeed(
-        "API_KEY=$(awk '/^auth:/{f=1} f && /apikey:/{gsub(/.*apikey: /, \"\"); print; exit}' /var/lib/bazarr/config/config.yaml) && "
-        "curl -sf http://localhost:6767/api/system/settings -H \"X-API-KEY: $API_KEY\" | "
-        "jq -e '.general.use_sonarr == true and (.sonarr.apikey // \"\") != \"\"'"
-    )
-  '';
-}

tests/minecraft.nix

@@ -60,6 +60,10 @@ testPkgs.testers.runNixOSTest {
         wants = lib.mkForce [ ];
         after = lib.mkForce [ ];
         requires = lib.mkForce [ ];
+        serviceConfig = {
+          Nice = lib.mkForce 0;
+          LimitMEMLOCK = lib.mkForce "infinity";
+        };
       };
 
       # Test-specific overrides only - reduce memory for testing

tests/ntfy-alerts.nix

@@ -0,0 +1,174 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testPkgs = pkgs.appendOverlays [ (import ../modules/overlays.nix) ];
+in
+testPkgs.testers.runNixOSTest {
+  name = "ntfy-alerts";
+
+  nodes.machine =
+    { pkgs, ... }:
+    {
+      imports = [
+        ../modules/ntfy-alerts.nix
+      ];
+
+      system.stateVersion = config.system.stateVersion;
+
+      virtualisation.memorySize = 2048;
+
+      environment.systemPackages = with pkgs; [
+        curl
+        jq
+      ];
+
+      # Create test topic file
+      systemd.tmpfiles.rules = [
+        "f /run/ntfy-test-topic 0644 root root - test-alerts"
+      ];
+
+      # Mock ntfy server that records POST requests
+      systemd.services.mock-ntfy =
+        let
+          mockNtfyScript = pkgs.writeScript "mock-ntfy.py" ''
+            import json
+            import os
+            from http.server import HTTPServer, BaseHTTPRequestHandler
+            from datetime import datetime
+
+            REQUESTS_FILE = "/tmp/ntfy-requests.json"
+
+            class MockNtfy(BaseHTTPRequestHandler):
+                def _respond(self, code=200, body=b"Ok"):
+                    self.send_response(code)
+                    self.send_header("Content-Type", "application/json")
+                    self.end_headers()
+                    self.wfile.write(body if isinstance(body, bytes) else body.encode())
+
+                def do_GET(self):
+                    self._respond()
+
+                def do_POST(self):
+                    content_length = int(self.headers.get("Content-Length", 0))
+                    body = self.rfile.read(content_length).decode() if content_length > 0 else ""
+
+                    request_data = {
+                        "timestamp": datetime.now().isoformat(),
+                        "path": self.path,
+                        "headers": dict(self.headers),
+                        "body": body,
+                    }
+
+                    # Load existing requests or start new list
+                    requests = []
+                    if os.path.exists(REQUESTS_FILE):
+                        try:
+                            with open(REQUESTS_FILE, "r") as f:
+                                requests = json.load(f)
+                        except:
+                            requests = []
+
+                    requests.append(request_data)
+
+                    with open(REQUESTS_FILE, "w") as f:
+                        json.dump(requests, f, indent=2)
+
+                    self._respond()
+
+                def log_message(self, format, *args):
+                    pass
+
+            HTTPServer(("0.0.0.0", 8080), MockNtfy).serve_forever()
+          '';
+        in
+        {
+          description = "Mock ntfy server";
+          wantedBy = [ "multi-user.target" ];
+          before = [ "ntfy-alert@test-fail.service" ];
+          serviceConfig = {
+            ExecStart = "${pkgs.python3}/bin/python3 ${mockNtfyScript}";
+            Type = "simple";
+          };
+        };
+
+      # Test service that will fail
+      systemd.services.test-fail = {
+        description = "Test service that fails";
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.coreutils}/bin/false";
+        };
+      };
+
+      # Configure ntfy-alerts to use mock server
+      services.ntfyAlerts = {
+        enable = true;
+        serverUrl = "http://localhost:8080";
+        topicFile = "/run/ntfy-test-topic";
+
+      };
+    };
+
+  testScript = ''
+    import json
+    import time
+
+    start_all()
+
+    # Wait for mock ntfy server to be ready
+    machine.wait_for_unit("mock-ntfy.service")
+    machine.wait_until_succeeds("curl -sf http://localhost:8080/", timeout=30)
+
+    # Verify the ntfy-alert@ template service exists
+    machine.succeed("systemctl list-unit-files | grep ntfy-alert@")
+
+    # Verify the global OnFailure drop-in is configured
+    machine.succeed("cat /etc/systemd/system/service.d/onfailure.conf | grep -q 'OnFailure=ntfy-alert@%p.service'")
+
+    # Trigger the test-fail service
+    machine.succeed("systemctl start test-fail.service || true")
+
+    # Wait a moment for the failure notification to be sent
+    time.sleep(2)
+
+    # Verify the ntfy-alert@test-fail service ran
+    machine.succeed("systemctl is-active ntfy-alert@test-fail.service || systemctl is-failed ntfy-alert@test-fail.service || true")
+
+    # Check that the mock server received a POST request
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+
+    # Verify the request content
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+
+    assert len(requests) >= 1, f"Expected at least 1 request, got {len(requests)}"
+
+    # Check the first request
+    req = requests[0]
+    assert "/test-alerts" in req["path"], f"Expected path to contain /test-alerts, got {req['path']}"
+    assert "Title" in req["headers"], "Expected Title header"
+    assert "test-fail" in req["headers"]["Title"], f"Expected Title to contain 'test-fail', got {req['headers']['Title']}"
+    assert req["headers"]["Priority"] == "high", f"Expected Priority 'high', got {req['headers'].get('Priority')}"
+    assert req["headers"]["Tags"] == "warning", f"Expected Tags 'warning', got {req['headers'].get('Tags')}"
+
+    print(f"Received notification: Title={req['headers']['Title']}, Body={req['body'][:100]}...")
+
+    # Idempotency test: trigger failure again
+    machine.succeed("rm /tmp/ntfy-requests.json")
+    machine.succeed("systemctl reset-failed test-fail.service || true")
+    machine.succeed("systemctl start test-fail.service || true")
+    time.sleep(2)
+
+    # Verify another notification was sent
+    machine.wait_until_succeeds("test -f /tmp/ntfy-requests.json", timeout=30)
+    result = machine.succeed("cat /tmp/ntfy-requests.json")
+    requests = json.loads(result)
+    assert len(requests) >= 1, f"Expected at least 1 request after second failure, got {len(requests)}"
+
+    print("All tests passed!")
+  '';
+}

tests/tests.nix

@@ -22,6 +22,6 @@ in
   fail2banImmichTest = handleTest ./fail2ban-immich.nix;
   fail2banJellyfinTest = handleTest ./fail2ban-jellyfin.nix;
 
-  # arr tests
-  arrInitTest = handleTest ./arr-init.nix;
+  # ntfy alerts test
+  ntfyAlertsTest = handleTest ./ntfy-alerts.nix;
 }