tests: fix all fail2ban NixOS VM tests

- Add explicit iptables banaction in security.nix for test compatibility
- Force IPv4 in all curl requests to prevent IPv4/IPv6 mismatch issues
- Fix caddy test: use basic_auth directive (not basicauth)
- Override service ports in tests to match direct connections (not via Caddy)
- Vaultwarden: override ROCKET_ADDRESS and ROCKET_LOG for external access
- Immich: increase VM memory to 4GB for stability
- Jellyfin: create placeholder log file and reload fail2ban after startup
- Add tests.nix entries for all 6 fail2ban tests

All tests now pass: ssh, caddy, gitea, vaultwarden, immich, jellyfin

modules/security.nix

@@ -28,5 +28,10 @@
     */
   };
 
-  services.fail2ban.enable = true;
+  services.fail2ban = {
+    enable = true;
+    # Use iptables actions for compatibility
+    banaction = "iptables-multiport";
+    banaction-allports = "iptables-allports";
+  };
 }

tests/fail2ban-caddy.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-caddy";
+
+  nodes = {
+    server =
+      { config, pkgs, lib, ... }:
+      {
+        imports = [
+          ../modules/security.nix
+        ];
+
+        # Set up Caddy with basic auth (minimal config, no production stuff)
+        # Using bcrypt hash generated with: caddy hash-password --plaintext testpass
+        services.caddy = {
+          enable = true;
+          virtualHosts.":80".extraConfig = ''
+            log {
+              output file /var/log/caddy/access-server.log
+              format json
+            }
+            basic_auth {
+              testuser $2a$14$XqaQlGTdmofswciqrLlMz.rv0/jiGQq8aU.fP6mh6gCGiLf6Cl3.a
+            }
+            respond "Authenticated!" 200
+          '';
+        };
+
+        # Add the fail2ban jail for caddy-auth (same as in services/caddy.nix)
+        services.fail2ban.jails.caddy-auth = {
+          enabled = true;
+          settings = {
+            backend = "auto";
+            port = "http,https";
+            logpath = "/var/log/caddy/access-*.log";
+            maxretry = 3; # Lower for testing
+          };
+          filter.Definition = {
+            failregex = ''^.*"remote_ip":"<HOST>".*"status":401.*$'';
+            ignoreregex = "";
+            datepattern = ''"ts":{Epoch}\.'';
+          };
+        };
+
+        # Create log directory and initial log file so fail2ban can start
+        systemd.tmpfiles.rules = [
+          "d /var/log/caddy 755 caddy caddy"
+          "f /var/log/caddy/access-server.log 644 caddy caddy"
+        ];
+
+        networking.firewall.allowedTCPPorts = [ 80 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("caddy.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(80)
+    time.sleep(2)
+
+    with subtest("Verify caddy-auth jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "caddy-auth" in status, f"caddy-auth jail not found in: {status}"
+
+    with subtest("Verify correct password works"):
+        # Use -4 to force IPv4 for consistency
+        result = client.succeed("curl -4 -s -u testuser:testpass http://server/")
+        print(f"Curl result: {result}")
+        assert "Authenticated" in result, f"Auth should succeed: {result}"
+
+    with subtest("Generate failed basic auth attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute("curl -4 -s -u testuser:wrongpass http://server/ || true")
+            time.sleep(1)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(5)
+        status = server.succeed("fail2ban-client status caddy-auth")
+        print(f"caddy-auth jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-gitea.nix

@@ -0,0 +1,114 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    gitea = {
+      dir = "/var/lib/gitea";
+      domain = "git.test.local";
+    };
+    postgres = {
+      socket = "/run/postgresql";
+    };
+    ports = {
+      gitea = 3000;
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool = serviceName: zpool: dirs: { ... }: { };
+    }
+  );
+
+  giteaModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/gitea.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-gitea";
+
+  nodes = {
+    server =
+      { config, lib, pkgs, ... }:
+      {
+        imports = [
+          ../modules/security.nix
+          giteaModule
+        ];
+
+        # Enable postgres for gitea
+        services.postgresql.enable = true;
+
+        # Disable ZFS mount dependency
+        systemd.services."gitea-mounts".enable = lib.mkForce false;
+        systemd.services.gitea = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ "postgresql.service" ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.gitea.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Gitea port, not via Caddy
+          port = lib.mkForce "3000";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 3000 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("postgresql.service")
+    server.wait_for_unit("gitea.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(3000)
+    time.sleep(3)
+
+    with subtest("Verify gitea jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "gitea" in status, f"gitea jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute(
+                "curl -4 -s -X POST http://server:3000/user/login -d 'user_name=baduser&password=badpass' || true"
+            )
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status gitea")
+        print(f"gitea jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:3000/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-immich.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      immich = 2283;
+    };
+    immich = {
+      dir = "/var/lib/immich";
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool = serviceName: zpool: dirs: { ... }: { };
+    }
+  );
+
+  immichModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/immich.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-immich";
+
+  nodes = {
+    server =
+      { config, lib, pkgs, ... }:
+      {
+        imports = [
+          ../modules/security.nix
+          immichModule
+        ];
+
+        # Immich needs postgres
+        services.postgresql.enable = true;
+
+        # Let immich create its own DB for testing
+        services.immich.database.createDB = lib.mkForce true;
+
+        # Disable ZFS mount dependencies
+        systemd.services."immich-server-mounts".enable = lib.mkForce false;
+        systemd.services."immich-machine-learning-mounts".enable = lib.mkForce false;
+        systemd.services.immich-server = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ "postgresql.service" ];
+          requires = lib.mkForce [ ];
+        };
+        systemd.services.immich-machine-learning = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.immich.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Immich port, not via Caddy
+          port = lib.mkForce "2283";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 2283 ];
+
+        # Immich needs more resources
+        virtualisation.diskSize = 4 * 1024;
+        virtualisation.memorySize = 4 * 1024;  # 4GB RAM for Immich
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("postgresql.service")
+    server.wait_for_unit("immich-server.service", timeout=120)
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(2283, timeout=60)
+    time.sleep(3)
+
+    with subtest("Verify immich jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "immich" in status, f"immich jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute(
+                "curl -4 -s -X POST http://server:2283/api/auth/login -H 'Content-Type: application/json' -d '{\"email\":\"bad@user.com\",\"password\":\"badpass\"}' || true"
+            )
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status immich")
+        print(f"immich jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:2283/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-jellyfin.nix

@@ -0,0 +1,138 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      jellyfin = 8096;
+    };
+    jellyfin = {
+      dataDir = "/var/lib/jellyfin";
+      cacheDir = "/var/cache/jellyfin";
+    };
+    media_group = "media";
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool = serviceName: zpool: dirs: { ... }: { };
+      optimizePackage = pkg: pkg; # No-op for testing
+    }
+  );
+
+  jellyfinModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/jellyfin.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-jellyfin";
+
+  nodes = {
+    server =
+      { config, lib, pkgs, ... }:
+      {
+        imports = [
+          ../modules/security.nix
+          jellyfinModule
+        ];
+
+        # Create the media group
+        users.groups.media = { };
+
+        # Disable ZFS mount dependency
+        systemd.services."jellyfin-mounts".enable = lib.mkForce false;
+        systemd.services.jellyfin = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.jellyfin.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Jellyfin port, not via Caddy
+          port = lib.mkForce "8096";
+        };
+
+        # Create log directory and placeholder log file for fail2ban
+        # Jellyfin logs to files, not systemd journal
+        systemd.tmpfiles.rules = [
+          "d /var/lib/jellyfin/log 0755 jellyfin jellyfin"
+          "f /var/lib/jellyfin/log/log_placeholder.log 0644 jellyfin jellyfin"
+        ];
+
+        # Make fail2ban start after Jellyfin
+        systemd.services.fail2ban = {
+          wants = [ "jellyfin.service" ];
+          after = [ "jellyfin.service" ];
+        };
+
+        # Give jellyfin more disk space and memory
+        virtualisation.diskSize = 3 * 1024;
+        virtualisation.memorySize = 2 * 1024;
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("jellyfin.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(8096)
+    server.wait_until_succeeds("curl -sf http://localhost:8096/health | grep -q Healthy", timeout=60)
+    time.sleep(2)
+
+    # Wait for Jellyfin to create real log files and reload fail2ban
+    server.wait_until_succeeds("ls /var/lib/jellyfin/log/log_2*.log", timeout=30)
+    server.succeed("fail2ban-client reload jellyfin")
+
+    with subtest("Verify jellyfin jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "jellyfin" in status, f"jellyfin jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute("""
+                curl -4 -s -X POST http://server:8096/Users/authenticatebyname \
+                  -H 'Content-Type: application/json' \
+                  -H 'X-Emby-Authorization: MediaBrowser Client="test", Device="test", DeviceId="test", Version="1.0"' \
+                  -d '{"Username":"baduser","Pw":"badpass"}' || true
+            """)
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status jellyfin")
+        print(f"jellyfin jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:8096/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/fail2ban-ssh.nix

@@ -0,0 +1,91 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    zpool_hdds = "";
+  };
+
+  securityModule = import ../modules/security.nix;
+
+  sshModule =
+    { config, lib, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/ssh.nix {
+          inherit config lib pkgs;
+          username = "testuser";
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-ssh";
+
+  nodes = {
+    server =
+      { config, lib, pkgs, ... }:
+      {
+        imports = [
+          securityModule
+          sshModule
+        ];
+
+        # Override for testing - enable password auth
+        services.openssh.settings.PasswordAuthentication = lib.mkForce true;
+
+        users.users.testuser = {
+          isNormalUser = true;
+          password = "correctpassword";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 22 ];
+      };
+
+    client = {
+      environment.systemPackages = with pkgs; [ sshpass openssh ];
+    };
+  };
+
+  testScript = ''
+    import time
+
+    start_all()
+    server.wait_for_unit("sshd.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(22)
+    time.sleep(2)
+
+    with subtest("Verify sshd jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "sshd" in status, f"sshd jail not found in: {status}"
+
+    with subtest("Generate failed SSH login attempts"):
+        # Use -4 to force IPv4, timeout and NumberOfPasswordPrompts=1 to ensure quick failure
+        # maxRetry is 3 in our config, so 4 attempts should trigger a ban
+        for i in range(4):
+            client.execute(
+                "timeout 5 sshpass -p 'wrongpassword' ssh -4 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=3 -o NumberOfPasswordPrompts=1 testuser@server echo test 2>/dev/null || true"
+            )
+            time.sleep(1)
+
+    with subtest("Verify IP is banned"):
+        # Wait for fail2ban to process the logs and apply the ban
+        time.sleep(5)
+        status = server.succeed("fail2ban-client status sshd")
+        print(f"sshd jail status: {status}")
+        # Check that at least 1 IP is banned
+        import re
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("timeout 3 nc -4 -z -w 2 server 22")[0]
+        assert exit_code != 0, "Connection should be blocked for banned IP"
+  '';
+}

tests/fail2ban-vaultwarden.nix

@@ -0,0 +1,128 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  testServiceConfigs = {
+    zpool_ssds = "";
+    https = {
+      domain = "test.local";
+    };
+    ports = {
+      vaultwarden = 8222;
+    };
+    vaultwarden = {
+      path = "/var/lib/vaultwarden";
+    };
+  };
+
+  testLib = lib.extend (
+    final: prev: {
+      serviceMountWithZpool = serviceName: zpool: dirs: { ... }: { };
+    }
+  );
+
+  vaultwardenModule =
+    { config, pkgs, ... }:
+    {
+      imports = [
+        (import ../services/bitwarden.nix {
+          inherit config pkgs;
+          lib = testLib;
+          service_configs = testServiceConfigs;
+        })
+      ];
+    };
+in
+pkgs.testers.runNixOSTest {
+  name = "fail2ban-vaultwarden";
+
+  nodes = {
+    server =
+      { config, lib, pkgs, ... }:
+      {
+        imports = [
+          ../modules/security.nix
+          vaultwardenModule
+        ];
+
+        # Disable ZFS mount dependencies
+        systemd.services."vaultwarden-mounts".enable = lib.mkForce false;
+        systemd.services."backup-vaultwarden-mounts".enable = lib.mkForce false;
+        systemd.services.vaultwarden = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+        systemd.services.backup-vaultwarden = {
+          wants = lib.mkForce [ ];
+          after = lib.mkForce [ ];
+          requires = lib.mkForce [ ];
+        };
+
+        # Override Vaultwarden settings for testing
+        # - Listen on all interfaces (not just localhost)
+        # - Enable logging at info level to capture failed login attempts
+        services.vaultwarden.config = {
+          ROCKET_ADDRESS = lib.mkForce "0.0.0.0";
+          ROCKET_LOG = lib.mkForce "info";
+        };
+
+        # Override for faster testing and correct port
+        services.fail2ban.jails.vaultwarden.settings = {
+          maxretry = lib.mkForce 3;
+          # In test, we connect directly to Vaultwarden port, not via Caddy
+          port = lib.mkForce "8222";
+        };
+
+        networking.firewall.allowedTCPPorts = [ 8222 ];
+      };
+
+    client = {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+  };
+
+  testScript = ''
+    import time
+    import re
+
+    start_all()
+    server.wait_for_unit("vaultwarden.service")
+    server.wait_for_unit("fail2ban.service")
+    server.wait_for_open_port(8222)
+    time.sleep(2)
+
+    with subtest("Verify vaultwarden jail is active"):
+        status = server.succeed("fail2ban-client status")
+        assert "vaultwarden" in status, f"vaultwarden jail not found in: {status}"
+
+    with subtest("Generate failed login attempts"):
+        # Use -4 to force IPv4 for consistent IP tracking
+        for i in range(4):
+            client.execute("""
+                curl -4 -s -X POST 'http://server:8222/identity/connect/token' \
+                  -H 'Content-Type: application/x-www-form-urlencoded' \
+                  -H 'Bitwarden-Client-Name: web' \
+                  -H 'Bitwarden-Client-Version: 2024.1.0' \
+                  -d 'grant_type=password&username=bad@user.com&password=badpass&scope=api+offline_access&client_id=web&deviceType=10&deviceIdentifier=test&deviceName=test' \
+                  || true
+            """)
+            time.sleep(0.5)
+
+    with subtest("Verify IP is banned"):
+        time.sleep(3)
+        status = server.succeed("fail2ban-client status vaultwarden")
+        print(f"vaultwarden jail status: {status}")
+        # Check that at least 1 IP is banned
+        match = re.search(r"Currently banned:\s*(\d+)", status)
+        assert match and int(match.group(1)) >= 1, f"Expected at least 1 banned IP, got: {status}"
+
+    with subtest("Verify banned client cannot connect"):
+        # Use -4 to test with same IP that was banned
+        exit_code = client.execute("curl -4 -s --max-time 3 http://server:8222/ 2>&1")[0]
+        assert exit_code != 0, "Connection should be blocked"
+  '';
+}

tests/tests.nix

@@ -12,4 +12,12 @@ in
   testTest = handleTest ./testTest.nix;
   minecraftTest = handleTest ./minecraft.nix;
   jellyfinQbittorrentMonitorTest = handleTest ./jellyfin-qbittorrent-monitor.nix;
+
+  # fail2ban tests
+  fail2banSshTest = handleTest ./fail2ban-ssh.nix;
+  fail2banCaddyTest = handleTest ./fail2ban-caddy.nix;
+  fail2banGiteaTest = handleTest ./fail2ban-gitea.nix;
+  fail2banVaultwardenTest = handleTest ./fail2ban-vaultwarden.nix;
+  fail2banImmichTest = handleTest ./fail2ban-immich.nix;
+  fail2banJellyfinTest = handleTest ./fail2ban-jellyfin.nix;
 }