llama-cpp: compiler optimizations

configuration.nix

@@ -28,7 +28,7 @@
     # ./services/owntracks.nix
     ./services/soulseek.nix
 
-    # ./services/llama-cpp.nix
+    ./services/llama-cpp.nix
   ];
 
   systemd.targets = {

flake.lock

@@ -200,11 +200,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1743366063,
-        "narHash": "sha256-Jrvjz9A8oGVo4KCP2miGX2VxIm5mRNPVIEcEaVBDRsE=",
+        "lastModified": 1743424621,
+        "narHash": "sha256-M4dHt10aGASKjoRtafFMJfewANh/7+O6t+ITb9oPsNY=",
         "owner": "ggml-org",
         "repo": "llama.cpp",
-        "rev": "2c3f8b850a4a6cff0f5dda2135c03fc81d33ed8b",
+        "rev": "a8a1f3356786cbf8bcc3422e3c8737fc33b453e7",
         "type": "github"
       },
       "original": {
@@ -237,11 +237,11 @@
     },
     "nixos-hardware": {
       "locked": {
-        "lastModified": 1743167577,
-        "narHash": "sha256-I09SrXIO0UdyBFfh0fxDq5WnCDg8XKmZ1HQbaXzMA1k=",
+        "lastModified": 1743420942,
+        "narHash": "sha256-b/exDDQSLmENZZgbAEI3qi9yHkuXAXCPbormD8CSJXo=",
         "owner": "NixOS",
         "repo": "nixos-hardware",
-        "rev": "0ed819e708af17bfc4bbc63ee080ef308a24aa42",
+        "rev": "de6fc5551121c59c01e2a3d45b277a6d05077bc4",
         "type": "github"
       },
       "original": {
@@ -253,11 +253,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1743354925,
-        "narHash": "sha256-B+AxLrBmlMmnkGD/5PWtoy2zdk/0epewoUw4EDlc8lA=",
+        "lastModified": 1743398199,
+        "narHash": "sha256-Zy9o4AiBVjmswfXtw5l0YTUSEp676YKAqC9Z2d6MvI0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "47d6fd35d221f4a8fe8d45cbaec844d59c7dd6c2",
+        "rev": "e9549075069ef5e68ddf29cb07c0e38e2a915242",
         "type": "github"
       },
       "original": {

services/bitmagnet.nix

@@ -36,7 +36,6 @@
   };
 
   services.caddy.virtualHosts."bitmagnet.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
     ${builtins.readFile ../secrets/caddy_auth}
     reverse_proxy ${service_configs.https.wg_ip}:${builtins.toString service_configs.ports.bitmagnet}
   '';

services/llama-cpp.nix

@@ -5,24 +5,48 @@
   inputs,
   ...
 }:
+let
+
+  # stolen from: https://stackoverflow.com/a/42398526
+  optimizeWithFlags =
+    pkg: flags:
+    pkgs.lib.overrideDerivation pkg (
+      old:
+      let
+        newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags;
+        oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else "";
+      in
+      {
+        NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}";
+        stdenv = pkgs.clang19Stdenv;
+      }
+    );
+in
 {
   services.llama-cpp = {
     enable = true;
     model = builtins.toString (
       pkgs.fetchurl {
-        url = "https://huggingface.co/bartowski/google_gemma-3-27b-it-GGUF/resolve/main/google_gemma-3-27b-it-IQ4_XS.gguf";
-        sha256 = "bd2f188c66d8ccb0bffcb0c91e4dbbb72754bb1732e0bca323a2f266a35e01c8";
+        url = "https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-IQ4_XS.gguf";
+        sha256 = "aa7b7ae0b17931c379ede82da59b01f246046925aeb752af1ab4285a3b0d69db";
       }
     );
     port = service_configs.ports.llama_cpp;
     host = "0.0.0.0";
-    package = inputs.llamacpp.packages.${pkgs.system}.default;
+    package = (
+      optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [
+        "-O3"
+        "-march=znver2"
+        "-mtune=znver2"
+      ]
+    );
     extraFlags = [
 
     ];
   };
 
   services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
+    ${builtins.readFile ../secrets/caddy_auth}
     reverse_proxy :${builtins.toString config.services.llama-cpp.port}
   '';
 }

services/minecraft.nix

@@ -111,7 +111,6 @@
   };
 
   services.caddy.virtualHosts."map.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
     root * ${service_configs.minecraft.parent_dir}/${service_configs.minecraft.server_name}/squaremap/web
     file_server browse
   '';

services/qbittorrent.nix

@@ -121,7 +121,6 @@
   };
 
   services.caddy.virtualHosts."torrent.${service_configs.https.domain}".extraConfig = ''
-    # tls internal
     ${builtins.readFile ../secrets/caddy_auth}
     reverse_proxy ${service_configs.https.wg_ip}:${builtins.toString config.services.qbittorrent.webuiPort}
   '';