From 6cc3d96362ec9678e52d2fc23d8ff26b1a743f87 Mon Sep 17 00:00:00 2001
From: Simon Gardling <titaniumtown@proton.me>
Date: Mon, 31 Mar 2025 11:17:32 -0400
Subject: [PATCH] llama-cpp: compiler optimizations

---
 services/llama-cpp.nix | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/services/llama-cpp.nix b/services/llama-cpp.nix
index 94e5eff..76918de 100644
--- a/services/llama-cpp.nix
+++ b/services/llama-cpp.nix
@@ -5,6 +5,23 @@
   inputs,
   ...
 }:
+let
+
+  # stolen from: https://stackoverflow.com/a/42398526
+  optimizeWithFlags =
+    pkg: flags:
+    pkgs.lib.overrideDerivation pkg (
+      old:
+      let
+        newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags;
+        oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else "";
+      in
+      {
+        NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}";
+        stdenv = pkgs.clang19Stdenv;
+      }
+    );
+in
 {
   services.llama-cpp = {
     enable = true;
@@ -16,7 +33,13 @@
     );
     port = service_configs.ports.llama_cpp;
     host = "0.0.0.0";
-    package = inputs.llamacpp.packages.${pkgs.system}.default;
+    package = (
+      optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [
+        "-O3"
+        "-march=znver2"
+        "-mtune=znver2"
+      ]
+    );
     extraFlags = [
 
     ];