From 1482429a00c0b5520ef8c6d3bfa9b8c5b22d4f6b Mon Sep 17 00:00:00 2001
From: Simon Gardling <titaniumtown@proton.me>
Date: Mon, 31 Mar 2025 12:02:38 -0400
Subject: [PATCH] llm: enable AVX2

---
 services/llama-cpp.nix | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/services/llama-cpp.nix b/services/llama-cpp.nix
index 76918de..7a4f568 100644
--- a/services/llama-cpp.nix
+++ b/services/llama-cpp.nix
@@ -34,14 +34,20 @@ in
     port = service_configs.ports.llama_cpp;
     host = "0.0.0.0";
     package = (
-      optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [
-        "-O3"
-        "-march=znver2"
-        "-mtune=znver2"
-      ]
+      optimizeWithFlags
+        (inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: {
+          cmakeFlags = old.cmakeFlags ++ [
+            "-DGGML_AVX2=ON"
+          ];
+        }))
+        [
+          "-O3"
+          "-march=znver2"
+          "-mtune=znver2"
+        ]
     );
     extraFlags = [
-
+      "--flash-attn"
     ];
   };