llm: enable AVX2

2025-03-31 12:02:38 -04:00 · 2025-03-31 12:02:38 -04:00 · 1482429a00
commit 1482429a00
parent 6cc3d96362
1 changed files with 12 additions and 6 deletions
--- a/services/llama-cpp.nix
+++ b/services/llama-cpp.nix
@ -34,14 +34,20 @@ in
    port = service_configs.ports.llama_cpp;
    host = "0.0.0.0";
    package = (
-      optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [
-        "-O3"
-        "-march=znver2"
-        "-mtune=znver2"
-      ]
+      optimizeWithFlags
+        (inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: {
+          cmakeFlags = old.cmakeFlags ++ [
+            "-DGGML_AVX2=ON"
+          ];
+        }))
+        [
+          "-O3"
+          "-march=znver2"
+          "-mtune=znver2"
+        ]
    );
    extraFlags = [
-
+      "--flash-attn"
    ];
  };