From fb4043712ecd6ce0dbf237e2ad9bb083f7a32488 Mon Sep 17 00:00:00 2001
From: Simon Gardling <titaniumtown@proton.me>
Date: Wed, 30 Apr 2025 12:59:07 -0400
Subject: [PATCH] llm: use vulkan

---
 services/llama-cpp.nix | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/services/llama-cpp.nix b/services/llama-cpp.nix
index 393c090..3dc4573 100644
--- a/services/llama-cpp.nix
+++ b/services/llama-cpp.nix
@@ -4,6 +4,7 @@
   config,
   inputs,
   optimizePackage,
+  lib,
   ...
 }:
 {
@@ -17,20 +18,17 @@
     );
     port = service_configs.ports.llama_cpp;
     host = "0.0.0.0";
-    package = (
-      optimizePackage (
-        inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: {
-          cmakeFlags = old.cmakeFlags ++ [
-            "-DGGML_AVX2=ON"
-          ];
-        })
-      )
-    );
+    package = (optimizePackage inputs.llamacpp.packages.${pkgs.system}.vulkan);
     extraFlags = [
       "--flash-attn"
+      "-ngl"
+      "9999"
     ];
   };
 
+  # have to do this in order to get vulkan to work
+  systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false;
+
   services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
     ${builtins.readFile ../secrets/caddy_auth}
     reverse_proxy :${builtins.toString config.services.llama-cpp.port}