titaniumtown/server-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm: use vulkan

Browse Source
This commit is contained in:
Simon Gardling 2025-04-30 12:59:07 -04:00
parent 870093686a
commit fb4043712e
Signed by: titaniumtown
GPG Key ID: 9AB28AC10ECE533D
1 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
services/llama-cpp.nix
View File

@ -4,6 +4,7 @@
config, config,
inputs, inputs,
optimizePackage, optimizePackage,
lib,
... ...
}: }:
{ {
@ -17,20 +18,17 @@
); );
port = service_configs.ports.llama_cpp; port = service_configs.ports.llama_cpp;
host = "0.0.0.0"; host = "0.0.0.0";
package = ( package = (optimizePackage inputs.llamacpp.packages.${pkgs.system}.vulkan);
optimizePackage (
inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: {
cmakeFlags = old.cmakeFlags ++ [
"-DGGML_AVX2=ON"
];
})
)
);
extraFlags = [ extraFlags = [
"--flash-attn" "--flash-attn"
"-ngl"
"9999"
]; ];
}; };
# have to do this in order to get vulkan to work
systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false;
services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = '' services.caddy.virtualHosts."llm.${service_configs.https.domain}".extraConfig = ''
${builtins.readFile ../secrets/caddy_auth} ${builtins.readFile ../secrets/caddy_auth}
reverse_proxy :${builtins.toString config.services.llama-cpp.port} reverse_proxy :${builtins.toString config.services.llama-cpp.port}