titaniumtown/server-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm: enable AVX2

Browse Source
This commit is contained in:
Simon Gardling 2025-03-31 12:02:38 -04:00
parent 6cc3d96362
commit 1482429a00
Signed by: titaniumtown
GPG Key ID: 9AB28AC10ECE533D
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
services/llama-cpp.nix
View File

@ -34,14 +34,20 @@ in
port = service_configs.ports.llama_cpp; port = service_configs.ports.llama_cpp;
host = "0.0.0.0"; host = "0.0.0.0";
package = ( package = (
optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [ optimizeWithFlags
"-O3" (inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: {
"-march=znver2" cmakeFlags = old.cmakeFlags ++ [
"-mtune=znver2" "-DGGML_AVX2=ON"
] ];
}))
[
"-O3"
"-march=znver2"
"-mtune=znver2"
]
); );
extraFlags = [ extraFlags = [
"--flash-attn"
]; ];
}; };