titaniumtown/server-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llama-cpp: compiler optimizations

Browse Source
This commit is contained in:
Simon Gardling 2025-03-31 11:17:32 -04:00
parent d5ac5c8cd8
commit 6cc3d96362
Signed by: titaniumtown
GPG Key ID: 9AB28AC10ECE533D
1 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
services/llama-cpp.nix
View File

@ -5,6 +5,23 @@
inputs,
...
}:
let
# stolen from: https://stackoverflow.com/a/42398526
optimizeWithFlags =
pkg: flags:
pkgs.lib.overrideDerivation pkg (
old:
let
newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags;
oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else "";
in
{
NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}";
stdenv = pkgs.clang19Stdenv;
}
);
in
{
services.llama-cpp = {
enable = true;
@ -16,7 +33,13 @@
);
port = service_configs.ports.llama_cpp;
host = "0.0.0.0";
package = inputs.llamacpp.packages.${pkgs.system}.default;
package = (
optimizeWithFlags inputs.llamacpp.packages.${pkgs.system}.default [
"-O3"
"-march=znver2"
"-mtune=znver2"
]
);
extraFlags = [
];