titaniumtown/server-config
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llama.cpp: testing

Browse Source
This commit is contained in:
Simon Gardling
2025-10-06 01:42:37 -04:00
parent 03729c90c1
commit a4bf2a0ea9
1 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
services/llama-cpp.nix
View File

@@ -11,8 +11,8 @@
enable = true; enable = true;
model = builtins.toString ( model = builtins.toString (
pkgs.fetchurl { pkgs.fetchurl {
url = "https://huggingface.co/rodrigomt/Qwen3-30B-A3B-Thinking-Deepseek-Distill-2507-v3.1-V2-GGUF/resolve/main/Qwen3-30B-A3B-Thinking-Deepseek-Distill-2507-v3.1-V2-UD-Q4_K_XL.gguf"; url = "https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf";
sha256 = "1a3abffc8463041e24cdc43af26c99b6cfab1d2ee78fef0d793033ec0e5b58aa"; sha256 = "be37a636aca0fc1aae0d32325f82f6b4d21495f06823b5fbc1898ae0303e9935";
} }
); );
port = service_configs.ports.llama_cpp; port = service_configs.ports.llama_cpp;
@@ -20,14 +20,16 @@
# vulkan broken: https://github.com/ggml-org/llama.cpp/issues/13801 # vulkan broken: https://github.com/ggml-org/llama.cpp/issues/13801
package = ( package = (
lib.optimizePackage ( lib.optimizePackage (
inputs.llamacpp.packages.${pkgs.system}.default.overrideAttrs (old: { inputs.llamacpp.packages.${pkgs.system}.vulkan.overrideAttrs (old: {
postPatch = ""; postPatch = "";
}) })
) )
); );
extraFlags = [ extraFlags = [
# "-ngl" "-ngl"
# "9999" "5"
"-c"
"16384"
]; ];
}; };