diff --git a/home-manager/progs/opencode.nix b/home-manager/progs/opencode.nix index 3478d3e..eaed9ff 100644 --- a/home-manager/progs/opencode.nix +++ b/home-manager/progs/opencode.nix @@ -50,17 +50,6 @@ apiKey = "{file:${../secrets/openrouter_api_key}}"; }; }; - # Local llama-cpp server with OpenAI-compatible API - "llama.cpp" = { - npm = "@ai-sdk/openai-compatible"; - options = { - baseURL = "http://127.0.0.1:8012/v1"; - }; - - models = { - "local" = { }; - }; - }; }; }; }; diff --git a/system/common.nix b/system/common.nix index 4db9ce2..c5727d2 100644 --- a/system/common.nix +++ b/system/common.nix @@ -15,7 +15,6 @@ # ./vm.nix ./steam.nix ./networking.nix - ./llama-cpp.nix inputs.disko.nixosModules.disko inputs.lanzaboote.nixosModules.lanzaboote diff --git a/system/llama-cpp.nix b/system/llama-cpp.nix deleted file mode 100644 index dcd9b5a..0000000 --- a/system/llama-cpp.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - config, - pkgs, - lib, - hostname, - inputs, - ... -}: -let - # stolen from: https://stackoverflow.com/a/42398526 - optimizeWithFlags = - pkg: flags: - lib.overrideDerivation pkg ( - old: - let - newflags = lib.foldl' (acc: x: "${acc} ${x}") "" flags; - oldflags = if (lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else ""; - in - { - NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}"; - # stdenv = pkgs.clang19Stdenv; - } - ); - - optimizePackage = - pkg: - optimizeWithFlags pkg [ - "-O3" - "-march=znver5" - "-mtune=znver5" - ]; -in -{ - services.llama-cpp = { - enable = true; - model = pkgs.fetchurl { - url = "https://huggingface.co/bartowski/Nanbeige_Nanbeige4-3B-Thinking-2511-GGUF/resolve/main/Nanbeige_Nanbeige4-3B-Thinking-2511-IQ4_NL.gguf"; - sha256 = "c49cd4652cffc3da91986ac2c3dccdf9ab19afe007445f30fe83001eb29b44cf"; - }; - - host = "127.0.0.1"; - port = 8012; - package = pkgs.llama-cpp.override { vulkanSupport = true; }; - - extraFlags = [ - # Context size - "-c" - "32768" - - "-ngl" - "99" - - "--alias" - "local" - ]; - }; - - # have to do this in order to get vulkan to work - systemd.services.llama-cpp.serviceConfig.DynamicUser = lib.mkForce false; -}