From e3e3036c066746efeddf7bab748ac7009e45255a Mon Sep 17 00:00:00 2001 From: Simon Gardling Date: Fri, 20 Sep 2024 13:16:23 -0400 Subject: [PATCH] gpt4all: add 14B qwen model --- nix/home-manager/progs/gpt4all/gpt4all.nix | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nix/home-manager/progs/gpt4all/gpt4all.nix b/nix/home-manager/progs/gpt4all/gpt4all.nix index 9055f4d..737c8de 100644 --- a/nix/home-manager/progs/gpt4all/gpt4all.nix +++ b/nix/home-manager/progs/gpt4all/gpt4all.nix @@ -3,12 +3,22 @@ let models = [ { name = "Qwen2.5-7B-Instruct-Q6_K_L.gguf"; - context_length = "32768"; + context_length = "131072"; + gen_length = "8192"; source = pkgs.fetchurl { url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true"; sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM="; }; } + { + name = "Qwen2.5-14B-Instruct-Q4_K_S.gguf"; + context_length = "131072"; + gen_length = "8192"; + source = pkgs.fetchurl { + url = "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q4_K_S.gguf?download=true"; + sha256 = "E1CmWUhMMbTXEjIRczzA3rSrVuR8qOL8BLagw7LiyZk="; + }; + } ]; #stolen from: https://stackoverflow.com/a/42398526 @@ -92,7 +102,7 @@ in [model-${model.name}] contextLength=${model.context_length} filename=${model.name} - maxLength=${model.context_length} + maxLength=${model.gen_length} promptBatchSize=256 promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|>