titaniumtown/dotfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gpt4all: add 14B qwen model

Browse Source
This commit is contained in:
Simon Gardling
2024-09-20 13:16:23 -04:00
parent c674d854d4
commit e3e3036c06
1 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
nix/home-manager/progs/gpt4all/gpt4all.nix
View File

@@ -3,12 +3,22 @@ let
models = [ models = [
{ {
name = "Qwen2.5-7B-Instruct-Q6_K_L.gguf"; name = "Qwen2.5-7B-Instruct-Q6_K_L.gguf";
context_length = "32768"; context_length = "131072";
gen_length = "8192";
source = pkgs.fetchurl { source = pkgs.fetchurl {
url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true"; url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true";
sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM="; sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM=";
}; };
} }
{
name = "Qwen2.5-14B-Instruct-Q4_K_S.gguf";
context_length = "131072";
gen_length = "8192";
source = pkgs.fetchurl {
url = "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q4_K_S.gguf?download=true";
sha256 = "E1CmWUhMMbTXEjIRczzA3rSrVuR8qOL8BLagw7LiyZk=";
};
}
]; ];
#stolen from: https://stackoverflow.com/a/42398526 #stolen from: https://stackoverflow.com/a/42398526
@@ -92,7 +102,7 @@ in
[model-${model.name}] [model-${model.name}]
contextLength=${model.context_length} contextLength=${model.context_length}
filename=${model.name} filename=${model.name}
maxLength=${model.context_length} maxLength=${model.gen_length}
promptBatchSize=256 promptBatchSize=256
promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n
systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|> systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|>