From b046534eb7969587e7bb91a690af1f1a06cce9d9 Mon Sep 17 00:00:00 2001 From: Simon Gardling Date: Fri, 20 Sep 2024 11:31:58 -0400 Subject: [PATCH] gpt4all: overhaul --- nix/etcnixos/flake.lock | 6 +- nix/home-manager/flake.lock | 12 +- nix/home-manager/gui.nix | 2 +- nix/home-manager/progs/gpt4all.nix | 85 --------------- .../gpt4all-HEAD-disable-settings-err.patch | 0 .../gpt4all-HEAD-embeddings-model.patch | 0 nix/home-manager/progs/gpt4all/gpt4all.nix | 103 ++++++++++++++++++ 7 files changed, 113 insertions(+), 95 deletions(-) delete mode 100644 nix/home-manager/progs/gpt4all.nix rename nix/home-manager/progs/{ => gpt4all}/gpt4all-HEAD-disable-settings-err.patch (100%) rename nix/home-manager/progs/{ => gpt4all}/gpt4all-HEAD-embeddings-model.patch (100%) create mode 100644 nix/home-manager/progs/gpt4all/gpt4all.nix diff --git a/nix/etcnixos/flake.lock b/nix/etcnixos/flake.lock index 2b63686..20cb5ea 100644 --- a/nix/etcnixos/flake.lock +++ b/nix/etcnixos/flake.lock @@ -332,11 +332,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1726830936, - "narHash": "sha256-ARM4KG4uxsHfONxyT4kSmcPpZuWsnrxDvTgxuzT31XI=", + "lastModified": 1726844355, + "narHash": "sha256-sbvo+lzHkAdQp7lum7/IbJtUMLJQSoERsQI2Zoytg7I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "039db25806f936e6a4ad3ab3efd0689a8cc7c23e", + "rev": "ff5752aab38a462024e8fce1475e37dcbd0e7526", "type": "github" }, "original": { diff --git a/nix/home-manager/flake.lock b/nix/home-manager/flake.lock index 9a0c377..88081f5 100644 --- a/nix/home-manager/flake.lock +++ b/nix/home-manager/flake.lock @@ -127,11 +127,11 @@ ] }, "locked": { - "lastModified": 1726823634, - "narHash": "sha256-rU8Yy62KSLU8Q2J64F+50OJKORNdogxbXl2w4rFw13o=", + "lastModified": 1726825546, + "narHash": "sha256-HiBzfzgqojA9OjPB+vdi2o+gy4Zw/MEipuGopgGsZEw=", "owner": "nix-community", "repo": "home-manager", - "rev": "4803bf558bdf20cb067aceb8830b7ad70113f4e3", + "rev": "0b052dd8119005c6ba819db48bcc657e48f401b7", "type": "github" }, "original": { @@ -201,11 +201,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1726836432, - "narHash": "sha256-fk9P0RY2m7r3vAqqSRaR/MZoQJo6yg6vuv4h7D1I2/8=", + "lastModified": 1726844355, + "narHash": "sha256-sbvo+lzHkAdQp7lum7/IbJtUMLJQSoERsQI2Zoytg7I=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b3e9ef326d3d60dd97c262c6d16cc255175d4902", + "rev": "ff5752aab38a462024e8fce1475e37dcbd0e7526", "type": "github" }, "original": { diff --git a/nix/home-manager/gui.nix b/nix/home-manager/gui.nix index db57f03..f04115d 100644 --- a/nix/home-manager/gui.nix +++ b/nix/home-manager/gui.nix @@ -8,7 +8,7 @@ imports = [ ./no-gui.nix # ./progs/librewolf.nix - ./progs/gpt4all.nix + ./progs/gpt4all/gpt4all.nix ]; nixpkgs.config.allowUnfreePredicate = diff --git a/nix/home-manager/progs/gpt4all.nix b/nix/home-manager/progs/gpt4all.nix deleted file mode 100644 index 49b3bee..0000000 --- a/nix/home-manager/progs/gpt4all.nix +++ /dev/null @@ -1,85 +0,0 @@ -{ pkgs, ... }: -{ - home.packages = - let - #stolen from: https://stackoverflow.com/a/42398526 - optimizeWithFlags = - pkg: flags: - pkgs.lib.overrideDerivation pkg ( - old: - let - newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags; - oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else ""; - in - { - NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}"; - stdenv = pkgs.clangStdenv; - } - ); - in - with pkgs; - [ - (optimizeWithFlags - (gpt4all.overrideAttrs { - src = fetchFromGitHub { - fetchSubmodules = true; - owner = "nomic-ai"; - repo = "gpt4all"; - rev = "HEAD"; - sha256 = "lGvxTOBg7/UgrCqeAFNFFXD9VjpUk3IVCktdUYuF6Eo="; - }; - patches = [ - ./gpt4all-HEAD-embeddings-model.patch - ./gpt4all-HEAD-disable-settings-err.patch - ]; - }) - [ - "-Ofast" - "-march=native" - "-mtune=native" - "-fno-protect-parens" - "-fno-finite-math-only" # https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461 - ] - ) - ]; - home.file.".config/nomic.ai/GPT4All.ini" = { - text = - let - system_prompt = "You are an expert AI assistant that explains your reasoning step by step. For each step, provide a title that describes what you're doing in that step, along with the content. Decide if you need another step or if you're ready to give the final answer. USE AS MANY REASONING STEPS AS POSSIBLE. AT LEAST 3. BE AWARE OF YOUR LIMITATIONS AS AN LLM AND WHAT YOU CAN AND CANNOT DO. EXPLORE ALTERNATE ANSWERS AND CONSIDER THAT YOUR ANSWER MAY BE WRONG. IDENTIFY POSSIBLE ERRORS IN YOUR REASONING AND WHERE SUCH ERRORS MAY BE. FULLY TEST ALL OTHER POSSIBILITIES. YOU CAN BE WRONG. WHEN YOU SAY YOU ARE RE-EXAMINING, ACTUALLY RE-EXAMINE, AND USE ANOTHER APPROACH TO DO SO. DO NOT JUST SAY YOU ARE RE-EXAMINING. SHOW ALL YOUR WORK. USE AT LEAST 3 METHODS TO DERIVE THE ANSWER. USE BEST PRACTICES. WORK FROM FIRST PRINCIPLES TO CREATE YOUR ANSWER."; - in - '' - [General] - chatTheme=Dark - height=940 - suggestionMode=Off - threadCount=8 - uniqueId=7096f2d2-448d-4272-a132-d37e77f8a781 - userDefaultModel=Qwen2.5-7B-Instruct-Q6_K_L.gguf - width=1472 - x=0 - y=0 - - [download] - lastVersionStarted=3.3.0-dev0 - - [model-Qwen2.5-7B-Instruct-Q6_K_L.gguf] - contextLength=32768 - filename=Qwen2.5-7B-Instruct-Q6_K_L.gguf - maxLength=32768 - promptBatchSize=512 - promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n - systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|>" - - [network] - isActive=true - usageStatsActive=true - ''; - }; - - home.file.".local/share/nomic.ai/GPT4All/Qwen2.5-7B-Instruct-Q6_K_L.gguf" = { - source = pkgs.fetchurl { - url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true"; - sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM="; - }; - }; -} diff --git a/nix/home-manager/progs/gpt4all-HEAD-disable-settings-err.patch b/nix/home-manager/progs/gpt4all/gpt4all-HEAD-disable-settings-err.patch similarity index 100% rename from nix/home-manager/progs/gpt4all-HEAD-disable-settings-err.patch rename to nix/home-manager/progs/gpt4all/gpt4all-HEAD-disable-settings-err.patch diff --git a/nix/home-manager/progs/gpt4all-HEAD-embeddings-model.patch b/nix/home-manager/progs/gpt4all/gpt4all-HEAD-embeddings-model.patch similarity index 100% rename from nix/home-manager/progs/gpt4all-HEAD-embeddings-model.patch rename to nix/home-manager/progs/gpt4all/gpt4all-HEAD-embeddings-model.patch diff --git a/nix/home-manager/progs/gpt4all/gpt4all.nix b/nix/home-manager/progs/gpt4all/gpt4all.nix new file mode 100644 index 0000000..bcf2ed9 --- /dev/null +++ b/nix/home-manager/progs/gpt4all/gpt4all.nix @@ -0,0 +1,103 @@ +{ pkgs, lib, ... }: +let + models = [ + { + name = "Qwen2.5-7B-Instruct-Q6_K_L.gguf"; + context_length = "32768"; + source = pkgs.fetchurl { + url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true"; + sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM="; + }; + } + ]; + + #stolen from: https://stackoverflow.com/a/42398526 + optimizeWithFlags = + pkg: flags: + pkgs.lib.overrideDerivation pkg ( + old: + let + newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags; + oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else ""; + in + { + NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}"; + stdenv = pkgs.clangStdenv; + } + ); + + model_files = builtins.listToAttrs ( + map (f: { + name = ".local/share/nomic.ai/GPT4All/${f.name}"; + value = { + source = f.source; + }; + }) models + ); +in +{ + home.packages = with pkgs; [ + (optimizeWithFlags + (gpt4all.overrideAttrs { + src = fetchFromGitHub { + fetchSubmodules = true; + owner = "nomic-ai"; + repo = "gpt4all"; + rev = "HEAD"; + sha256 = "lGvxTOBg7/UgrCqeAFNFFXD9VjpUk3IVCktdUYuF6Eo="; + }; + + patches = [ + ./gpt4all-HEAD-embeddings-model.patch + ./gpt4all-HEAD-disable-settings-err.patch + ]; + }) + # compile flags + [ + "-Ofast" + "-march=alderlake" + "-mtune=alderlake" + "-fno-protect-parens" + "-fno-finite-math-only" # https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461 + ] + ) + ]; + + home.file = lib.recursiveUpdate { + ".config/nomic.ai/GPT4All.ini".text = + let + system_prompt = "You are an expert AI assistant that explains your reasoning step by step. For each step, provide a title that describes what you're doing in that step, along with the content. Decide if you need another step or if you're ready to give the final answer. USE AS MANY REASONING STEPS AS POSSIBLE. AT LEAST 3. BE AWARE OF YOUR LIMITATIONS AS AN LLM AND WHAT YOU CAN AND CANNOT DO. EXPLORE ALTERNATE ANSWERS AND CONSIDER THAT YOUR ANSWER MAY BE WRONG. IDENTIFY POSSIBLE ERRORS IN YOUR REASONING AND WHERE SUCH ERRORS MAY BE. FULLY TEST ALL OTHER POSSIBILITIES. YOU CAN BE WRONG. WHEN YOU SAY YOU ARE RE-EXAMINING, ACTUALLY RE-EXAMINE, AND USE ANOTHER APPROACH TO DO SO. DO NOT JUST SAY YOU ARE RE-EXAMINING. SHOW ALL YOUR WORK. USE AT LEAST 3 METHODS TO DERIVE THE ANSWER. USE BEST PRACTICES. WORK FROM FIRST PRINCIPLES TO CREATE YOUR ANSWER."; + in + '' + [General] + chatTheme=Dark + height=940 + suggestionMode=Off + threadCount=8 + uniqueId=7096f2d2-448d-4272-a132-d37e77f8a781 + userDefaultModel=Qwen2.5-7B-Instruct-Q6_K_L.gguf + width=1472 + x=0 + y=0 + + [download] + lastVersionStarted=3.3.0-dev0 + + [network] + isActive=true + usageStatsActive=true + '' + + (lib.concatStrings ( + map (model: '' + [model-${model.name}] + contextLength=${model.context_length} + filename=${model.name} + maxLength=${model.context_length} + promptBatchSize=256 + promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n + systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|> + \n" + '') models + )); + } model_files; +}