gpt4all: overhaul

nix/home-manager/progs/gpt4all/gpt4all-HEAD-disable-settings-err.patch

@@ -0,0 +1,19 @@
+diff --git a/gpt4all-chat/src/main.qml b/gpt4all-chat/src/main.qml
+index 1e685385..7c747eba 100644
+--- a/src/main.qml
++++ b/src/main.qml
+@@ -72,12 +72,14 @@ Window {
+             return;
+         }
+ 
++        /*
+         // check if we have access to settings and if not show an error
+         if (!hasShownSettingsAccess && !LLM.hasSettingsAccess()) {
+             errorSettingsAccess.open();
+             hasShownSettingsAccess = true;
+             return;
+         }
++        */
+ 
+         // check for first time start of this version
+         if (!hasCheckedFirstStart) {

nix/home-manager/progs/gpt4all/gpt4all-HEAD-embeddings-model.patch

@@ -0,0 +1,53 @@
+commit 425b33877c819dd88f3692aae37452c767371f6b
+Author: Simon Gardling <titaniumtown@proton.me>
+Date:   Thu Sep 19 10:00:39 2024 -0400
+
+    use locally downloaded embeddings
+
+diff --git a/gpt4all-chat/CMakeLists.txt b/gpt4all-chat/CMakeLists.txt
+index 900307ae..802fc31a 100644
+--- a//CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -120,6 +120,7 @@ elseif (APPLE)
+ endif()
+ 
+ # Embedding model
++#[[
+ set(LOCAL_EMBEDDING_MODEL "nomic-embed-text-v1.5.f16.gguf")
+ set(LOCAL_EMBEDDING_MODEL_MD5 "a5401e7f7e46ed9fcaed5b60a281d547")
+ set(LOCAL_EMBEDDING_MODEL_PATH "${CMAKE_BINARY_DIR}/resources/${LOCAL_EMBEDDING_MODEL}")
+@@ -134,6 +135,7 @@ message(STATUS "Embedding model downloaded to ${LOCAL_EMBEDDING_MODEL_PATH}")
+ if (APPLE)
+     list(APPEND CHAT_EXE_RESOURCES "${LOCAL_EMBEDDING_MODEL_PATH}")
+ endif()
++]]
+ 
+ set(QAPPLICATION_CLASS QGuiApplication)
+ add_subdirectory(deps/SingleApplication)
+@@ -348,11 +350,13 @@ if (LLMODEL_CUDA)
+     endif()
+ endif()
+ 
++#[[
+ if (NOT APPLE)
+     install(FILES "${LOCAL_EMBEDDING_MODEL_PATH}"
+             DESTINATION resources
+             COMPONENT ${COMPONENT_NAME_MAIN})
+ endif()
++]]
+ 
+ set(CPACK_GENERATOR "IFW")
+ set(CPACK_VERBATIM_VARIABLES YES)
+diff --git a/gpt4all-chat/src/embllm.cpp b/gpt4all-chat/src/embllm.cpp
+index 81b1e9e1..e3266cc7 100644
+--- a/src/embllm.cpp
++++ b/src/embllm.cpp
+@@ -84,7 +84,7 @@ bool EmbeddingLLMWorker::loadModel()
+ 
+     QString filePath = embPathFmt.arg(QCoreApplication::applicationDirPath(), LOCAL_EMBEDDING_MODEL);
+     if (!QFileInfo::exists(filePath)) {
+-        qWarning() << "embllm WARNING: Local embedding model not found";
++        qWarning() << "embllm WARNING: Local embedding model not found: " << filePath;
+         return false;
+     }
+ 

nix/home-manager/progs/gpt4all/gpt4all.nix

@@ -0,0 +1,103 @@
+{ pkgs, lib, ... }:
+let
+  models = [
+    {
+      name = "Qwen2.5-7B-Instruct-Q6_K_L.gguf";
+      context_length = "32768";
+      source = pkgs.fetchurl {
+        url = "https://huggingface.co/bartowski/Qwen2.5-7B-Instruct-GGUF/resolve/main/Qwen2.5-7B-Instruct-Q6_K_L.gguf?download=true";
+        sha256 = "thEXN06T/UVGfzdB83jlgpG7kuTzZtz1ZUAdupAnErM=";
+      };
+    }
+  ];
+
+  #stolen from: https://stackoverflow.com/a/42398526
+  optimizeWithFlags =
+    pkg: flags:
+    pkgs.lib.overrideDerivation pkg (
+      old:
+      let
+        newflags = pkgs.lib.foldl' (acc: x: "${acc} ${x}") "" flags;
+        oldflags = if (pkgs.lib.hasAttr "NIX_CFLAGS_COMPILE" old) then "${old.NIX_CFLAGS_COMPILE}" else "";
+      in
+      {
+        NIX_CFLAGS_COMPILE = "${oldflags} ${newflags}";
+        stdenv = pkgs.clangStdenv;
+      }
+    );
+
+  model_files = builtins.listToAttrs (
+    map (f: {
+      name = ".local/share/nomic.ai/GPT4All/${f.name}";
+      value = {
+        source = f.source;
+      };
+    }) models
+  );
+in
+{
+  home.packages = with pkgs; [
+    (optimizeWithFlags
+      (gpt4all.overrideAttrs {
+        src = fetchFromGitHub {
+          fetchSubmodules = true;
+          owner = "nomic-ai";
+          repo = "gpt4all";
+          rev = "HEAD";
+          sha256 = "lGvxTOBg7/UgrCqeAFNFFXD9VjpUk3IVCktdUYuF6Eo=";
+        };
+
+        patches = [
+          ./gpt4all-HEAD-embeddings-model.patch
+          ./gpt4all-HEAD-disable-settings-err.patch
+        ];
+      })
+      # compile flags
+      [
+        "-Ofast"
+        "-march=alderlake"
+        "-mtune=alderlake"
+        "-fno-protect-parens"
+        "-fno-finite-math-only" # https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461
+      ]
+    )
+  ];
+
+  home.file = lib.recursiveUpdate {
+    ".config/nomic.ai/GPT4All.ini".text =
+      let
+        system_prompt = "You are an expert AI assistant that explains your reasoning step by step. For each step, provide a title that describes what you're doing in that step, along with the content. Decide if you need another step or if you're ready to give the final answer. USE AS MANY REASONING STEPS AS POSSIBLE. AT LEAST 3. BE AWARE OF YOUR LIMITATIONS AS AN LLM AND WHAT YOU CAN AND CANNOT DO. EXPLORE ALTERNATE ANSWERS AND CONSIDER THAT YOUR ANSWER MAY BE WRONG. IDENTIFY POSSIBLE ERRORS IN YOUR REASONING AND WHERE SUCH ERRORS MAY BE. FULLY TEST ALL OTHER POSSIBILITIES. YOU CAN BE WRONG. WHEN YOU SAY YOU ARE RE-EXAMINING, ACTUALLY RE-EXAMINE, AND USE ANOTHER APPROACH TO DO SO. DO NOT JUST SAY YOU ARE RE-EXAMINING. SHOW ALL YOUR WORK. USE AT LEAST 3 METHODS TO DERIVE THE ANSWER. USE BEST PRACTICES. WORK FROM FIRST PRINCIPLES TO CREATE YOUR ANSWER.";
+      in
+      ''
+        [General]
+        chatTheme=Dark
+        height=940
+        suggestionMode=Off
+        threadCount=8
+        uniqueId=7096f2d2-448d-4272-a132-d37e77f8a781
+        userDefaultModel=Qwen2.5-7B-Instruct-Q6_K_L.gguf
+        width=1472
+        x=0
+        y=0
+
+        [download]
+        lastVersionStarted=3.3.0-dev0
+
+        [network]
+        isActive=true
+        usageStatsActive=true
+      ''
+      + (lib.concatStrings (
+        map (model: ''
+          [model-${model.name}]
+          contextLength=${model.context_length}
+          filename=${model.name}
+          maxLength=${model.context_length}
+          promptBatchSize=256
+          promptTemplate=<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n
+          systemPrompt="<|im_start|>system\n${system_prompt}<|im_end|>
+          \n"
+        '') models
+      ));
+  } model_files;
+}