diff --git a/extra/ghcide-workaround.diff b/extra/ghcide-workaround.diff
new file mode 100644
index 00000000..e7275c20
--- /dev/null
+++ b/extra/ghcide-workaround.diff
@@ -0,0 +1,13 @@
+diff --git a/session-loader/Development/IDE/Session.hs b/session-loader/Development/IDE/Session.hs
+index bdd27f3d..2639631d 100644
+--- a/session-loader/Development/IDE/Session.hs
++++ b/session-loader/Development/IDE/Session.hs
+@@ -880,7 +880,7 @@ newComponentCache recorder exts cradlePath _cfp hsc_env old_cis new_cis = do
+               getSession
+ #endif
+       henv <- createHscEnvEq thisEnv (zip uids dfs)
+-      let targetEnv = (if isBad ci then multi_errs else [], Just henv)
++      let targetEnv = ([], Just henv)
+           targetDepends = componentDependencyInfo ci
+           res = ( targetEnv, targetDepends)
+       logWith recorder Debug $ LogNewComponentCache res
diff --git a/tool-map.nix b/tool-map.nix
index 1b7fa6f8..727fda90 100644
--- a/tool-map.nix
+++ b/tool-map.nix
@@ -41,6 +41,13 @@ compiler-nix-name: tool: {
       # `tool` normally ignores the `cabal.project` (if there is one in the hackage source).
       # We need to use the github one (since it has settings to make hls build).
       cabalProject = __readFile (src + "/cabal.project");
+      configureArgs = "--disable-benchmarks --disable-tests";
+      modules = [{
+        packages.ghcide.patches = [
+          # https://github.com/haskell/haskell-language-server/issues/4046#issuecomment-1926242056
+          ./extra/ghcide-workaround.diff
+        ];
+      }];
   };
   happy = { version = "1.20.1.1"; inherit cabalProjectLocal; };
   alex = { version = "3.2.7.3"; inherit cabalProjectLocal; };