From 2e808c79ef776eaefad91c54915ead3236ec8b70 Mon Sep 17 00:00:00 2001
From: Yohan Lasorsa <noda@free.fr>
Date: Fri, 22 Mar 2024 10:54:05 +0100
Subject: [PATCH] fix: text splitter with tiny documents (fixes #193) (#196)

---
 packages/indexer/src/lib/document-processor.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/indexer/src/lib/document-processor.ts b/packages/indexer/src/lib/document-processor.ts
index 074b31a9..c867c94b 100644
--- a/packages/indexer/src/lib/document-processor.ts
+++ b/packages/indexer/src/lib/document-processor.ts
@@ -75,6 +75,10 @@ export class DocumentProcessor {
     let start = 0;
     let end = length;
 
+    if (end <= MAX_SECTION_LENGTH) {
+      return [{ content: allText, page: findPage(0) }];
+    }
+
     while (start + SECTION_OVERLAP < length) {
       let lastWord = -1;
       end = start + MAX_SECTION_LENGTH;