From db490fb3ca10a2a20a2c6698f51fd2aeab32d7aa Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Tue, 9 Apr 2024 23:54:58 +0200
Subject: [PATCH] KB: Specify chunk size

---
 example/webui/knowledgebase.html | 4 ++++
 example/webui/main.go            | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/example/webui/knowledgebase.html b/example/webui/knowledgebase.html
index 33fd82c..ec8f5a8 100644
--- a/example/webui/knowledgebase.html
+++ b/example/webui/knowledgebase.html
@@ -18,6 +18,10 @@
             <label for="url" class="block text-lg font-medium text-gray-400">URL</label>
             <input type="text" name="url" id="url" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="Name">
         </div>
+        <div class="mb-6">
+            <label for="chunk_size" class="block text-lg font-medium text-gray-400">Chunk size</label>
+            <input type="text" name="chunk_size" id="chunk_size" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="380">
+        </div>
 
     
         <div class="flex items-center justify-between">
diff --git a/example/webui/main.go b/example/webui/main.go
index 42a7fd2..4fc6d95 100644
--- a/example/webui/main.go
+++ b/example/webui/main.go
@@ -132,7 +132,8 @@ func main() {
 func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 		payload := struct {
-			URL string `json:"url"`
+			URL       string `json:"url"`
+			ChunkSize int    `json:"chunk_size"`
 		}{}
 
 		if err := c.BodyParser(&payload); err != nil {
@@ -152,7 +153,7 @@ func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
 			fmt.Println("Found pages: ", len(content))
 
 			for _, c := range content {
-				chunks := splitParagraphIntoChunks(c, 256)
+				chunks := splitParagraphIntoChunks(c, payload.ChunkSize)
 				fmt.Println("chunks: ", len(chunks))
 				for _, chunk := range chunks {
 					db.AddEntry(chunk)