alex/LocalAGI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KB: Specify chunk size

Browse Source
This commit is contained in:
Ettore Di Giacinto
2024-04-09 23:54:58 +02:00
parent a1edf005a9
commit db490fb3ca
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
example/webui/knowledgebase.html
View File

@@ -18,6 +18,10 @@
<label for="url" class="block text-lg font-medium text-gray-400">URL</label> <label for="url" class="block text-lg font-medium text-gray-400">URL</label>
<input type="text" name="url" id="url" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="Name"> <input type="text" name="url" id="url" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="Name">
</div> </div>
<div class="mb-6">
<label for="chunk_size" class="block text-lg font-medium text-gray-400">Chunk size</label>
<input type="text" name="chunk_size" id="chunk_size" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="380">
</div>
<div class="flex items-center justify-between"> <div class="flex items-center justify-between">

5
example/webui/main.go
View File

@@ -132,7 +132,8 @@ func main() {
func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error { func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error { return func(c *fiber.Ctx) error {
payload := struct { payload := struct {
URL string `json:"url"` URL string `json:"url"`
ChunkSize int `json:"chunk_size"`
}{} }{}
if err := c.BodyParser(&payload); err != nil { if err := c.BodyParser(&payload); err != nil {
@@ -152,7 +153,7 @@ func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
fmt.Println("Found pages: ", len(content)) fmt.Println("Found pages: ", len(content))
for _, c := range content { for _, c := range content {
chunks := splitParagraphIntoChunks(c, 256) chunks := splitParagraphIntoChunks(c, payload.ChunkSize)
fmt.Println("chunks: ", len(chunks)) fmt.Println("chunks: ", len(chunks))
for _, chunk := range chunks { for _, chunk := range chunks {
db.AddEntry(chunk) db.AddEntry(chunk)