alex/LocalAGI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KB: add webui sections

Browse Source
This commit is contained in:
Ettore Di Giacinto
2024-04-09 23:52:39 +02:00
parent 78ba7871e9
commit a1edf005a9
7 changed files with 281 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
example/webui/agentpool.go
View File

@@ -37,6 +37,8 @@ type AgentConfig struct {
IdentityGuidance string `json:"identity_guidance" form:"identity_guidance"` IdentityGuidance string `json:"identity_guidance" form:"identity_guidance"`
PeriodicRuns string `json:"periodic_runs" form:"periodic_runs"` PeriodicRuns string `json:"periodic_runs" form:"periodic_runs"`
PermanentGoal string `json:"permanent_goal" form:"permanent_goal"` PermanentGoal string `json:"permanent_goal" form:"permanent_goal"`
EnableKnowledgeBase bool `json:"enable_kb" form:"enable_kb"`
KnowledgeBaseResults int `json:"kb_results" form:"kb_results"`
} }
type AgentPool struct { type AgentPool struct {
@@ -286,6 +288,14 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig) error
} }
} }
if config.EnableKnowledgeBase {
opts = append(opts, EnableKnowledgeBase)
}
if config.KnowledgeBaseResults > 0 {
opts = append(opts, EnableKnowledgeBaseWithResults(config.KnowledgeBaseResults))
}
fmt.Println("Starting agent", name) fmt.Println("Starting agent", name)
fmt.Printf("Config %+v\n", config) fmt.Printf("Config %+v\n", config)
agent, err := New(opts...) agent, err := New(opts...)

6
example/webui/create.html
View File

@@ -84,6 +84,12 @@
<div class="mb-4"> <div class="mb-4">
</div> </div>
<div class="mb-4"> <div class="mb-4">
<label for="enable_kb" class="block text-lg font-medium text-gray-400">Enable Knowledge Base</label>
<input type="checkbox" name="enable_kb" id="enable_kb" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white">
<div class="mb-6">
<label for="kb_results" class="block text-lg font-medium text-gray-400">Knowledge base results</label>
<input type="text" name="kb_results" id="kb_results" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="3">
</div>
<label for="debug" class="block text-lg font-medium text-gray-400">Debug</label> <label for="debug" class="block text-lg font-medium text-gray-400">Debug</label>
<input type="checkbox" name="debug" id="debug" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white"> <input type="checkbox" name="debug" id="debug" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white">

31
example/webui/knowledgebase.html Normal file
View File

@@ -0,0 +1,31 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>KnowledgeBase</title>
<link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
<script src="https://unpkg.com/htmx.org"></script>
<script src="https://unpkg.com/htmx.org/dist/ext/sse.js"></script>
<script src="https://unpkg.com/hyperscript.org@0.9.12"></script>
</head>
<body class="bg-gray-900 p-4 text-white">
<div class="max-w-2xl mx-auto my-12 bg-gray-800 p-8 rounded-lg shadow-lg">
<h1 class="text-3xl font-bold text-center mb-10 text-blue-400">Knowledgebase (items: {{.KnowledgebaseItemsCount}})</h1>
<form action="/knowledgebase" method="POST" class="space-y-6">
Add sites to KB
<div class="mb-6">
<label for="url" class="block text-lg font-medium text-gray-400">URL</label>
<input type="text" name="url" id="url" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="Name">
</div>
<div class="flex items-center justify-between">
<button type="submit" class="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-500 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500">
Add Site
</button>
</div>
</form>
</div>
</body>
</html>

101
example/webui/main.go
View File

@@ -23,6 +23,7 @@ type (
var testModel = os.Getenv("TEST_MODEL") var testModel = os.Getenv("TEST_MODEL")
var apiURL = os.Getenv("API_URL") var apiURL = os.Getenv("API_URL")
var apiKey = os.Getenv("API_KEY")
func init() { func init() {
if testModel == "" { if testModel == "" {
@@ -45,13 +46,25 @@ func main() {
if err != nil { if err != nil {
panic(err) panic(err)
} }
os.MkdirAll(cwd+"/pool", 0755)
pool, err := NewAgentPool(testModel, apiURL, cwd+"/pool") stateDir := cwd + "/pool"
os.MkdirAll(stateDir, 0755)
pool, err := NewAgentPool(testModel, apiURL, stateDir)
if err != nil { if err != nil {
panic(err) panic(err)
} }
db, err := NewInMemoryDB(stateDir)
if err != nil {
panic(err)
}
// Reload store
// if err := db.SaveToStore(apiKey, apiURL); err != nil {
// fmt.Println("Error storing in the KB", err)
// }
app := &App{ app := &App{
htmx: htmx.New(), htmx: htmx.New(),
pool: pool, pool: pool,
@@ -61,19 +74,6 @@ func main() {
panic(err) panic(err)
} }
// go func() {
// for {
// clientsStr := ""
// clients := sseManager.Clients()
// for _, c := range clients {
// clientsStr += c + ", "
// }
// time.Sleep(1 * time.Second) // Send a message every seconds
// sseManager.Send(NewMessage(fmt.Sprintf("connected clients: %v", clientsStr)).WithEvent("clients"))
// }
// }()
// Initialize a new Fiber app // Initialize a new Fiber app
webapp := fiber.New() webapp := fiber.New()
@@ -94,6 +94,13 @@ func main() {
}) })
}) })
webapp.Get("/knowledgebase", func(c *fiber.Ctx) error {
return c.Render("knowledgebase.html", fiber.Map{
"Title": "Hello, World!",
"KnowledgebaseItemsCount": len(db.Database),
})
})
// Define a route for the GET method on the root path '/' // Define a route for the GET method on the root path '/'
webapp.Get("/sse/:name", func(c *fiber.Ctx) error { webapp.Get("/sse/:name", func(c *fiber.Ctx) error {
@@ -110,6 +117,7 @@ func main() {
webapp.Post("/chat/:name", app.Chat(pool)) webapp.Post("/chat/:name", app.Chat(pool))
webapp.Post("/create", app.Create(pool)) webapp.Post("/create", app.Create(pool))
webapp.Get("/delete/:name", app.Delete(pool)) webapp.Get("/delete/:name", app.Delete(pool))
webapp.Post("/knowledgebase", app.KnowledgeBase(db))
webapp.Get("/talk/:name", func(c *fiber.Ctx) error { webapp.Get("/talk/:name", func(c *fiber.Ctx) error {
return c.Render("chat.html", fiber.Map{ return c.Render("chat.html", fiber.Map{
@@ -119,29 +127,48 @@ func main() {
}) })
log.Fatal(webapp.Listen(":3000")) log.Fatal(webapp.Listen(":3000"))
// mux := http.NewServeMux()
// mux.Handle("GET /", http.HandlerFunc(app.Home(agent)))
// // External notifications (e.g. webhook)
// mux.Handle("POST /notify", http.HandlerFunc(app.Notify))
// // User chat
// mux.Handle("POST /chat", http.HandlerFunc(app.Chat(sseManager)))
// // Server Sent Events
// //mux.Handle("GET /sse", http.HandlerFunc(app.SSE))
// fmt.Print("Server started at http://localhost:3210")
// err = http.ListenAndServe(":3210", mux)
// log.Fatal(err)
} }
// func (a *App) SSE(w http.ResponseWriter, r *http.Request) { func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
// cl := sse.NewClient(randStringRunes(10)) return func(c *fiber.Ctx) error {
// sseManager.Handle(w, r, cl) payload := struct {
// } URL string `json:"url"`
}{}
if err := c.BodyParser(&payload); err != nil {
return err
}
website := payload.URL
if website == "" {
return fmt.Errorf("please enter a URL")
}
go func() {
content, err := Sitemap(website)
if err != nil {
fmt.Println("Error walking sitemap for website", err)
}
fmt.Println("Found pages: ", len(content))
for _, c := range content {
chunks := splitParagraphIntoChunks(c, 256)
fmt.Println("chunks: ", len(chunks))
for _, chunk := range chunks {
db.AddEntry(chunk)
}
db.SaveDB()
}
if err := db.SaveToStore(apiKey, apiURL); err != nil {
fmt.Println("Error storing in the KB", err)
}
}()
return c.Redirect("/knowledgebase")
}
}
func (a *App) Notify(pool *AgentPool) func(c *fiber.Ctx) error { func (a *App) Notify(pool *AgentPool) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error { return func(c *fiber.Ctx) error {

156
example/webui/rag.go Normal file
View File

@@ -0,0 +1,156 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"jaytaylor.com/html2text"
"github.com/mudler/local-agent-framework/llm"
sitemap "github.com/oxffaa/gopher-parse-sitemap"
)
type InMemoryDatabase struct {
sync.Mutex
Database []string
path string
}
func loadDB(path string) ([]string, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
poolData := []string{}
err = json.Unmarshal(data, &poolData)
return poolData, err
}
func NewInMemoryDB(knowledgebase string) (*InMemoryDatabase, error) {
// if file exists, try to load an existing pool.
// if file does not exist, create a new pool.
poolfile := filepath.Join(knowledgebase, "knowledgebase.json")
if _, err := os.Stat(poolfile); err != nil {
// file does not exist, return a new pool
return &InMemoryDatabase{
Database: []string{},
path: poolfile,
}, nil
}
poolData, err := loadDB(poolfile)
if err != nil {
return nil, err
}
return &InMemoryDatabase{
Database: poolData,
path: poolfile,
}, nil
}
func (db *InMemoryDatabase) SaveToStore(apiKey string, apiURL string) error {
for _, d := range db.Database {
lai := llm.NewClient(apiKey, apiURL+"/v1")
laiStore := llm.NewStoreClient(apiURL, apiKey)
err := llm.StoreStringEmbeddingInVectorDB(laiStore, lai, d)
if err != nil {
return fmt.Errorf("Error storing in the KB: %w", err)
}
}
return nil
}
func (db *InMemoryDatabase) AddEntry(entry string) error {
db.Lock()
defer db.Unlock()
db.Database = append(db.Database, entry)
return nil
}
func (db *InMemoryDatabase) SaveDB() error {
db.Lock()
defer db.Unlock()
data, err := json.Marshal(db.Database)
if err != nil {
return err
}
err = os.WriteFile(db.path, data, 0644)
return err
}
func getWebPage(url string) (string, error) {
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return html2text.FromString(string(body), html2text.Options{PrettyTables: true})
}
func Sitemap(url string) (res []string, err error) {
err = sitemap.ParseFromSite(url, func(e sitemap.Entry) error {
fmt.Println("Sitemap page: " + e.GetLocation())
content, err := getWebPage(e.GetLocation())
if err == nil {
res = append(res, content)
}
return nil
})
return
}
// splitParagraphIntoChunks takes a paragraph and a maxChunkSize as input,
// and returns a slice of strings where each string is a chunk of the paragraph
// that is at most maxChunkSize long, ensuring that words are not split.
func splitParagraphIntoChunks(paragraph string, maxChunkSize int) []string {
// Check if the paragraph length is less than or equal to maxChunkSize.
// If so, return the paragraph as the only chunk.
if len(paragraph) <= maxChunkSize {
return []string{paragraph}
}
var chunks []string
var currentChunk strings.Builder
words := strings.Fields(paragraph) // Splits the paragraph into words.
for _, word := range words {
// Check if adding the next word would exceed the maxChunkSize.
// If so, add the currentChunk to the chunks slice and start a new chunk.
if currentChunk.Len()+len(word) > maxChunkSize {
chunks = append(chunks, currentChunk.String())
currentChunk.Reset()
}
// Add a space before the word if it's not the beginning of a new chunk.
if currentChunk.Len() > 0 {
currentChunk.WriteString(" ")
}
// Add the word to the current chunk.
currentChunk.WriteString(word)
}
// Add the last chunk if it's not empty.
if currentChunk.Len() > 0 {
chunks = append(chunks, currentChunk.String())
}
return chunks
}

4
go.mod
View File

@@ -33,7 +33,10 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 // indirect
github.com/rivo/uniseg v0.2.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/stretchr/testify v1.9.0 // indirect github.com/stretchr/testify v1.9.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect
@@ -43,4 +46,5 @@ require (
golang.org/x/text v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.1 // indirect golang.org/x/tools v0.16.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
) )

9
go.sum
View File

@@ -48,12 +48,17 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 h1:2vmb32OdDhjZf2ETGDlr9n8RYXx7c+jXPxMiPbwnA+8=
github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4/go.mod h1:2JQx4jDHmWrbABvpOayg/+OTU6ehN0IyK2EHzceXpJo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
@@ -64,6 +69,8 @@ github.com/sashabaranov/go-openai v1.18.3 h1:dspFGkmZbhjg1059KhqLYSV2GaCiRIn+bOu
github.com/sashabaranov/go-openai v1.18.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sashabaranov/go-openai v1.18.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/slack-go/slack v0.12.5 h1:ddZ6uz6XVaB+3MTDhoW04gG+Vc/M/X1ctC+wssy2cqs= github.com/slack-go/slack v0.12.5 h1:ddZ6uz6XVaB+3MTDhoW04gG+Vc/M/X1ctC+wssy2cqs=
github.com/slack-go/slack v0.12.5/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/slack-go/slack v0.12.5/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -107,3 +114,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056 h1:6YFJoB+0fUH6X3xU/G2tQqCYg+PkGtnZ5nMR5rpw72g=
jaytaylor.com/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4=