alex/LocalAGI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KB: add webui sections

Browse Source
This commit is contained in:
Ettore Di Giacinto
2024-04-09 23:52:39 +02:00
parent 78ba7871e9
commit a1edf005a9
7 changed files with 281 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
example/webui/agentpool.go
View File

@@ -37,6 +37,8 @@ type AgentConfig struct {
IdentityGuidance string `json:"identity_guidance" form:"identity_guidance"`
PeriodicRuns string `json:"periodic_runs" form:"periodic_runs"`
PermanentGoal string `json:"permanent_goal" form:"permanent_goal"`
EnableKnowledgeBase bool `json:"enable_kb" form:"enable_kb"`
KnowledgeBaseResults int `json:"kb_results" form:"kb_results"`
}
type AgentPool struct {
@@ -286,6 +288,14 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig) error
}
}
if config.EnableKnowledgeBase {
opts = append(opts, EnableKnowledgeBase)
}
if config.KnowledgeBaseResults > 0 {
opts = append(opts, EnableKnowledgeBaseWithResults(config.KnowledgeBaseResults))
}
fmt.Println("Starting agent", name)
fmt.Printf("Config %+v\n", config)
agent, err := New(opts...)

8
example/webui/create.html
View File

@@ -84,7 +84,13 @@
<div class="mb-4">
</div>
<div class="mb-4">
<label for="enable_kb" class="block text-lg font-medium text-gray-400">Enable Knowledge Base</label>
<input type="checkbox" name="enable_kb" id="enable_kb" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white">
<div class="mb-6">
<label for="kb_results" class="block text-lg font-medium text-gray-400">Knowledge base results</label>
<input type="text" name="kb_results" id="kb_results" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="3">
</div>
<label for="debug" class="block text-lg font-medium text-gray-400">Debug</label>
<input type="checkbox" name="debug" id="debug" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white">
<label for="standalone_job" class="block text-lg font-medium text-gray-400">Standalone Job</label>

31
example/webui/knowledgebase.html Normal file
View File

@@ -0,0 +1,31 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>KnowledgeBase</title>
<link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
<script src="https://unpkg.com/htmx.org"></script>
<script src="https://unpkg.com/htmx.org/dist/ext/sse.js"></script>
<script src="https://unpkg.com/hyperscript.org@0.9.12"></script>
</head>
<body class="bg-gray-900 p-4 text-white">
<div class="max-w-2xl mx-auto my-12 bg-gray-800 p-8 rounded-lg shadow-lg">
<h1 class="text-3xl font-bold text-center mb-10 text-blue-400">Knowledgebase (items: {{.KnowledgebaseItemsCount}})</h1>
<form action="/knowledgebase" method="POST" class="space-y-6">
Add sites to KB
<div class="mb-6">
<label for="url" class="block text-lg font-medium text-gray-400">URL</label>
<input type="text" name="url" id="url" class="mt-1 focus:ring-indigo-500 focus:border-indigo-500 block w-full shadow-sm sm:text-lg border-gray-300 rounded-md bg-gray-700 text-white" placeholder="Name">
</div>
<div class="flex items-center justify-between">
<button type="submit" class="w-full flex justify-center py-2 px-4 border border-transparent rounded-md shadow-sm text-sm font-medium text-white bg-blue-500 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500">
Add Site
</button>
</div>
</form>
</div>
</body>
</html>

101
example/webui/main.go
View File

@@ -23,6 +23,7 @@ type (
var testModel = os.Getenv("TEST_MODEL")
var apiURL = os.Getenv("API_URL")
var apiKey = os.Getenv("API_KEY")
func init() {
if testModel == "" {
@@ -45,13 +46,25 @@ func main() {
if err != nil {
panic(err)
}
os.MkdirAll(cwd+"/pool", 0755)
pool, err := NewAgentPool(testModel, apiURL, cwd+"/pool")
stateDir := cwd + "/pool"
os.MkdirAll(stateDir, 0755)
pool, err := NewAgentPool(testModel, apiURL, stateDir)
if err != nil {
panic(err)
}
db, err := NewInMemoryDB(stateDir)
if err != nil {
panic(err)
}
// Reload store
// if err := db.SaveToStore(apiKey, apiURL); err != nil {
// fmt.Println("Error storing in the KB", err)
// }
app := &App{
htmx: htmx.New(),
pool: pool,
@@ -61,19 +74,6 @@ func main() {
panic(err)
}
// go func() {
// for {
// clientsStr := ""
// clients := sseManager.Clients()
// for _, c := range clients {
// clientsStr += c + ", "
// }
// time.Sleep(1 * time.Second) // Send a message every seconds
// sseManager.Send(NewMessage(fmt.Sprintf("connected clients: %v", clientsStr)).WithEvent("clients"))
// }
// }()
// Initialize a new Fiber app
webapp := fiber.New()
@@ -94,6 +94,13 @@ func main() {
})
})
webapp.Get("/knowledgebase", func(c *fiber.Ctx) error {
return c.Render("knowledgebase.html", fiber.Map{
"Title": "Hello, World!",
"KnowledgebaseItemsCount": len(db.Database),
})
})
// Define a route for the GET method on the root path '/'
webapp.Get("/sse/:name", func(c *fiber.Ctx) error {
@@ -110,6 +117,7 @@ func main() {
webapp.Post("/chat/:name", app.Chat(pool))
webapp.Post("/create", app.Create(pool))
webapp.Get("/delete/:name", app.Delete(pool))
webapp.Post("/knowledgebase", app.KnowledgeBase(db))
webapp.Get("/talk/:name", func(c *fiber.Ctx) error {
return c.Render("chat.html", fiber.Map{
@@ -119,29 +127,48 @@ func main() {
})
log.Fatal(webapp.Listen(":3000"))
// mux := http.NewServeMux()
// mux.Handle("GET /", http.HandlerFunc(app.Home(agent)))
// // External notifications (e.g. webhook)
// mux.Handle("POST /notify", http.HandlerFunc(app.Notify))
// // User chat
// mux.Handle("POST /chat", http.HandlerFunc(app.Chat(sseManager)))
// // Server Sent Events
// //mux.Handle("GET /sse", http.HandlerFunc(app.SSE))
// fmt.Print("Server started at http://localhost:3210")
// err = http.ListenAndServe(":3210", mux)
// log.Fatal(err)
}
// func (a *App) SSE(w http.ResponseWriter, r *http.Request) {
// cl := sse.NewClient(randStringRunes(10))
// sseManager.Handle(w, r, cl)
// }
func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error {
payload := struct {
URL string `json:"url"`
}{}
if err := c.BodyParser(&payload); err != nil {
return err
}
website := payload.URL
if website == "" {
return fmt.Errorf("please enter a URL")
}
go func() {
content, err := Sitemap(website)
if err != nil {
fmt.Println("Error walking sitemap for website", err)
}
fmt.Println("Found pages: ", len(content))
for _, c := range content {
chunks := splitParagraphIntoChunks(c, 256)
fmt.Println("chunks: ", len(chunks))
for _, chunk := range chunks {
db.AddEntry(chunk)
}
db.SaveDB()
}
if err := db.SaveToStore(apiKey, apiURL); err != nil {
fmt.Println("Error storing in the KB", err)
}
}()
return c.Redirect("/knowledgebase")
}
}
func (a *App) Notify(pool *AgentPool) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error {

156
example/webui/rag.go Normal file
View File

@@ -0,0 +1,156 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"jaytaylor.com/html2text"
"github.com/mudler/local-agent-framework/llm"
sitemap "github.com/oxffaa/gopher-parse-sitemap"
)
type InMemoryDatabase struct {
sync.Mutex
Database []string
path string
}
func loadDB(path string) ([]string, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
poolData := []string{}
err = json.Unmarshal(data, &poolData)
return poolData, err
}
func NewInMemoryDB(knowledgebase string) (*InMemoryDatabase, error) {
// if file exists, try to load an existing pool.
// if file does not exist, create a new pool.
poolfile := filepath.Join(knowledgebase, "knowledgebase.json")
if _, err := os.Stat(poolfile); err != nil {
// file does not exist, return a new pool
return &InMemoryDatabase{
Database: []string{},
path: poolfile,
}, nil
}
poolData, err := loadDB(poolfile)
if err != nil {
return nil, err
}
return &InMemoryDatabase{
Database: poolData,
path: poolfile,
}, nil
}
func (db *InMemoryDatabase) SaveToStore(apiKey string, apiURL string) error {
for _, d := range db.Database {
lai := llm.NewClient(apiKey, apiURL+"/v1")
laiStore := llm.NewStoreClient(apiURL, apiKey)
err := llm.StoreStringEmbeddingInVectorDB(laiStore, lai, d)
if err != nil {
return fmt.Errorf("Error storing in the KB: %w", err)
}
}
return nil
}
func (db *InMemoryDatabase) AddEntry(entry string) error {
db.Lock()
defer db.Unlock()
db.Database = append(db.Database, entry)
return nil
}
func (db *InMemoryDatabase) SaveDB() error {
db.Lock()
defer db.Unlock()
data, err := json.Marshal(db.Database)
if err != nil {
return err
}
err = os.WriteFile(db.path, data, 0644)
return err
}
func getWebPage(url string) (string, error) {
resp, err := http.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return html2text.FromString(string(body), html2text.Options{PrettyTables: true})
}
func Sitemap(url string) (res []string, err error) {
err = sitemap.ParseFromSite(url, func(e sitemap.Entry) error {
fmt.Println("Sitemap page: " + e.GetLocation())
content, err := getWebPage(e.GetLocation())
if err == nil {
res = append(res, content)
}
return nil
})
return
}
// splitParagraphIntoChunks takes a paragraph and a maxChunkSize as input,
// and returns a slice of strings where each string is a chunk of the paragraph
// that is at most maxChunkSize long, ensuring that words are not split.
func splitParagraphIntoChunks(paragraph string, maxChunkSize int) []string {
// Check if the paragraph length is less than or equal to maxChunkSize.
// If so, return the paragraph as the only chunk.
if len(paragraph) <= maxChunkSize {
return []string{paragraph}
}
var chunks []string
var currentChunk strings.Builder
words := strings.Fields(paragraph) // Splits the paragraph into words.
for _, word := range words {
// Check if adding the next word would exceed the maxChunkSize.
// If so, add the currentChunk to the chunks slice and start a new chunk.
if currentChunk.Len()+len(word) > maxChunkSize {
chunks = append(chunks, currentChunk.String())
currentChunk.Reset()
}
// Add a space before the word if it's not the beginning of a new chunk.
if currentChunk.Len() > 0 {
currentChunk.WriteString(" ")
}
// Add the word to the current chunk.
currentChunk.WriteString(word)
}
// Add the last chunk if it's not empty.
if currentChunk.Len() > 0 {
chunks = append(chunks, currentChunk.String())
}
return chunks
}