alex/LocalAGI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add an index

Browse Source
This commit is contained in:
mudler
2024-04-10 19:40:39 +02:00
parent 7c70f09834
commit 82ac74ac5d
7 changed files with 111 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
agent/agent.go
View File

@@ -176,6 +176,8 @@ func (a *Agent) consumeJob(job *Job, role string) {
// We are self evaluating if we consume the job as a system role // We are self evaluating if we consume the job as a system role
selfEvaluation := role == SystemRole selfEvaluation := role == SystemRole
memory := a.options.enableKB && a.options.ragdb != nil
a.Lock() a.Lock()
// Set the action context // Set the action context
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -206,7 +208,7 @@ func (a *Agent) consumeJob(job *Job, role string) {
} }
// RAG // RAG
if a.options.enableKB && a.options.ragdb != nil { if memory {
// Walk conversation from bottom to top, and find the first message of the user // Walk conversation from bottom to top, and find the first message of the user
// to use it as a query to the KB // to use it as a query to the KB
var userMessage string var userMessage string

63
example/webui/agents.html Normal file
View File

@@ -0,0 +1,63 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Agent List</title>
<link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
</head>
<body class="bg-gray-900 p-4 text-white">
<div class="max-w-6xl mx-auto">
<div class="text-center mb-6">
<h1 class="text-2xl font-bold">Smart Agent List</h1>
</div>
<div class="flex flex-col">
<div class="-my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
<div class="py-2 align-middle inline-block min-w-full sm:px-6 lg:px-8">
<div class="shadow overflow-hidden border-b border-gray-700 sm:rounded-lg">
<table class="min-w-full divide-y divide-gray-700">
<thead class="bg-gray-700">
<tr>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-400 uppercase tracking-wider">
Name
</th>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-400 uppercase tracking-wider">
Status
</th>
<th scope="col" class="relative px-6 py-3">
<span class="sr-only">Talk</span>
</th>
<th scope="col" class="relative px-6 py-3">
<span class="sr-only">Delete</span>
</th>
</tr>
</thead>
<tbody class="bg-gray-800 divide-y divide-gray-700">
<!-- Dynamic agent rows go here -->
{{ range .Agents }}
<tr>
<td class="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-300">{{.}}</td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-300">Online</td>
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
<a href="/talk/{{.}}" class="text-blue-500 hover:text-blue-400">Talk</a>
</td>
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
<a href="/delete/{{.}}" class="text-blue-500 hover:text-blue-400">Delete</a>
</td>
</tr>
{{ end }}
<!-- Repeat for each agent -->
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="mt-6">
<a href="/create" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
Add New Agent
</a>
</div>
</div>
</body>
</html>

60
example/webui/index.html
View File

@@ -2,60 +2,22 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<title>Agent List</title> <title>Smart Assistant Dashboard</title>
<link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet"> <link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
</head> </head>
<body class="bg-gray-900 p-4 text-white"> <body class="bg-gray-900 p-4 text-white">
<div class="max-w-6xl mx-auto"> <div class="max-w-4xl mx-auto">
<div class="text-center mb-6"> <div class="text-center mb-6">
<h1 class="text-2xl font-bold">Smart Agent List</h1> <h1 class="text-3xl font-bold">Smart Assistant Dashboard</h1>
</div> </div>
<div class="flex flex-col"> <div class="flex flex-col items-center space-y-4">
<div class="-my-2 overflow-x-auto sm:-mx-6 lg:-mx-8"> <!-- Button to Agent List Page -->
<div class="py-2 align-middle inline-block min-w-full sm:px-6 lg:px-8"> <a href="/agents" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
<div class="shadow overflow-hidden border-b border-gray-700 sm:rounded-lg"> Agent List
<table class="min-w-full divide-y divide-gray-700"> </a>
<thead class="bg-gray-700"> <!-- Button to Knowledgebase Management Page -->
<tr> <a href="/knowledgebase" class="bg-green-500 hover:bg-green-700 text-white font-bold py-2 px-4 rounded">
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-400 uppercase tracking-wider"> Manage Knowledgebase
Name
</th>
<th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-400 uppercase tracking-wider">
Status
</th>
<th scope="col" class="relative px-6 py-3">
<span class="sr-only">Talk</span>
</th>
<th scope="col" class="relative px-6 py-3">
<span class="sr-only">Delete</span>
</th>
</tr>
</thead>
<tbody class="bg-gray-800 divide-y divide-gray-700">
<!-- Dynamic agent rows go here -->
{{ range .Agents }}
<tr>
<td class="px-6 py-4 whitespace-nowrap text-sm font-medium text-gray-300">{{.}}</td>
<td class="px-6 py-4 whitespace-nowrap text-sm text-gray-300">Online</td>
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
<a href="/talk/{{.}}" class="text-blue-500 hover:text-blue-400">Talk</a>
</td>
<td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
<a href="/delete/{{.}}" class="text-blue-500 hover:text-blue-400">Delete</a>
</td>
</tr>
{{ end }}
<!-- Repeat for each agent -->
</tbody>
</table>
</div>
</div>
</div>
</div>
<div class="mt-6">
<a href="/create" class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
Add New Agent
</a> </a>
</div> </div>
</div> </div>

35
example/webui/main.go
View File

@@ -106,6 +106,12 @@ func main() {
}) })
}) })
webapp.Get("/agents", func(c *fiber.Ctx) error {
return c.Render("agents.html", fiber.Map{
"Agents": pool.List(),
})
})
webapp.Get("/create", func(c *fiber.Ctx) error { webapp.Get("/create", func(c *fiber.Ctx) error {
return c.Render("create.html", fiber.Map{ return c.Render("create.html", fiber.Map{
"Title": "Hello, World!", "Title": "Hello, World!",
@@ -152,8 +158,8 @@ func main() {
func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error { func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error { return func(c *fiber.Ctx) error {
payload := struct { payload := struct {
URL string `json:"url"` URL string `form:"url"`
ChunkSize int `json:"chunk_size"` ChunkSize int `form:"chunk_size"`
}{} }{}
if err := c.BodyParser(&payload); err != nil { if err := c.BodyParser(&payload); err != nil {
@@ -169,28 +175,7 @@ func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
chunkSize = payload.ChunkSize chunkSize = payload.ChunkSize
} }
go func() { go WebsiteToKB(website, chunkSize, db)
content, err := Sitemap(website)
if err != nil {
fmt.Println("Error walking sitemap for website", err)
}
fmt.Println("Found pages: ", len(content))
fmt.Println("ChunkSize: ", chunkSize)
for _, c := range content {
chunks := splitParagraphIntoChunks(c, chunkSize)
fmt.Println("chunks: ", len(chunks))
for _, chunk := range chunks {
db.AddEntry(chunk)
}
db.SaveDB()
}
if err := db.SaveToStore(); err != nil {
fmt.Println("Error storing in the KB", err)
}
}()
return c.Redirect("/knowledgebase") return c.Redirect("/knowledgebase")
} }
@@ -199,7 +184,7 @@ func (a *App) KnowledgeBase(db *InMemoryDatabase) func(c *fiber.Ctx) error {
func (a *App) Notify(pool *AgentPool) func(c *fiber.Ctx) error { func (a *App) Notify(pool *AgentPool) func(c *fiber.Ctx) error {
return func(c *fiber.Ctx) error { return func(c *fiber.Ctx) error {
payload := struct { payload := struct {
Message string `json:"message"` Message string `form:"message"`
}{} }{}
if err := c.BodyParser(&payload); err != nil { if err := c.BodyParser(&payload); err != nil {

24
example/webui/rag.go
View File

@@ -119,6 +119,30 @@ func Sitemap(url string) (res []string, err error) {
return return
} }
func WebsiteToKB(website string, chunkSize int, db *InMemoryDatabase) {
content, err := Sitemap(website)
if err != nil {
fmt.Println("Error walking sitemap for website", err)
}
fmt.Println("Found pages: ", len(content))
fmt.Println("ChunkSize: ", chunkSize)
for _, c := range content {
chunks := splitParagraphIntoChunks(c, chunkSize)
fmt.Println("chunks: ", len(chunks))
for _, chunk := range chunks {
fmt.Println("Chunk size: ", len(chunk))
db.AddEntry(chunk)
}
db.SaveDB()
}
if err := db.SaveToStore(); err != nil {
fmt.Println("Error storing in the KB", err)
}
}
// splitParagraphIntoChunks takes a paragraph and a maxChunkSize as input, // splitParagraphIntoChunks takes a paragraph and a maxChunkSize as input,
// and returns a slice of strings where each string is a chunk of the paragraph // and returns a slice of strings where each string is a chunk of the paragraph
// that is at most maxChunkSize long, ensuring that words are not split. // that is at most maxChunkSize long, ensuring that words are not split.

2
llm/rag/rag_chromem.go → llm/rag/chromem.go
View File

@@ -24,7 +24,6 @@ func NewChromemDB(collection, path string, openaiClient *openai.Client) (*Chrome
embeddingFunc := chromem.EmbeddingFunc( embeddingFunc := chromem.EmbeddingFunc(
func(ctx context.Context, text string) ([]float32, error) { func(ctx context.Context, text string) ([]float32, error) {
fmt.Println("Creating embeddings")
resp, err := openaiClient.CreateEmbeddings(ctx, resp, err := openaiClient.CreateEmbeddings(ctx,
openai.EmbeddingRequestStrings{ openai.EmbeddingRequestStrings{
Input: []string{text}, Input: []string{text},
@@ -64,7 +63,6 @@ func (c *ChromemDB) Store(s string) error {
if s == "" { if s == "" {
return fmt.Errorf("empty string") return fmt.Errorf("empty string")
} }
fmt.Println("Trying to store", s)
return c.collection.AddDocuments(context.Background(), []chromem.Document{ return c.collection.AddDocuments(context.Background(), []chromem.Document{
{ {
Content: s, Content: s,

0
llm/rag/rag_localai.go → llm/rag/localai.go
View File