alex/LocalAGI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: make slack process images

Browse Source
This commit is contained in:
mudler
2025-03-09 18:50:54 +01:00
committed by Ettore Di Giacinto
parent bc60dde94f
commit 0b71d8dc10
4 changed files with 309 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
core/agent/actions.go
View File

@@ -119,6 +119,20 @@ func (m Messages) Exist(content string) bool {
return false
}
func (m Messages) RemoveLastUserMessage() Messages {
if len(m) == 0 {
return m
}
for i := len(m) - 1; i >= 0; i-- {
if m[i].Role == UserRole {
return append(m[:i], m[i+1:]...)
}
}
return m
}
func (m Messages) Save(path string) error {
content, err := json.MarshalIndent(m, "", " ")
if err != nil {

12
core/agent/agent.go
View File

@@ -358,14 +358,18 @@ func (a *Agent) processUserInputs(job *Job, role string) {
} else {
// We replace the user message with the image description
// and add the user text to the conversation
lastUserMessage.Content = fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription)
lastUserMessage.MultiContent = nil
lastUserMessage.Role = "system"
explainerMessage := openai.ChatCompletionMessage{
Role: "system",
Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription),
}
// remove lastUserMessage from the conversation
a.currentConversation = a.currentConversation.RemoveLastUserMessage()
a.currentConversation = append(a.currentConversation, explainerMessage)
a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
Role: role,
Content: text,
})
xlog.Debug("Conversation after image description", "conversation", a.currentConversation)
}
}
}

11
core/agent/knowledgebase.go
View File

@@ -20,14 +20,9 @@ func (a *Agent) knowledgeBaseLookup() {
// Walk conversation from bottom to top, and find the first message of the user
// to use it as a query to the KB
var userMessage string
for i := len(a.currentConversation) - 1; i >= 0; i-- {
xlog.Info("[Knowledge Base Lookup] Conversation", "role", a.currentConversation[i].Role, "Content", a.currentConversation[i].Content)
if a.currentConversation[i].Role == "user" {
userMessage = a.currentConversation[i].Content
break
}
}
xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage)
userMessage = a.currentConversation.GetLatestUserMessage().Content
xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage, "lastMessage", a.currentConversation.GetLatestUserMessage())
if userMessage == "" {
xlog.Info("[Knowledge Base Lookup] No user message found in conversation", "agent", a.Character.Name)

390
services/connectors/slack.go
View File

@@ -1,7 +1,10 @@
package connectors
import (
"bytes"
"encoding/base64"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
@@ -97,6 +100,285 @@ func generateAttachmentsFromJobResponse(j *agent.JobResult) (attachments []slack
return
}
func (t *Slack) handleChannelMessage(
a *agent.Agent,
api *slack.Client, ev *slackevents.MessageEvent, b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
// Skip messages from other channels
xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
return
}
if b.UserID == ev.User {
// Skip messages from ourselves
return
}
message := cleanUpUsernameFromMessage(ev.Text, b)
go func() {
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
// Fetch the message using the API
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ev.TimeStamp,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
} else {
for _, msg := range messages {
if len(msg.Files) == 0 {
continue
}
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
mimeType = attachment.Mimetype
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
}
}
}
}
agentOptions := []agent.JobOption{agent.WithText(message)}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
} else {
agentOptions = append(agentOptions, agent.WithImage(fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64)))
}
}
res := a.Ask(
agentOptions...,
)
res.Response = githubmarkdownconvertergo.Slack(res.Response)
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionPostMessageParameters(postMessageParams),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
// slack.MsgOptionTS(ts),
)
if err != nil {
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
}
}()
}
// Function to download the image from a URL and encode it to base64
func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) {
// WRITE THIS SOMEWHERE
ioutil.WriteFile("image.jpg", imageBytes.Bytes(), 0644)
// Encode the image data to base64
base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes())
return base64Image, nil
}
func (t *Slack) handleMention(
a *agent.Agent, api *slack.Client, ev *slackevents.AppMentionEvent,
b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
if b.UserID == ev.User {
// Skip messages from ourselves
return
}
message := cleanUpUsernameFromMessage(ev.Text, b)
// strip our id from the message
xlog.Info("Message", message)
go func() {
ts := ev.ThreadTimeStamp
var threadMessages []openai.ChatCompletionMessage
// A thread already exists
// so we reconstruct the conversation
if ts != "" {
// Fetch the thread messages
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ts,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
} else {
for i, msg := range messages {
role := "assistant"
if msg.User != b.UserID {
role = "user"
}
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
xlog.Debug(fmt.Sprintf("Message: %+v", msg))
if len(msg.Files) > 0 {
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
mimeType = attachment.Mimetype
// download image with slack api
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
}
}
}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 {
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
}
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
Role: role,
MultiContent: []openai.ChatMessagePart{
{
Text: cleanUpUsernameFromMessage(msg.Text, b),
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
// URL: imgUrl,
},
},
},
},
)
} else {
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
Role: role,
Content: cleanUpUsernameFromMessage(msg.Text, b),
},
)
}
}
}
} else {
imageBytes := new(bytes.Buffer)
mimeType := "image/jpeg"
// Fetch the message using the API
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ev.TimeStamp,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
} else {
for _, msg := range messages {
if len(msg.Files) == 0 {
continue
}
for _, attachment := range msg.Files {
if attachment.URLPrivate != "" {
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
// download image with slack api
mimeType = attachment.Mimetype
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
}
}
}
}
}
// If the last message has an image, we send it as a multi content message
if len(imageBytes.Bytes()) > 0 {
// // Encode the image to base64
imgBase64, err := encodeImageFromURL(*imageBytes)
if err != nil {
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
}
threadMessages = append(
threadMessages,
openai.ChatCompletionMessage{
Role: "user",
MultiContent: []openai.ChatMessagePart{
{
Text: cleanUpUsernameFromMessage(message, b),
Type: openai.ChatMessagePartTypeText,
},
{
Type: openai.ChatMessagePartTypeImageURL,
ImageURL: &openai.ChatMessageImageURL{
// URL: imgURL,
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
},
},
},
},
)
} else {
threadMessages = append(threadMessages, openai.ChatCompletionMessage{
Role: "user",
Content: cleanUpUsernameFromMessage(message, b),
})
}
}
res := a.Ask(
// agent.WithText(message),
agent.WithConversationHistory(threadMessages),
)
res.Response = githubmarkdownconvertergo.Slack(res.Response)
var err error
if ts != "" {
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionPostMessageParameters(
postMessageParams,
),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
slack.MsgOptionTS(ts))
} else {
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
slack.MsgOptionPostMessageParameters(
postMessageParams,
),
slack.MsgOptionTS(ev.TimeStamp))
}
if err != nil {
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
}
}()
}
func (t *Slack) Start(a *agent.Agent) {
api := slack.New(
t.botToken,
@@ -145,113 +427,9 @@ func (t *Slack) Start(a *agent.Agent) {
switch ev := innerEvent.Data.(type) {
case *slackevents.MessageEvent:
if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
// Skip messages from other channels
xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
continue
}
if b.UserID == ev.User {
// Skip messages from ourselves
continue
}
message := cleanUpUsernameFromMessage(ev.Text, b)
go func() {
//ts := ev.ThreadTimeStamp
res := a.Ask(
agent.WithText(message),
)
res.Response = githubmarkdownconvertergo.Slack(res.Response)
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionPostMessageParameters(postMessageParams),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
// slack.MsgOptionTS(ts),
)
if err != nil {
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
}
}()
t.handleChannelMessage(a, api, ev, b, postMessageParams)
case *slackevents.AppMentionEvent:
if b.UserID == ev.User {
// Skip messages from ourselves
continue
}
message := cleanUpUsernameFromMessage(ev.Text, b)
// strip our id from the message
xlog.Info("Message", message)
go func() {
ts := ev.ThreadTimeStamp
var threadMessages []openai.ChatCompletionMessage
if ts != "" {
// Fetch the thread messages
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
ChannelID: ev.Channel,
Timestamp: ts,
})
if err != nil {
xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
} else {
for _, msg := range messages {
role := "assistant"
if msg.User != b.UserID {
role = "user"
}
threadMessages = append(threadMessages,
openai.ChatCompletionMessage{
Role: role,
Content: cleanUpUsernameFromMessage(msg.Text, b),
},
)
}
}
} else {
threadMessages = append(threadMessages, openai.ChatCompletionMessage{
Role: "user",
Content: cleanUpUsernameFromMessage(message, b),
})
}
res := a.Ask(
// agent.WithText(message),
agent.WithConversationHistory(threadMessages),
)
res.Response = githubmarkdownconvertergo.Slack(res.Response)
if ts != "" {
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionPostMessageParameters(
postMessageParams,
),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
slack.MsgOptionTS(ts))
} else {
_, _, err = api.PostMessage(ev.Channel,
slack.MsgOptionText(res.Response, true),
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
slack.MsgOptionPostMessageParameters(
postMessageParams,
),
slack.MsgOptionTS(ev.TimeStamp))
}
if err != nil {
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
}
}()
t.handleMention(a, api, ev, b, postMessageParams)
case *slackevents.MemberJoinedChannelEvent:
xlog.Error(fmt.Sprintf("user %q joined to channel %q", ev.User, ev.Channel))
}