From 0b71d8dc10499e2db8cb13ad2dd8ede9560425d3 Mon Sep 17 00:00:00 2001 From: mudler Date: Sun, 9 Mar 2025 18:50:54 +0100 Subject: [PATCH] feat: make slack process images --- core/agent/actions.go | 14 ++ core/agent/agent.go | 12 +- core/agent/knowledgebase.go | 11 +- services/connectors/slack.go | 390 +++++++++++++++++++++++++---------- 4 files changed, 309 insertions(+), 118 deletions(-) diff --git a/core/agent/actions.go b/core/agent/actions.go index 5df6a40..7c4024d 100644 --- a/core/agent/actions.go +++ b/core/agent/actions.go @@ -119,6 +119,20 @@ func (m Messages) Exist(content string) bool { return false } +func (m Messages) RemoveLastUserMessage() Messages { + if len(m) == 0 { + return m + } + + for i := len(m) - 1; i >= 0; i-- { + if m[i].Role == UserRole { + return append(m[:i], m[i+1:]...) + } + } + + return m +} + func (m Messages) Save(path string) error { content, err := json.MarshalIndent(m, "", " ") if err != nil { diff --git a/core/agent/agent.go b/core/agent/agent.go index 704c70e..fe88af5 100644 --- a/core/agent/agent.go +++ b/core/agent/agent.go @@ -358,14 +358,18 @@ func (a *Agent) processUserInputs(job *Job, role string) { } else { // We replace the user message with the image description // and add the user text to the conversation - lastUserMessage.Content = fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription) - lastUserMessage.MultiContent = nil - lastUserMessage.Role = "system" + explainerMessage := openai.ChatCompletionMessage{ + Role: "system", + Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription), + } + + // remove lastUserMessage from the conversation + a.currentConversation = a.currentConversation.RemoveLastUserMessage() + a.currentConversation = append(a.currentConversation, explainerMessage) a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{ Role: role, Content: text, }) - xlog.Debug("Conversation after image description", "conversation", a.currentConversation) } } } diff --git a/core/agent/knowledgebase.go b/core/agent/knowledgebase.go index 29104da..898ee7e 100644 --- a/core/agent/knowledgebase.go +++ b/core/agent/knowledgebase.go @@ -20,14 +20,9 @@ func (a *Agent) knowledgeBaseLookup() { // Walk conversation from bottom to top, and find the first message of the user // to use it as a query to the KB var userMessage string - for i := len(a.currentConversation) - 1; i >= 0; i-- { - xlog.Info("[Knowledge Base Lookup] Conversation", "role", a.currentConversation[i].Role, "Content", a.currentConversation[i].Content) - if a.currentConversation[i].Role == "user" { - userMessage = a.currentConversation[i].Content - break - } - } - xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage) + userMessage = a.currentConversation.GetLatestUserMessage().Content + + xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage, "lastMessage", a.currentConversation.GetLatestUserMessage()) if userMessage == "" { xlog.Info("[Knowledge Base Lookup] No user message found in conversation", "agent", a.Character.Name) diff --git a/services/connectors/slack.go b/services/connectors/slack.go index 2e87ac5..30af50e 100644 --- a/services/connectors/slack.go +++ b/services/connectors/slack.go @@ -1,7 +1,10 @@ package connectors import ( + "bytes" + "encoding/base64" "fmt" + "io/ioutil" "log" "os" "strings" @@ -97,6 +100,285 @@ func generateAttachmentsFromJobResponse(j *agent.JobResult) (attachments []slack return } +func (t *Slack) handleChannelMessage( + a *agent.Agent, + api *slack.Client, ev *slackevents.MessageEvent, b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) { + if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID + t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel + // Skip messages from other channels + xlog.Info("Skipping reply to channel", ev.Channel, t.channelID) + return + } + + if b.UserID == ev.User { + // Skip messages from ourselves + return + } + + message := cleanUpUsernameFromMessage(ev.Text, b) + + go func() { + + imageBytes := new(bytes.Buffer) + mimeType := "image/jpeg" + + // Fetch the message using the API + messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ + ChannelID: ev.Channel, + Timestamp: ev.TimeStamp, + }) + + if err != nil { + xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) + } else { + for _, msg := range messages { + if len(msg.Files) == 0 { + continue + } + for _, attachment := range msg.Files { + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + // download image with slack api + mimeType = attachment.Mimetype + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + } + } + } + } + } + + agentOptions := []agent.JobOption{agent.WithText(message)} + + // If the last message has an image, we send it as a multi content message + if len(imageBytes.Bytes()) > 0 { + + // // Encode the image to base64 + imgBase64, err := encodeImageFromURL(*imageBytes) + if err != nil { + xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) + } else { + agentOptions = append(agentOptions, agent.WithImage(fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64))) + } + } + + res := a.Ask( + agentOptions..., + ) + + res.Response = githubmarkdownconvertergo.Slack(res.Response) + + _, _, err = api.PostMessage(ev.Channel, + slack.MsgOptionText(res.Response, true), + slack.MsgOptionPostMessageParameters(postMessageParams), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + // slack.MsgOptionTS(ts), + ) + if err != nil { + xlog.Error(fmt.Sprintf("Error posting message: %v", err)) + } + }() +} + +// Function to download the image from a URL and encode it to base64 +func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) { + + // WRITE THIS SOMEWHERE + ioutil.WriteFile("image.jpg", imageBytes.Bytes(), 0644) + + // Encode the image data to base64 + base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes()) + return base64Image, nil +} + +func (t *Slack) handleMention( + a *agent.Agent, api *slack.Client, ev *slackevents.AppMentionEvent, + b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) { + + if b.UserID == ev.User { + // Skip messages from ourselves + return + } + message := cleanUpUsernameFromMessage(ev.Text, b) + + // strip our id from the message + xlog.Info("Message", message) + + go func() { + ts := ev.ThreadTimeStamp + + var threadMessages []openai.ChatCompletionMessage + + // A thread already exists + // so we reconstruct the conversation + if ts != "" { + // Fetch the thread messages + messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ + ChannelID: ev.Channel, + Timestamp: ts, + }) + if err != nil { + xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err)) + } else { + for i, msg := range messages { + role := "assistant" + if msg.User != b.UserID { + role = "user" + } + + imageBytes := new(bytes.Buffer) + mimeType := "image/jpeg" + + xlog.Debug(fmt.Sprintf("Message: %+v", msg)) + if len(msg.Files) > 0 { + for _, attachment := range msg.Files { + + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + mimeType = attachment.Mimetype + // download image with slack api + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + } + } + } + } + // If the last message has an image, we send it as a multi content message + if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 { + + // // Encode the image to base64 + imgBase64, err := encodeImageFromURL(*imageBytes) + if err != nil { + xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) + } + + threadMessages = append( + threadMessages, + openai.ChatCompletionMessage{ + Role: role, + MultiContent: []openai.ChatMessagePart{ + { + Text: cleanUpUsernameFromMessage(msg.Text, b), + Type: openai.ChatMessagePartTypeText, + }, + { + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64), + // URL: imgUrl, + }, + }, + }, + }, + ) + } else { + threadMessages = append( + threadMessages, + openai.ChatCompletionMessage{ + Role: role, + Content: cleanUpUsernameFromMessage(msg.Text, b), + }, + ) + } + } + } + } else { + + imageBytes := new(bytes.Buffer) + mimeType := "image/jpeg" + + // Fetch the message using the API + messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ + ChannelID: ev.Channel, + Timestamp: ev.TimeStamp, + }) + + if err != nil { + xlog.Error(fmt.Sprintf("Error fetching messages: %v", err)) + } else { + for _, msg := range messages { + if len(msg.Files) == 0 { + continue + } + for _, attachment := range msg.Files { + if attachment.URLPrivate != "" { + xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment)) + // download image with slack api + mimeType = attachment.Mimetype + if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil { + xlog.Error(fmt.Sprintf("Error downloading image: %v", err)) + } + } + } + } + } + + // If the last message has an image, we send it as a multi content message + if len(imageBytes.Bytes()) > 0 { + + // // Encode the image to base64 + imgBase64, err := encodeImageFromURL(*imageBytes) + if err != nil { + xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err)) + } + + threadMessages = append( + threadMessages, + openai.ChatCompletionMessage{ + Role: "user", + MultiContent: []openai.ChatMessagePart{ + { + Text: cleanUpUsernameFromMessage(message, b), + Type: openai.ChatMessagePartTypeText, + }, + { + Type: openai.ChatMessagePartTypeImageURL, + ImageURL: &openai.ChatMessageImageURL{ + // URL: imgURL, + URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64), + }, + }, + }, + }, + ) + } else { + threadMessages = append(threadMessages, openai.ChatCompletionMessage{ + Role: "user", + Content: cleanUpUsernameFromMessage(message, b), + }) + } + } + + res := a.Ask( + // agent.WithText(message), + agent.WithConversationHistory(threadMessages), + ) + + res.Response = githubmarkdownconvertergo.Slack(res.Response) + var err error + if ts != "" { + _, _, err = api.PostMessage(ev.Channel, + slack.MsgOptionText(res.Response, true), + slack.MsgOptionPostMessageParameters( + postMessageParams, + ), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionTS(ts)) + } else { + _, _, err = api.PostMessage(ev.Channel, + slack.MsgOptionText(res.Response, true), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionPostMessageParameters( + postMessageParams, + ), + slack.MsgOptionTS(ev.TimeStamp)) + } + if err != nil { + xlog.Error(fmt.Sprintf("Error posting message: %v", err)) + } + }() +} + func (t *Slack) Start(a *agent.Agent) { api := slack.New( t.botToken, @@ -145,113 +427,9 @@ func (t *Slack) Start(a *agent.Agent) { switch ev := innerEvent.Data.(type) { case *slackevents.MessageEvent: - if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID - t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel - // Skip messages from other channels - xlog.Info("Skipping reply to channel", ev.Channel, t.channelID) - continue - } - - if b.UserID == ev.User { - // Skip messages from ourselves - continue - } - - message := cleanUpUsernameFromMessage(ev.Text, b) - go func() { - - //ts := ev.ThreadTimeStamp - - res := a.Ask( - agent.WithText(message), - ) - - res.Response = githubmarkdownconvertergo.Slack(res.Response) - - _, _, err = api.PostMessage(ev.Channel, - slack.MsgOptionText(res.Response, true), - slack.MsgOptionPostMessageParameters(postMessageParams), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), - // slack.MsgOptionTS(ts), - ) - if err != nil { - xlog.Error(fmt.Sprintf("Error posting message: %v", err)) - } - }() + t.handleChannelMessage(a, api, ev, b, postMessageParams) case *slackevents.AppMentionEvent: - - if b.UserID == ev.User { - // Skip messages from ourselves - continue - } - message := cleanUpUsernameFromMessage(ev.Text, b) - - // strip our id from the message - xlog.Info("Message", message) - - go func() { - ts := ev.ThreadTimeStamp - - var threadMessages []openai.ChatCompletionMessage - - if ts != "" { - // Fetch the thread messages - messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{ - ChannelID: ev.Channel, - Timestamp: ts, - }) - if err != nil { - xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err)) - } else { - for _, msg := range messages { - role := "assistant" - if msg.User != b.UserID { - role = "user" - } - threadMessages = append(threadMessages, - openai.ChatCompletionMessage{ - Role: role, - Content: cleanUpUsernameFromMessage(msg.Text, b), - }, - ) - - } - } - } else { - threadMessages = append(threadMessages, openai.ChatCompletionMessage{ - Role: "user", - Content: cleanUpUsernameFromMessage(message, b), - }) - } - - res := a.Ask( - // agent.WithText(message), - agent.WithConversationHistory(threadMessages), - ) - - res.Response = githubmarkdownconvertergo.Slack(res.Response) - - if ts != "" { - _, _, err = api.PostMessage(ev.Channel, - slack.MsgOptionText(res.Response, true), - slack.MsgOptionPostMessageParameters( - postMessageParams, - ), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), - slack.MsgOptionTS(ts)) - } else { - _, _, err = api.PostMessage(ev.Channel, - slack.MsgOptionText(res.Response, true), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), - slack.MsgOptionPostMessageParameters( - postMessageParams, - ), - slack.MsgOptionTS(ev.TimeStamp)) - } - if err != nil { - xlog.Error(fmt.Sprintf("Error posting message: %v", err)) - } - }() + t.handleMention(a, api, ev, b, postMessageParams) case *slackevents.MemberJoinedChannelEvent: xlog.Error(fmt.Sprintf("user %q joined to channel %q", ev.User, ev.Channel)) }