feat: make slack process images

core/agent/actions.go

@@ -119,6 +119,20 @@ func (m Messages) Exist(content string) bool {
 	return false
 }
 
+func (m Messages) RemoveLastUserMessage() Messages {
+	if len(m) == 0 {
+		return m
+	}
+
+	for i := len(m) - 1; i >= 0; i-- {
+		if m[i].Role == UserRole {
+			return append(m[:i], m[i+1:]...)
+		}
+	}
+
+	return m
+}
+
 func (m Messages) Save(path string) error {
 	content, err := json.MarshalIndent(m, "", "  ")
 	if err != nil {

core/agent/agent.go

@@ -358,14 +358,18 @@ func (a *Agent) processUserInputs(job *Job, role string) {
 				} else {
 					// We replace the user message with the image description
 					// and add the user text to the conversation
-					lastUserMessage.Content = fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription)
+					explainerMessage := openai.ChatCompletionMessage{
-					lastUserMessage.MultiContent = nil
+						Role:    "system",
-					lastUserMessage.Role = "system"
+						Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription),
+					}
+
+					// remove lastUserMessage from the conversation
+					a.currentConversation = a.currentConversation.RemoveLastUserMessage()
+					a.currentConversation = append(a.currentConversation, explainerMessage)
 					a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
 						Role:    role,
 						Content: text,
 					})
-					xlog.Debug("Conversation after image description", "conversation", a.currentConversation)
 				}
 			}
 		}

core/agent/knowledgebase.go

@@ -20,14 +20,9 @@ func (a *Agent) knowledgeBaseLookup() {
 	// Walk conversation from bottom to top, and find the first message of the user
 	// to use it as a query to the KB
 	var userMessage string
-	for i := len(a.currentConversation) - 1; i >= 0; i-- {
+	userMessage = a.currentConversation.GetLatestUserMessage().Content
-		xlog.Info("[Knowledge Base Lookup] Conversation", "role", a.currentConversation[i].Role, "Content", a.currentConversation[i].Content)
+
-		if a.currentConversation[i].Role == "user" {
+	xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage, "lastMessage", a.currentConversation.GetLatestUserMessage())
-			userMessage = a.currentConversation[i].Content
-			break
-		}
-	}
-	xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage)
 
 	if userMessage == "" {
 		xlog.Info("[Knowledge Base Lookup] No user message found in conversation", "agent", a.Character.Name)

services/connectors/slack.go

@@ -1,7 +1,10 @@
 package connectors
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -97,6 +100,285 @@ func generateAttachmentsFromJobResponse(j *agent.JobResult) (attachments []slack
 	return
 }
 
+func (t *Slack) handleChannelMessage(
+	a *agent.Agent,
+	api *slack.Client, ev *slackevents.MessageEvent, b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
+	if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
+		t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
+		// Skip messages from other channels
+		xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
+		return
+	}
+
+	if b.UserID == ev.User {
+		// Skip messages from ourselves
+		return
+	}
+
+	message := cleanUpUsernameFromMessage(ev.Text, b)
+
+	go func() {
+
+		imageBytes := new(bytes.Buffer)
+		mimeType := "image/jpeg"
+
+		// Fetch the message using the API
+		messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
+			ChannelID: ev.Channel,
+			Timestamp: ev.TimeStamp,
+		})
+
+		if err != nil {
+			xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
+		} else {
+			for _, msg := range messages {
+				if len(msg.Files) == 0 {
+					continue
+				}
+				for _, attachment := range msg.Files {
+					if attachment.URLPrivate != "" {
+						xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
+						// download image with slack api
+						mimeType = attachment.Mimetype
+						if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
+							xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
+						}
+					}
+				}
+			}
+		}
+
+		agentOptions := []agent.JobOption{agent.WithText(message)}
+
+		// If the last message has an image, we send it as a multi content message
+		if len(imageBytes.Bytes()) > 0 {
+
+			// // Encode the image to base64
+			imgBase64, err := encodeImageFromURL(*imageBytes)
+			if err != nil {
+				xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
+			} else {
+				agentOptions = append(agentOptions, agent.WithImage(fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64)))
+			}
+		}
+
+		res := a.Ask(
+			agentOptions...,
+		)
+
+		res.Response = githubmarkdownconvertergo.Slack(res.Response)
+
+		_, _, err = api.PostMessage(ev.Channel,
+			slack.MsgOptionText(res.Response, true),
+			slack.MsgOptionPostMessageParameters(postMessageParams),
+			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+		//	slack.MsgOptionTS(ts),
+		)
+		if err != nil {
+			xlog.Error(fmt.Sprintf("Error posting message: %v", err))
+		}
+	}()
+}
+
+// Function to download the image from a URL and encode it to base64
+func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) {
+
+	// WRITE THIS SOMEWHERE
+	ioutil.WriteFile("image.jpg", imageBytes.Bytes(), 0644)
+
+	// Encode the image data to base64
+	base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes())
+	return base64Image, nil
+}
+
+func (t *Slack) handleMention(
+	a *agent.Agent, api *slack.Client, ev *slackevents.AppMentionEvent,
+	b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
+
+	if b.UserID == ev.User {
+		// Skip messages from ourselves
+		return
+	}
+	message := cleanUpUsernameFromMessage(ev.Text, b)
+
+	// strip our id from the message
+	xlog.Info("Message", message)
+
+	go func() {
+		ts := ev.ThreadTimeStamp
+
+		var threadMessages []openai.ChatCompletionMessage
+
+		// A thread already exists
+		// so we reconstruct the conversation
+		if ts != "" {
+			// Fetch the thread messages
+			messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
+				ChannelID: ev.Channel,
+				Timestamp: ts,
+			})
+			if err != nil {
+				xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
+			} else {
+				for i, msg := range messages {
+					role := "assistant"
+					if msg.User != b.UserID {
+						role = "user"
+					}
+
+					imageBytes := new(bytes.Buffer)
+					mimeType := "image/jpeg"
+
+					xlog.Debug(fmt.Sprintf("Message: %+v", msg))
+					if len(msg.Files) > 0 {
+						for _, attachment := range msg.Files {
+
+							if attachment.URLPrivate != "" {
+								xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
+								mimeType = attachment.Mimetype
+								// download image with slack api
+								if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
+									xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
+								}
+							}
+						}
+					}
+					// If the last message has an image, we send it as a multi content message
+					if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 {
+
+						// // Encode the image to base64
+						imgBase64, err := encodeImageFromURL(*imageBytes)
+						if err != nil {
+							xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
+						}
+
+						threadMessages = append(
+							threadMessages,
+							openai.ChatCompletionMessage{
+								Role: role,
+								MultiContent: []openai.ChatMessagePart{
+									{
+										Text: cleanUpUsernameFromMessage(msg.Text, b),
+										Type: openai.ChatMessagePartTypeText,
+									},
+									{
+										Type: openai.ChatMessagePartTypeImageURL,
+										ImageURL: &openai.ChatMessageImageURL{
+											URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
+											//	URL: imgUrl,
+										},
+									},
+								},
+							},
+						)
+					} else {
+						threadMessages = append(
+							threadMessages,
+							openai.ChatCompletionMessage{
+								Role:    role,
+								Content: cleanUpUsernameFromMessage(msg.Text, b),
+							},
+						)
+					}
+				}
+			}
+		} else {
+
+			imageBytes := new(bytes.Buffer)
+			mimeType := "image/jpeg"
+
+			// Fetch the message using the API
+			messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
+				ChannelID: ev.Channel,
+				Timestamp: ev.TimeStamp,
+			})
+
+			if err != nil {
+				xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
+			} else {
+				for _, msg := range messages {
+					if len(msg.Files) == 0 {
+						continue
+					}
+					for _, attachment := range msg.Files {
+						if attachment.URLPrivate != "" {
+							xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
+							// download image with slack api
+							mimeType = attachment.Mimetype
+							if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
+								xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
+							}
+						}
+					}
+				}
+			}
+
+			// If the last message has an image, we send it as a multi content message
+			if len(imageBytes.Bytes()) > 0 {
+
+				// // Encode the image to base64
+				imgBase64, err := encodeImageFromURL(*imageBytes)
+				if err != nil {
+					xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
+				}
+
+				threadMessages = append(
+					threadMessages,
+					openai.ChatCompletionMessage{
+						Role: "user",
+						MultiContent: []openai.ChatMessagePart{
+							{
+								Text: cleanUpUsernameFromMessage(message, b),
+								Type: openai.ChatMessagePartTypeText,
+							},
+							{
+								Type: openai.ChatMessagePartTypeImageURL,
+								ImageURL: &openai.ChatMessageImageURL{
+									//	URL: imgURL,
+									URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
+								},
+							},
+						},
+					},
+				)
+			} else {
+				threadMessages = append(threadMessages, openai.ChatCompletionMessage{
+					Role:    "user",
+					Content: cleanUpUsernameFromMessage(message, b),
+				})
+			}
+		}
+
+		res := a.Ask(
+			//	agent.WithText(message),
+			agent.WithConversationHistory(threadMessages),
+		)
+
+		res.Response = githubmarkdownconvertergo.Slack(res.Response)
+		var err error
+		if ts != "" {
+			_, _, err = api.PostMessage(ev.Channel,
+				slack.MsgOptionText(res.Response, true),
+				slack.MsgOptionPostMessageParameters(
+					postMessageParams,
+				),
+				slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+				slack.MsgOptionTS(ts))
+		} else {
+			_, _, err = api.PostMessage(ev.Channel,
+				slack.MsgOptionText(res.Response, true),
+				slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+				slack.MsgOptionPostMessageParameters(
+					postMessageParams,
+				),
+				slack.MsgOptionTS(ev.TimeStamp))
+		}
+		if err != nil {
+			xlog.Error(fmt.Sprintf("Error posting message: %v", err))
+		}
+	}()
+}
+
 func (t *Slack) Start(a *agent.Agent) {
 	api := slack.New(
 		t.botToken,
@@ -145,113 +427,9 @@ func (t *Slack) Start(a *agent.Agent) {
 
 					switch ev := innerEvent.Data.(type) {
 					case *slackevents.MessageEvent:
-						if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
+						t.handleChannelMessage(a, api, ev, b, postMessageParams)
-							t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
-							// Skip messages from other channels
-							xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
-							continue
-						}
-
-						if b.UserID == ev.User {
-							// Skip messages from ourselves
-							continue
-						}
-
-						message := cleanUpUsernameFromMessage(ev.Text, b)
-						go func() {
-
-							//ts := ev.ThreadTimeStamp
-
-							res := a.Ask(
-								agent.WithText(message),
-							)
-
-							res.Response = githubmarkdownconvertergo.Slack(res.Response)
-
-							_, _, err = api.PostMessage(ev.Channel,
-								slack.MsgOptionText(res.Response, true),
-								slack.MsgOptionPostMessageParameters(postMessageParams),
-								slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
-							//	slack.MsgOptionTS(ts),
-							)
-							if err != nil {
-								xlog.Error(fmt.Sprintf("Error posting message: %v", err))
-							}
-						}()
 					case *slackevents.AppMentionEvent:
-
+						t.handleMention(a, api, ev, b, postMessageParams)
-						if b.UserID == ev.User {
-							// Skip messages from ourselves
-							continue
-						}
-						message := cleanUpUsernameFromMessage(ev.Text, b)
-
-						// strip our id from the message
-						xlog.Info("Message", message)
-
-						go func() {
-							ts := ev.ThreadTimeStamp
-
-							var threadMessages []openai.ChatCompletionMessage
-
-							if ts != "" {
-								// Fetch the thread messages
-								messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
-									ChannelID: ev.Channel,
-									Timestamp: ts,
-								})
-								if err != nil {
-									xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
-								} else {
-									for _, msg := range messages {
-										role := "assistant"
-										if msg.User != b.UserID {
-											role = "user"
-										}
-										threadMessages = append(threadMessages,
-											openai.ChatCompletionMessage{
-												Role:    role,
-												Content: cleanUpUsernameFromMessage(msg.Text, b),
-											},
-										)
-
-									}
-								}
-							} else {
-								threadMessages = append(threadMessages, openai.ChatCompletionMessage{
-									Role:    "user",
-									Content: cleanUpUsernameFromMessage(message, b),
-								})
-							}
-
-							res := a.Ask(
-								//	agent.WithText(message),
-								agent.WithConversationHistory(threadMessages),
-							)
-
-							res.Response = githubmarkdownconvertergo.Slack(res.Response)
-
-							if ts != "" {
-								_, _, err = api.PostMessage(ev.Channel,
-									slack.MsgOptionText(res.Response, true),
-									slack.MsgOptionPostMessageParameters(
-										postMessageParams,
-									),
-									slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
-									slack.MsgOptionTS(ts))
-							} else {
-								_, _, err = api.PostMessage(ev.Channel,
-									slack.MsgOptionText(res.Response, true),
-									slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
-									slack.MsgOptionPostMessageParameters(
-										postMessageParams,
-									),
-									slack.MsgOptionTS(ev.TimeStamp))
-							}
-							if err != nil {
-								xlog.Error(fmt.Sprintf("Error posting message: %v", err))
-							}
-						}()
 					case *slackevents.MemberJoinedChannelEvent:
 						xlog.Error(fmt.Sprintf("user %q joined to channel %q", ev.User, ev.Channel))
 					}