feat: make slack process images
This commit is contained in:
mudler
committed by
Ettore Di Giacinto
Ettore Di Giacinto
parent
bc60dde94f
commit
0b71d8dc10
@@ -119,6 +119,20 @@ func (m Messages) Exist(content string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (m Messages) RemoveLastUserMessage() Messages {
|
||||
if len(m) == 0 {
|
||||
return m
|
||||
}
|
||||
|
||||
for i := len(m) - 1; i >= 0; i-- {
|
||||
if m[i].Role == UserRole {
|
||||
return append(m[:i], m[i+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func (m Messages) Save(path string) error {
|
||||
content, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
|
||||
@@ -358,14 +358,18 @@ func (a *Agent) processUserInputs(job *Job, role string) {
|
||||
} else {
|
||||
// We replace the user message with the image description
|
||||
// and add the user text to the conversation
|
||||
lastUserMessage.Content = fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription)
|
||||
lastUserMessage.MultiContent = nil
|
||||
lastUserMessage.Role = "system"
|
||||
explainerMessage := openai.ChatCompletionMessage{
|
||||
Role: "system",
|
||||
Content: fmt.Sprintf("The user shared an image which can be described as: %s", imageDescription),
|
||||
}
|
||||
|
||||
// remove lastUserMessage from the conversation
|
||||
a.currentConversation = a.currentConversation.RemoveLastUserMessage()
|
||||
a.currentConversation = append(a.currentConversation, explainerMessage)
|
||||
a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
|
||||
Role: role,
|
||||
Content: text,
|
||||
})
|
||||
xlog.Debug("Conversation after image description", "conversation", a.currentConversation)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,14 +20,9 @@ func (a *Agent) knowledgeBaseLookup() {
|
||||
// Walk conversation from bottom to top, and find the first message of the user
|
||||
// to use it as a query to the KB
|
||||
var userMessage string
|
||||
for i := len(a.currentConversation) - 1; i >= 0; i-- {
|
||||
xlog.Info("[Knowledge Base Lookup] Conversation", "role", a.currentConversation[i].Role, "Content", a.currentConversation[i].Content)
|
||||
if a.currentConversation[i].Role == "user" {
|
||||
userMessage = a.currentConversation[i].Content
|
||||
break
|
||||
}
|
||||
}
|
||||
xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage)
|
||||
userMessage = a.currentConversation.GetLatestUserMessage().Content
|
||||
|
||||
xlog.Info("[Knowledge Base Lookup] Last user message", "agent", a.Character.Name, "message", userMessage, "lastMessage", a.currentConversation.GetLatestUserMessage())
|
||||
|
||||
if userMessage == "" {
|
||||
xlog.Info("[Knowledge Base Lookup] No user message found in conversation", "agent", a.Character.Name)
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package connectors
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -97,6 +100,285 @@ func generateAttachmentsFromJobResponse(j *agent.JobResult) (attachments []slack
|
||||
return
|
||||
}
|
||||
|
||||
func (t *Slack) handleChannelMessage(
|
||||
a *agent.Agent,
|
||||
api *slack.Client, ev *slackevents.MessageEvent, b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
|
||||
if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
|
||||
t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
|
||||
// Skip messages from other channels
|
||||
xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
|
||||
return
|
||||
}
|
||||
|
||||
if b.UserID == ev.User {
|
||||
// Skip messages from ourselves
|
||||
return
|
||||
}
|
||||
|
||||
message := cleanUpUsernameFromMessage(ev.Text, b)
|
||||
|
||||
go func() {
|
||||
|
||||
imageBytes := new(bytes.Buffer)
|
||||
mimeType := "image/jpeg"
|
||||
|
||||
// Fetch the message using the API
|
||||
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
|
||||
ChannelID: ev.Channel,
|
||||
Timestamp: ev.TimeStamp,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
|
||||
} else {
|
||||
for _, msg := range messages {
|
||||
if len(msg.Files) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, attachment := range msg.Files {
|
||||
if attachment.URLPrivate != "" {
|
||||
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
|
||||
// download image with slack api
|
||||
mimeType = attachment.Mimetype
|
||||
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
agentOptions := []agent.JobOption{agent.WithText(message)}
|
||||
|
||||
// If the last message has an image, we send it as a multi content message
|
||||
if len(imageBytes.Bytes()) > 0 {
|
||||
|
||||
// // Encode the image to base64
|
||||
imgBase64, err := encodeImageFromURL(*imageBytes)
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
|
||||
} else {
|
||||
agentOptions = append(agentOptions, agent.WithImage(fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64)))
|
||||
}
|
||||
}
|
||||
|
||||
res := a.Ask(
|
||||
agentOptions...,
|
||||
)
|
||||
|
||||
res.Response = githubmarkdownconvertergo.Slack(res.Response)
|
||||
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionPostMessageParameters(postMessageParams),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
// slack.MsgOptionTS(ts),
|
||||
)
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Function to download the image from a URL and encode it to base64
|
||||
func encodeImageFromURL(imageBytes bytes.Buffer) (string, error) {
|
||||
|
||||
// WRITE THIS SOMEWHERE
|
||||
ioutil.WriteFile("image.jpg", imageBytes.Bytes(), 0644)
|
||||
|
||||
// Encode the image data to base64
|
||||
base64Image := base64.StdEncoding.EncodeToString(imageBytes.Bytes())
|
||||
return base64Image, nil
|
||||
}
|
||||
|
||||
func (t *Slack) handleMention(
|
||||
a *agent.Agent, api *slack.Client, ev *slackevents.AppMentionEvent,
|
||||
b *slack.AuthTestResponse, postMessageParams slack.PostMessageParameters) {
|
||||
|
||||
if b.UserID == ev.User {
|
||||
// Skip messages from ourselves
|
||||
return
|
||||
}
|
||||
message := cleanUpUsernameFromMessage(ev.Text, b)
|
||||
|
||||
// strip our id from the message
|
||||
xlog.Info("Message", message)
|
||||
|
||||
go func() {
|
||||
ts := ev.ThreadTimeStamp
|
||||
|
||||
var threadMessages []openai.ChatCompletionMessage
|
||||
|
||||
// A thread already exists
|
||||
// so we reconstruct the conversation
|
||||
if ts != "" {
|
||||
// Fetch the thread messages
|
||||
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
|
||||
ChannelID: ev.Channel,
|
||||
Timestamp: ts,
|
||||
})
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
|
||||
} else {
|
||||
for i, msg := range messages {
|
||||
role := "assistant"
|
||||
if msg.User != b.UserID {
|
||||
role = "user"
|
||||
}
|
||||
|
||||
imageBytes := new(bytes.Buffer)
|
||||
mimeType := "image/jpeg"
|
||||
|
||||
xlog.Debug(fmt.Sprintf("Message: %+v", msg))
|
||||
if len(msg.Files) > 0 {
|
||||
for _, attachment := range msg.Files {
|
||||
|
||||
if attachment.URLPrivate != "" {
|
||||
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
|
||||
mimeType = attachment.Mimetype
|
||||
// download image with slack api
|
||||
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// If the last message has an image, we send it as a multi content message
|
||||
if len(imageBytes.Bytes()) > 0 && i == len(messages)-1 {
|
||||
|
||||
// // Encode the image to base64
|
||||
imgBase64, err := encodeImageFromURL(*imageBytes)
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
|
||||
}
|
||||
|
||||
threadMessages = append(
|
||||
threadMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: role,
|
||||
MultiContent: []openai.ChatMessagePart{
|
||||
{
|
||||
Text: cleanUpUsernameFromMessage(msg.Text, b),
|
||||
Type: openai.ChatMessagePartTypeText,
|
||||
},
|
||||
{
|
||||
Type: openai.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &openai.ChatMessageImageURL{
|
||||
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
|
||||
// URL: imgUrl,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
} else {
|
||||
threadMessages = append(
|
||||
threadMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: role,
|
||||
Content: cleanUpUsernameFromMessage(msg.Text, b),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
imageBytes := new(bytes.Buffer)
|
||||
mimeType := "image/jpeg"
|
||||
|
||||
// Fetch the message using the API
|
||||
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
|
||||
ChannelID: ev.Channel,
|
||||
Timestamp: ev.TimeStamp,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error fetching messages: %v", err))
|
||||
} else {
|
||||
for _, msg := range messages {
|
||||
if len(msg.Files) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, attachment := range msg.Files {
|
||||
if attachment.URLPrivate != "" {
|
||||
xlog.Debug(fmt.Sprintf("Getting Attachment: %+v", attachment))
|
||||
// download image with slack api
|
||||
mimeType = attachment.Mimetype
|
||||
if err := api.GetFile(attachment.URLPrivate, imageBytes); err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error downloading image: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If the last message has an image, we send it as a multi content message
|
||||
if len(imageBytes.Bytes()) > 0 {
|
||||
|
||||
// // Encode the image to base64
|
||||
imgBase64, err := encodeImageFromURL(*imageBytes)
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error encoding image to base64: %v", err))
|
||||
}
|
||||
|
||||
threadMessages = append(
|
||||
threadMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: "user",
|
||||
MultiContent: []openai.ChatMessagePart{
|
||||
{
|
||||
Text: cleanUpUsernameFromMessage(message, b),
|
||||
Type: openai.ChatMessagePartTypeText,
|
||||
},
|
||||
{
|
||||
Type: openai.ChatMessagePartTypeImageURL,
|
||||
ImageURL: &openai.ChatMessageImageURL{
|
||||
// URL: imgURL,
|
||||
URL: fmt.Sprintf("data:%s;base64,%s", mimeType, imgBase64),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
} else {
|
||||
threadMessages = append(threadMessages, openai.ChatCompletionMessage{
|
||||
Role: "user",
|
||||
Content: cleanUpUsernameFromMessage(message, b),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
res := a.Ask(
|
||||
// agent.WithText(message),
|
||||
agent.WithConversationHistory(threadMessages),
|
||||
)
|
||||
|
||||
res.Response = githubmarkdownconvertergo.Slack(res.Response)
|
||||
var err error
|
||||
if ts != "" {
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionPostMessageParameters(
|
||||
postMessageParams,
|
||||
),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
slack.MsgOptionTS(ts))
|
||||
} else {
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
slack.MsgOptionPostMessageParameters(
|
||||
postMessageParams,
|
||||
),
|
||||
slack.MsgOptionTS(ev.TimeStamp))
|
||||
}
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (t *Slack) Start(a *agent.Agent) {
|
||||
api := slack.New(
|
||||
t.botToken,
|
||||
@@ -145,113 +427,9 @@ func (t *Slack) Start(a *agent.Agent) {
|
||||
|
||||
switch ev := innerEvent.Data.(type) {
|
||||
case *slackevents.MessageEvent:
|
||||
if t.channelID == "" && !t.alwaysReply || // If we have set alwaysReply and no channelID
|
||||
t.channelID != ev.Channel { // If we have a channelID and it's not the same as the event channel
|
||||
// Skip messages from other channels
|
||||
xlog.Info("Skipping reply to channel", ev.Channel, t.channelID)
|
||||
continue
|
||||
}
|
||||
|
||||
if b.UserID == ev.User {
|
||||
// Skip messages from ourselves
|
||||
continue
|
||||
}
|
||||
|
||||
message := cleanUpUsernameFromMessage(ev.Text, b)
|
||||
go func() {
|
||||
|
||||
//ts := ev.ThreadTimeStamp
|
||||
|
||||
res := a.Ask(
|
||||
agent.WithText(message),
|
||||
)
|
||||
|
||||
res.Response = githubmarkdownconvertergo.Slack(res.Response)
|
||||
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionPostMessageParameters(postMessageParams),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
// slack.MsgOptionTS(ts),
|
||||
)
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
|
||||
}
|
||||
}()
|
||||
t.handleChannelMessage(a, api, ev, b, postMessageParams)
|
||||
case *slackevents.AppMentionEvent:
|
||||
|
||||
if b.UserID == ev.User {
|
||||
// Skip messages from ourselves
|
||||
continue
|
||||
}
|
||||
message := cleanUpUsernameFromMessage(ev.Text, b)
|
||||
|
||||
// strip our id from the message
|
||||
xlog.Info("Message", message)
|
||||
|
||||
go func() {
|
||||
ts := ev.ThreadTimeStamp
|
||||
|
||||
var threadMessages []openai.ChatCompletionMessage
|
||||
|
||||
if ts != "" {
|
||||
// Fetch the thread messages
|
||||
messages, _, _, err := api.GetConversationReplies(&slack.GetConversationRepliesParameters{
|
||||
ChannelID: ev.Channel,
|
||||
Timestamp: ts,
|
||||
})
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error fetching thread messages: %v", err))
|
||||
} else {
|
||||
for _, msg := range messages {
|
||||
role := "assistant"
|
||||
if msg.User != b.UserID {
|
||||
role = "user"
|
||||
}
|
||||
threadMessages = append(threadMessages,
|
||||
openai.ChatCompletionMessage{
|
||||
Role: role,
|
||||
Content: cleanUpUsernameFromMessage(msg.Text, b),
|
||||
},
|
||||
)
|
||||
|
||||
}
|
||||
}
|
||||
} else {
|
||||
threadMessages = append(threadMessages, openai.ChatCompletionMessage{
|
||||
Role: "user",
|
||||
Content: cleanUpUsernameFromMessage(message, b),
|
||||
})
|
||||
}
|
||||
|
||||
res := a.Ask(
|
||||
// agent.WithText(message),
|
||||
agent.WithConversationHistory(threadMessages),
|
||||
)
|
||||
|
||||
res.Response = githubmarkdownconvertergo.Slack(res.Response)
|
||||
|
||||
if ts != "" {
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionPostMessageParameters(
|
||||
postMessageParams,
|
||||
),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
slack.MsgOptionTS(ts))
|
||||
} else {
|
||||
_, _, err = api.PostMessage(ev.Channel,
|
||||
slack.MsgOptionText(res.Response, true),
|
||||
slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
|
||||
slack.MsgOptionPostMessageParameters(
|
||||
postMessageParams,
|
||||
),
|
||||
slack.MsgOptionTS(ev.TimeStamp))
|
||||
}
|
||||
if err != nil {
|
||||
xlog.Error(fmt.Sprintf("Error posting message: %v", err))
|
||||
}
|
||||
}()
|
||||
t.handleMention(a, api, ev, b, postMessageParams)
|
||||
case *slackevents.MemberJoinedChannelEvent:
|
||||
xlog.Error(fmt.Sprintf("user %q joined to channel %q", ev.User, ev.Channel))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user