From 748f60c11e7e806730e3d477b7503bc3725c2b81 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 24 Apr 2025 23:14:15 +0200 Subject: [PATCH] feat(browseragent): post screenshot on slack Signed-off-by: Ettore Di Giacinto --- pkg/localoperator/client.go | 2 +- services/actions/browseragentrunner.go | 6 +++- services/connectors/slack.go | 41 ++++++++++++++++++++++---- slack.yaml | 1 + 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/pkg/localoperator/client.go b/pkg/localoperator/client.go index c38b8e0..4eeb3db 100644 --- a/pkg/localoperator/client.go +++ b/pkg/localoperator/client.go @@ -1,4 +1,4 @@ -package api +package localoperator import ( "bytes" diff --git a/services/actions/browseragentrunner.go b/services/actions/browseragentrunner.go index 879c966..a8d52df 100644 --- a/services/actions/browseragentrunner.go +++ b/services/actions/browseragentrunner.go @@ -10,6 +10,10 @@ import ( "github.com/sashabaranov/go-openai/jsonschema" ) +const ( + MetadataBrowserAgentHistory = "browser_agent_history" +) + type BrowserAgentRunner struct { baseURL, customActionName string client *api.Client @@ -62,7 +66,7 @@ func (b *BrowserAgentRunner) Run(ctx context.Context, params types.ActionParams) return types.ActionResult{ Result: fmt.Sprintf("Browser agent completed successfully. History:\n%s", historyStr), - Metadata: map[string]interface{}{"browser_agent_history": stateHistory}, + Metadata: map[string]interface{}{MetadataBrowserAgentHistory: stateHistory}, }, nil } diff --git a/services/connectors/slack.go b/services/connectors/slack.go index 68070ca..84184ce 100644 --- a/services/connectors/slack.go +++ b/services/connectors/slack.go @@ -11,6 +11,7 @@ import ( "time" "github.com/mudler/LocalAGI/pkg/config" + "github.com/mudler/LocalAGI/pkg/localoperator" "github.com/mudler/LocalAGI/pkg/xlog" "github.com/mudler/LocalAGI/pkg/xstrings" "github.com/mudler/LocalAGI/services/actions" @@ -167,8 +168,38 @@ func replaceUserIDsWithNamesInMessage(api *slack.Client, message string) string return message } -func generateAttachmentsFromJobResponse(j *types.JobResult) (attachments []slack.Attachment) { +func generateAttachmentsFromJobResponse(j *types.JobResult, api *slack.Client, channelID, ts string) (attachments []slack.Attachment) { for _, state := range j.State { + // coming from the browser agent + if history, exists := state.Metadata[actions.MetadataBrowserAgentHistory]; exists { + if historyStruct, ok := history.(*localoperator.StateHistory); ok { + state := historyStruct.States[len(historyStruct.States)-1] + // Decode base64 screenshot and upload to Slack + if state.Screenshot != "" { + screenshotData, err := base64.StdEncoding.DecodeString(state.Screenshot) + if err != nil { + xlog.Error(fmt.Sprintf("Error decoding screenshot: %v", err)) + continue + } + + data := string(screenshotData) + // Upload the file to Slack + _, err = api.UploadFileV2(slack.UploadFileV2Parameters{ + Reader: bytes.NewReader(screenshotData), + FileSize: len(data), + ThreadTimestamp: ts, + Channel: channelID, + Filename: "screenshot.png", + InitialComment: "Browser Agent Screenshot", + }) + if err != nil { + xlog.Error(fmt.Sprintf("Error uploading screenshot: %v", err)) + continue + } + } + } + } + // coming from the search action if urls, exists := state.Metadata[actions.MetadataUrls]; exists { for _, url := range xstrings.UniqueSlice(urls.([]string)) { @@ -375,7 +406,7 @@ func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackeven slack.MsgOptionEnableLinkUnfurl(), slack.MsgOptionText(message, true), slack.MsgOptionPostMessageParameters(postMessageParams), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, "")...), ) if err != nil { xlog.Error(fmt.Sprintf("Error posting message: %v", err)) @@ -387,7 +418,7 @@ func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackeven slack.MsgOptionEnableLinkUnfurl(), slack.MsgOptionText(res.Response, true), slack.MsgOptionPostMessageParameters(postMessageParams), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, "")...), // slack.MsgOptionTS(ts), ) if err != nil { @@ -408,7 +439,7 @@ func replyToUpdateMessage(finalResponse string, api *slack.Client, ev *slackeven slack.MsgOptionLinkNames(true), slack.MsgOptionEnableLinkUnfurl(), slack.MsgOptionText(messages[0], true), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, msgTs)...), ) if err != nil { xlog.Error(fmt.Sprintf("Error updating final message: %v", err)) @@ -435,7 +466,7 @@ func replyToUpdateMessage(finalResponse string, api *slack.Client, ev *slackeven slack.MsgOptionLinkNames(true), slack.MsgOptionEnableLinkUnfurl(), slack.MsgOptionText(finalResponse, true), - slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...), + slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, msgTs)...), ) if err != nil { xlog.Error(fmt.Sprintf("Error updating final message: %v", err)) diff --git a/slack.yaml b/slack.yaml index f7bcf8f..dc36373 100644 --- a/slack.yaml +++ b/slack.yaml @@ -23,6 +23,7 @@ oauth_config: - commands - groups:history - files:read + - files:write - im:history - im:read - im:write