feat(ui): Action playground config and parameter forms

Signed-off-by: Richard Palethorpe <io@richiejp.com>

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - run: |
         # Add Docker's official GPG key:
         sudo apt-get update

core/action/custom.go

@@ -95,6 +95,11 @@ func (a *CustomAction) Run(ctx context.Context, params types.ActionParams) (type
 
 func (a *CustomAction) Definition() types.ActionDefinition {
 
+	if a.i == nil {
+		xlog.Error("Interpreter is not initialized for custom action", "action", a.config["name"])
+		return types.ActionDefinition{}
+	}
+
 	v, err := a.i.Eval(fmt.Sprintf("%s.Definition", a.config["name"]))
 	if err != nil {
 		xlog.Error("Error getting custom action definition", "error", err)

pkg/localoperator/client.go

@@ -1,4 +1,4 @@
-package api
+package localoperator
 
 import (
 	"bytes"

services/actions/browseragentrunner.go

@@ -10,6 +10,10 @@ import (
 	"github.com/sashabaranov/go-openai/jsonschema"
 )
 
+const (
+	MetadataBrowserAgentHistory = "browser_agent_history"
+)
+
 type BrowserAgentRunner struct {
 	baseURL, customActionName string
 	client                    *api.Client
@@ -62,7 +66,7 @@ func (b *BrowserAgentRunner) Run(ctx context.Context, params types.ActionParams)
 
 	return types.ActionResult{
 		Result:   fmt.Sprintf("Browser agent completed successfully. History:\n%s", historyStr),
-		Metadata: map[string]interface{}{"browser_agent_history": stateHistory},
+		Metadata: map[string]interface{}{MetadataBrowserAgentHistory: stateHistory},
 	}, nil
 }
 

services/connectors/slack.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/mudler/LocalAGI/pkg/config"
+	"github.com/mudler/LocalAGI/pkg/localoperator"
 	"github.com/mudler/LocalAGI/pkg/xlog"
 	"github.com/mudler/LocalAGI/pkg/xstrings"
 	"github.com/mudler/LocalAGI/services/actions"
@@ -167,8 +168,38 @@ func replaceUserIDsWithNamesInMessage(api *slack.Client, message string) string
 	return message
 }
 
-func generateAttachmentsFromJobResponse(j *types.JobResult) (attachments []slack.Attachment) {
+func generateAttachmentsFromJobResponse(j *types.JobResult, api *slack.Client, channelID, ts string) (attachments []slack.Attachment) {
 	for _, state := range j.State {
+		// coming from the browser agent
+		if history, exists := state.Metadata[actions.MetadataBrowserAgentHistory]; exists {
+			if historyStruct, ok := history.(*localoperator.StateHistory); ok {
+				state := historyStruct.States[len(historyStruct.States)-1]
+				// Decode base64 screenshot and upload to Slack
+				if state.Screenshot != "" {
+					screenshotData, err := base64.StdEncoding.DecodeString(state.Screenshot)
+					if err != nil {
+						xlog.Error(fmt.Sprintf("Error decoding screenshot: %v", err))
+						continue
+					}
+
+					data := string(screenshotData)
+					// Upload the file to Slack
+					_, err = api.UploadFileV2(slack.UploadFileV2Parameters{
+						Reader:          bytes.NewReader(screenshotData),
+						FileSize:        len(data),
+						ThreadTimestamp: ts,
+						Channel:         channelID,
+						Filename:        "screenshot.png",
+						InitialComment:  "Browser Agent Screenshot",
+					})
+					if err != nil {
+						xlog.Error(fmt.Sprintf("Error uploading screenshot: %v", err))
+						continue
+					}
+				}
+			}
+		}
+
 		// coming from the search action
 		if urls, exists := state.Metadata[actions.MetadataUrls]; exists {
 			for _, url := range xstrings.UniqueSlice(urls.([]string)) {
@@ -375,7 +406,7 @@ func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackeven
 				slack.MsgOptionEnableLinkUnfurl(),
 				slack.MsgOptionText(message, true),
 				slack.MsgOptionPostMessageParameters(postMessageParams),
-				slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+				slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, "")...),
 			)
 			if err != nil {
 				xlog.Error(fmt.Sprintf("Error posting message: %v", err))
@@ -387,7 +418,7 @@ func replyWithPostMessage(finalResponse string, api *slack.Client, ev *slackeven
 			slack.MsgOptionEnableLinkUnfurl(),
 			slack.MsgOptionText(res.Response, true),
 			slack.MsgOptionPostMessageParameters(postMessageParams),
-			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, "")...),
 		//	slack.MsgOptionTS(ts),
 		)
 		if err != nil {
@@ -408,7 +439,7 @@ func replyToUpdateMessage(finalResponse string, api *slack.Client, ev *slackeven
 			slack.MsgOptionLinkNames(true),
 			slack.MsgOptionEnableLinkUnfurl(),
 			slack.MsgOptionText(messages[0], true),
-			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, msgTs)...),
 		)
 		if err != nil {
 			xlog.Error(fmt.Sprintf("Error updating final message: %v", err))
@@ -435,7 +466,7 @@ func replyToUpdateMessage(finalResponse string, api *slack.Client, ev *slackeven
 			slack.MsgOptionLinkNames(true),
 			slack.MsgOptionEnableLinkUnfurl(),
 			slack.MsgOptionText(finalResponse, true),
-			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res)...),
+			slack.MsgOptionAttachments(generateAttachmentsFromJobResponse(res, api, ev.Channel, msgTs)...),
 		)
 		if err != nil {
 			xlog.Error(fmt.Sprintf("Error updating final message: %v", err))

slack.yaml

@@ -23,6 +23,7 @@ oauth_config:
       - commands
       - groups:history
       - files:read
+      - files:write
       - im:history
       - im:read
       - im:write

webui/app.go

@@ -419,6 +419,30 @@ func (a *App) Chat(pool *state.AgentPool) func(c *fiber.Ctx) error {
 	}
 }
 
+func (a *App) GetActionDefinition(pool *state.AgentPool) func(c *fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		payload := struct {
+			Config map[string]string `json:"config"`
+		}{}
+
+		if err := c.BodyParser(&payload); err != nil {
+			xlog.Error("Error parsing action payload", "error", err)
+			return errorJSONMessage(c, err.Error())
+		}
+
+		actionName := c.Params("name")
+
+		xlog.Debug("Executing action", "action", actionName, "config", payload.Config)
+		a, err := services.Action(actionName, "", payload.Config, pool, map[string]string{})
+		if err != nil {
+			xlog.Error("Error creating action", "error", err)
+			return errorJSONMessage(c, err.Error())
+		}
+
+		return c.JSON(a.Definition())
+	}
+}
+
 func (a *App) ExecuteAction(pool *state.AgentPool) func(c *fiber.Ctx) error {
 	return func(c *fiber.Ctx) error {
 		payload := struct {

webui/react-ui/src/pages/ActionsPlayground.jsx

@@ -1,6 +1,11 @@
 import { useState, useEffect } from 'react';
 import { useOutletContext, useNavigate } from 'react-router-dom';
-import { actionApi } from '../utils/api';
+import { actionApi, agentApi } from '../utils/api';
+import FormFieldDefinition from '../components/common/FormFieldDefinition';
+import hljs from 'highlight.js/lib/core';
+import json from 'highlight.js/lib/languages/json';
+import 'highlight.js/styles/monokai.css';
+hljs.registerLanguage('json', json);
 
 function ActionsPlayground() {
   const { showToast } = useOutletContext();
@@ -12,6 +17,10 @@ function ActionsPlayground() {
   const [result, setResult] = useState(null);
   const [loading, setLoading] = useState(false);
   const [loadingActions, setLoadingActions] = useState(true);
+  const [actionMetadata, setActionMetadata] = useState(null);
+  const [agentMetadata, setAgentMetadata] = useState(null);
+  const [configFields, setConfigFields] = useState([]);
+  const [paramFields, setParamFields] = useState([]);
 
   // Update document title
   useEffect(() => {
@@ -36,21 +45,106 @@ function ActionsPlayground() {
     };
 
     fetchActions();
-  }, [showToast]);
+  }, []);
+
+  // Fetch agent metadata on mount
+  useEffect(() => {
+    const fetchAgentMetadata = async () => {
+      try {
+        const metadata = await agentApi.getAgentConfigMetadata();
+        setAgentMetadata(metadata);
+      } catch (err) {
+        console.error('Error fetching agent metadata:', err);
+        showToast('Failed to load agent metadata', 'error');
+      }
+    };
+
+    fetchAgentMetadata();
+  }, []);
+
+  // Fetch action definition when action is selected or config changes
+  useEffect(() => {
+    if (!selectedAction) return;
+
+    const fetchActionDefinition = async () => {
+      try {
+        // Get config fields from agent metadata
+        const actionMeta = agentMetadata?.actions?.find(action => action.name === selectedAction);
+        const configFields = actionMeta?.fields || [];
+        console.debug('Config fields:', configFields);
+        setConfigFields(configFields);
+
+        // Parse current config to pass to action definition
+        let currentConfig = {};
+        try {
+          currentConfig = JSON.parse(configJson);
+        } catch (err) {
+          console.error('Error parsing current config:', err);
+        }
+
+        // Get parameter fields from action definition
+        const paramFields = await actionApi.getActionDefinition(selectedAction, currentConfig);
+        console.debug('Parameter fields:', paramFields);
+        setParamFields(paramFields);
+
+        // Reset JSON to match the new fields
+        setConfigJson(JSON.stringify(currentConfig, null, 2));
+        setParamsJson(JSON.stringify({}, null, 2));
+        setResult(null);
+      } catch (err) {
+        console.error('Error fetching action definition:', err);
+        showToast('Failed to load action definition', 'error');
+      }
+    };
+
+    fetchActionDefinition();
+  }, [selectedAction, agentMetadata]);
 
   // Handle action selection
   const handleActionChange = (e) => {
     setSelectedAction(e.target.value);
+    setConfigJson('{}');
+    setParamsJson('{}');
     setResult(null);
   };
 
-  // Handle JSON input changes
+  // Helper to generate onChange handlers for form fields
-  const handleConfigChange = (e) => {
+  const makeFieldChangeHandler = (fields, updateFn) => (e) => {
-    setConfigJson(e.target.value);
+    let value;
+    if (e && e.target) {
+      const fieldName = e.target.name;
+      const fieldDef = fields.find(f => f.name === fieldName);
+      const fieldType = fieldDef ? fieldDef.type : undefined;
+      if (fieldType === 'checkbox') {
+        value = e.target.checked;
+      } else if (fieldType === 'number') {
+        value = e.target.value === '' ? '' : Number(e.target.value);
+      } else {
+        value = e.target.value;
+      }
+      updateFn(fieldName, value);
+    }
   };
 
-  const handleParamsChange = (e) => {
+  // Handle form field changes
-    setParamsJson(e.target.value);
+  const handleConfigChange = (field, value) => {
+    try {
+      const config = JSON.parse(configJson);
+      config[field] = value;
+      setConfigJson(JSON.stringify(config, null, 2));
+    } catch (err) {
+      console.error('Error updating config:', err);
+    }
+  };
+
+  const handleParamsChange = (field, value) => {
+    try {
+      const params = JSON.parse(paramsJson);
+      params[field] = value;
+      setParamsJson(JSON.stringify(params, null, 2));
+    } catch (err) {
+      console.error('Error updating params:', err);
+    }
   };
 
   // Execute the selected action
@@ -135,34 +229,31 @@ function ActionsPlayground() {
         
         {selectedAction && (
           <div className="section-box">
-            <h2>Action Configuration</h2>
             
             <form onSubmit={handleExecuteAction}>
-              <div className="form-group mb-6">
+              {configFields.length > 0 && (
-                <label htmlFor="config-json">Configuration (JSON):</label>
+                <>
-                <textarea 
+                <h2>Configuration</h2>
-                  id="config-json"
+                <FormFieldDefinition
-                  value={configJson}
+                  fields={configFields}
-                  onChange={handleConfigChange}
+                  values={JSON.parse(configJson)}
-                  className="form-control"
+                  onChange={makeFieldChangeHandler(configFields, handleConfigChange)}
-                  rows="5"
+                  idPrefix="config_"
-                  placeholder='{"key": "value"}'
                 />
-                <p className="text-xs text-gray-400 mt-1">Enter JSON configuration for the action</p>
+                </>
-              </div>
+              )}
-              
+
-              <div className="form-group mb-6">
+              {paramFields.length > 0 && (
-                <label htmlFor="params-json">Parameters (JSON):</label>
+                <>
-                <textarea 
+                <h2>Parameters</h2>
-                  id="params-json"
+                <FormFieldDefinition
-                  value={paramsJson}
+                  fields={paramFields}
-                  onChange={handleParamsChange}
+                  values={JSON.parse(paramsJson)}
-                  className="form-control"
+                  onChange={makeFieldChangeHandler(paramFields, handleParamsChange)}
-                  rows="5"
+                  idPrefix="param_"
-                  placeholder='{"key": "value"}'
                 />
-                <p className="text-xs text-gray-400 mt-1">Enter JSON parameters for the action</p>
+                </>
-              </div>
+              )}
               
               <div className="flex justify-end">
                 <button 
@@ -194,9 +285,9 @@ function ActionsPlayground() {
               backgroundColor: 'rgba(30, 30, 30, 0.7)'
             }}>
               {typeof result === 'object' ? (
-                <pre style={{ margin: 0, whiteSpace: 'pre-wrap', wordBreak: 'break-word' }}>
+                <pre className="hljs"><code>
-                  {JSON.stringify(result, null, 2)}
+                  <div dangerouslySetInnerHTML={{ __html: hljs.highlight(JSON.stringify(result, null, 2), { language: 'json' }).value }}></div>
-                </pre>
+                </code></pre>
               ) : (
                 <pre style={{ margin: 0, whiteSpace: 'pre-wrap', wordBreak: 'break-word' }}>
                   {result}

webui/react-ui/src/utils/api.js

@@ -24,6 +24,50 @@ const buildUrl = (endpoint) => {
   return `${API_CONFIG.baseUrl}${endpoint.startsWith('/') ? endpoint.substring(1) : endpoint}`;
 };
 
+// Helper function to convert ActionDefinition to FormFieldDefinition format
+const convertActionDefinitionToFields = (definition) => {
+  if (!definition || !definition.Properties) {
+    return [];
+  }
+
+  const fields = [];
+  const required = definition.Required || [];
+
+  console.debug('Action definition:', definition);
+
+  Object.entries(definition.Properties).forEach(([name, property]) => {
+    const field = {
+      name,
+      label: name.charAt(0).toUpperCase() + name.slice(1),
+      type: 'text', // Default to text, we'll enhance this later
+      required: required.includes(name),
+      helpText: property.Description || '',
+      defaultValue: property.Default,
+    };
+
+    if (property.enum && property.enum.length > 0) {
+      field.type = 'select';
+      field.options = property.enum;
+    } else {
+      switch (property.type) {
+        case 'integer':
+          field.type = 'number';
+          field.min = property.Minimum;
+          field.max = property.Maximum;
+        break;
+      case 'boolean':
+        field.type = 'checkbox';
+        break;
+    }
+    // TODO: Handle Object and Array types which require nested fields
+  }
+
+    fields.push(field);
+  });
+
+  return fields;
+};
+
 // Agent-related API calls
 export const agentApi = {
   // Get list of all agents
@@ -215,7 +259,18 @@ export const actionApi = {
     });
     return handleResponse(response);
   },
-  
+
+  // Get action definition
+  getActionDefinition: async (name, config = {}) => {
+    const response = await fetch(buildUrl(API_CONFIG.endpoints.actionDefinition(name)), {
+      method: 'POST',
+      headers: API_CONFIG.headers,
+      body: JSON.stringify(config),
+    });
+    const definition = await handleResponse(response);
+    return convertActionDefinitionToFields(definition);
+  },
+
   // Execute an action for an agent
   executeAction: async (name, actionData) => {
     const response = await fetch(buildUrl(API_CONFIG.endpoints.executeAction(name)), {

webui/react-ui/src/utils/config.js

@@ -43,6 +43,7 @@ export const API_CONFIG = {
     
     // Action endpoints
     listActions: '/api/actions',
+    actionDefinition: (name) => `/api/action/${name}/definition`,
     executeAction: (name) => `/api/action/${name}/run`,
     
     // Status endpoint

webui/routes.go

@@ -188,6 +188,7 @@ func (app *App) registerRoutes(pool *state.AgentPool, webapp *fiber.App) {
 	// Add endpoint for getting agent config metadata
 	webapp.Get("/api/meta/agent/config", app.GetAgentConfigMeta())
 
+	webapp.Post("/api/action/:name/definition", app.GetActionDefinition(pool))
 	webapp.Post("/api/action/:name/run", app.ExecuteAction(pool))
 	webapp.Get("/api/actions", app.ListActions())