feat(planning): enable agent planning (#68)

* feat(planning): Allow the agent to plan subtasks

Signed-off-by: mudler <mudler@localai.io>

* feat(planning): enable planning toggle in the webui

Signed-off-by: mudler <mudler@localai.io>

* feat(planning): take in consideration the overall goal

Signed-off-by: mudler <mudler@localai.io>

* Update core/action/plan.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Signed-off-by: mudler <mudler@localai.io>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

README.md

@@ -289,6 +289,7 @@ Note: For proper SSE handling, you should use a client that supports SSE nativel
   "standalone_job": false,
   "random_identity": false,
   "initiate_conversations": true,
+  "enable_planning": true,
   "identity_guidance": "You are a helpful assistant.",
   "periodic_runs": "0 * * * *",
   "permanent_goal": "Help users with their questions.",

core/action/custom.go

@@ -75,6 +75,10 @@ func (a *CustomAction) initializeInterpreter() error {
 	return nil
 }
 
+func (a *CustomAction) Plannable() bool {
+	return true
+}
+
 func (a *CustomAction) Run(ctx context.Context, params ActionParams) (ActionResult, error) {
 	v, err := a.i.Eval(fmt.Sprintf("%s.Run", a.config["name"]))
 	if err != nil {

core/action/intention.go

@@ -25,6 +25,10 @@ func (a *IntentAction) Run(context.Context, ActionParams) (ActionResult, error)
 	return ActionResult{}, nil
 }
 
+func (a *IntentAction) Plannable() bool {
+	return false
+}
+
 func (a *IntentAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        "pick_tool",

core/action/newconversation.go

@@ -22,6 +22,10 @@ func (a *ConversationAction) Run(context.Context, ActionParams) (ActionResult, e
 	return ActionResult{}, nil
 }
 
+func (a *ConversationAction) Plannable() bool {
+	return false
+}
+
 func (a *ConversationAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        ConversationActionName,

core/action/noreply.go

@@ -16,6 +16,10 @@ func (a *StopAction) Run(context.Context, ActionParams) (ActionResult, error) {
 	return ActionResult{}, nil
 }
 
+func (a *StopAction) Plannable() bool {
+	return false
+}
+
 func (a *StopAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        StopActionName,

core/action/plan.go

@@ -10,22 +10,31 @@ import (
 // used by the LLM to schedule more actions
 const PlanActionName = "plan"
 
-func NewPlan() *PlanAction {
+func NewPlan(plannableActions []string) *PlanAction {
-	return &PlanAction{}
+	return &PlanAction{
+		plannables: plannableActions,
+	}
 }
 
-type PlanAction struct{}
+type PlanAction struct {
+	plannables []string
+}
 
 type PlanResult struct {
 	Subtasks []PlanSubtask `json:"subtasks"`
+	Goal     string        `json:"goal"`
 }
 type PlanSubtask struct {
 	Action    string `json:"action"`
 	Reasoning string `json:"reasoning"`
 }
 
-func (a *PlanAction) Run(context.Context, ActionParams) (string, error) {
+func (a *PlanAction) Run(context.Context, ActionParams) (ActionResult, error) {
-	return "no-op", nil
+	return ActionResult{}, nil
+}
+
+func (a *PlanAction) Plannable() bool {
+	return false
 }
 
 func (a *PlanAction) Definition() ActionDefinition {
@@ -40,6 +49,7 @@ func (a *PlanAction) Definition() ActionDefinition {
 					"action": {
 						Type:        jsonschema.String,
 						Description: "The action to call",
+						Enum:        a.plannables,
 					},
 					"reasoning": {
 						Type:        jsonschema.String,
@@ -47,7 +57,11 @@ func (a *PlanAction) Definition() ActionDefinition {
 					},
 				},
 			},
+			"goal": {
+				Type:        jsonschema.String,
+				Description: "The goal of this plan",
 			},
-		Required: []string{"subtasks"},
+		},
+		Required: []string{"subtasks", "goal"},
 	}
 }

core/action/reasoning.go

@@ -23,6 +23,10 @@ func (a *ReasoningAction) Run(context.Context, ActionParams) (ActionResult, erro
 	return ActionResult{}, nil
 }
 
+func (a *ReasoningAction) Plannable() bool {
+	return false
+}
+
 func (a *ReasoningAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        "pick_action",

core/action/reply.go

@@ -25,6 +25,10 @@ func (a *ReplyAction) Run(context.Context, ActionParams) (string, error) {
 	return "no-op", nil
 }
 
+func (a *ReplyAction) Plannable() bool {
+	return false
+}
+
 func (a *ReplyAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        ReplyActionName,

core/action/state.go

@@ -37,6 +37,10 @@ func (a *StateAction) Run(context.Context, ActionParams) (ActionResult, error) {
 	return ActionResult{Result: "internal state has been updated"}, nil
 }
 
+func (a *StateAction) Plannable() bool {
+	return false
+}
+
 func (a *StateAction) Definition() ActionDefinition {
 	return ActionDefinition{
 		Name:        StateActionName,

core/agent/actions.go

@@ -27,6 +27,7 @@ type ActionCurrentState struct {
 type Action interface {
 	Run(ctx context.Context, action action.ActionParams) (action.ActionResult, error)
 	Definition() action.ActionDefinition
+	Plannable() bool
 }
 
 type Actions []Action
@@ -211,8 +212,76 @@ func (a *Agent) generateParameters(ctx context.Context, pickTemplate string, act
 	)
 }
 
+func (a *Agent) handlePlanning(ctx context.Context, job *Job, chosenAction Action, actionParams action.ActionParams, reasoning string, pickTemplate string) error {
+	// Planning: run all the actions in sequence
+	if !chosenAction.Definition().Name.Is(action.PlanActionName) {
+		return nil
+	}
+
+	planResult := action.PlanResult{}
+	if err := actionParams.Unmarshal(&planResult); err != nil {
+		return fmt.Errorf("error unmarshalling plan result: %w", err)
+	}
+
+	xlog.Info("[Planning] starts", "agent", a.Character.Name, "goal", planResult.Goal)
+	for _, s := range planResult.Subtasks {
+		xlog.Info("[Planning] subtask", "agent", a.Character.Name, "action", s.Action, "reasoning", s.Reasoning)
+	}
+
+	if len(planResult.Subtasks) == 0 {
+		return fmt.Errorf("no subtasks")
+	}
+
+	// Execute all subtasks in sequence
+	for _, subtask := range planResult.Subtasks {
+		xlog.Info("[subtask] Generating parameters",
+			"agent", a.Character.Name,
+			"action", subtask.Action,
+			"reasoning", reasoning,
+		)
+
+		action := a.availableActions().Find(subtask.Action)
+
+		params, err := a.generateParameters(ctx, pickTemplate, action, a.currentConversation, fmt.Sprintf("%s, overall goal is: %s", subtask.Reasoning, planResult.Goal))
+		if err != nil {
+			return fmt.Errorf("error generating action's parameters: %w", err)
+
+		}
+		actionParams = params.actionParams
+
+		result, err := a.runAction(action, actionParams)
+		if err != nil {
+			return fmt.Errorf("error running action: %w", err)
+		}
+
+		stateResult := ActionState{ActionCurrentState{action, actionParams, subtask.Reasoning}, result}
+		job.Result.SetResult(stateResult)
+		job.CallbackWithResult(stateResult)
+		xlog.Debug("[subtask] Action executed", "agent", a.Character.Name, "action", action.Definition().Name, "result", result)
+		a.addFunctionResultToConversation(action, actionParams, result)
+	}
+
+	return nil
+}
+
 func (a *Agent) availableActions() Actions {
 	//	defaultActions := append(a.options.userActions, action.NewReply())
+
+	addPlanAction := func(actions Actions) Actions {
+		if !a.options.canPlan {
+			return actions
+		}
+		plannablesActions := []string{}
+		for _, a := range actions {
+			if a.Plannable() {
+				plannablesActions = append(plannablesActions, a.Definition().Name.String())
+			}
+		}
+		planAction := action.NewPlan(plannablesActions)
+		actions = append(actions, planAction)
+		return actions
+	}
+
 	defaultActions := append(a.mcpActions, a.options.userActions...)
 
 	if a.options.initiateConversations && a.selfEvaluationInProgress { // && self-evaluation..
@@ -224,7 +293,7 @@ func (a *Agent) availableActions() Actions {
 		//		acts = append(acts, action.NewStop())
 		//	}
 
-		return acts
+		return addPlanAction(acts)
 	}
 
 	if a.options.canStopItself {
@@ -232,14 +301,14 @@ func (a *Agent) availableActions() Actions {
 		if a.options.enableHUD {
 			acts = append(acts, action.NewState())
 		}
-		return acts
+		return addPlanAction(acts)
 	}
 
 	if a.options.enableHUD {
-		return append(defaultActions, action.NewState())
+		return addPlanAction(append(defaultActions, action.NewState()))
 	}
 
-	return defaultActions
+	return addPlanAction(defaultActions)
 }
 
 func (a *Agent) prepareHUD() (promptHUD *PromptHUD) {

core/agent/agent.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -571,6 +570,11 @@ func (a *Agent) consumeJob(job *Job, role string) {
 		return
 	}
 
+	if err := a.handlePlanning(ctx, job, chosenAction, actionParams, reasoning, pickTemplate); err != nil {
+		job.Result.Finish(fmt.Errorf("error running action: %w", err))
+		return
+	}
+
 	if !job.Callback(ActionCurrentState{chosenAction, actionParams, reasoning}) {
 		job.Result.SetResult(ActionState{ActionCurrentState{chosenAction, actionParams, reasoning}, action.ActionResult{Result: "stopped by callback"}})
 		job.Result.Conversation = a.currentConversation
@@ -620,27 +624,7 @@ func (a *Agent) consumeJob(job *Job, role string) {
 		job.CallbackWithResult(stateResult)
 		xlog.Debug("Action executed", "agent", a.Character.Name, "action", chosenAction.Definition().Name, "result", result)
 
-		// calling the function
+		a.addFunctionResultToConversation(chosenAction, actionParams, result)
-		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
-			Role: "assistant",
-			ToolCalls: []openai.ToolCall{
-				{
-					Type: openai.ToolTypeFunction,
-					Function: openai.FunctionCall{
-						Name:      chosenAction.Definition().Name.String(),
-						Arguments: actionParams.String(),
-					},
-				},
-			},
-		})
-
-		// result of calling the function
-		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
-			Role:       openai.ChatMessageRoleTool,
-			Content:    result.Result,
-			Name:       chosenAction.Definition().Name.String(),
-			ToolCallID: chosenAction.Definition().Name.String(),
-		})
 
 		//a.currentConversation = append(a.currentConversation, messages...)
 		//a.currentConversation = messages
@@ -776,8 +760,7 @@ func (a *Agent) consumeJob(job *Job, role string) {
 	}
 
 	// If we didn't got any message, we can use the response from the action
-	if chosenAction.Definition().Name.Is(action.ReplyActionName) && msg.Content == "" ||
+	if chosenAction.Definition().Name.Is(action.ReplyActionName) && msg.Content == "" {
-		strings.Contains(msg.Content, "<tool_call>") {
 		xlog.Info("No output returned from conversation, using the action response as a reply " + replyResponse.Message)
 
 		msg = openai.ChatCompletionMessage{
@@ -794,6 +777,30 @@ func (a *Agent) consumeJob(job *Job, role string) {
 	job.Result.Finish(nil)
 }
 
+func (a *Agent) addFunctionResultToConversation(chosenAction Action, actionParams action.ActionParams, result action.ActionResult) {
+	// calling the function
+	a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+		Role: "assistant",
+		ToolCalls: []openai.ToolCall{
+			{
+				Type: openai.ToolTypeFunction,
+				Function: openai.FunctionCall{
+					Name:      chosenAction.Definition().Name.String(),
+					Arguments: actionParams.String(),
+				},
+			},
+		},
+	})
+
+	// result of calling the function
+	a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+		Role:       openai.ChatMessageRoleTool,
+		Content:    result.Result,
+		Name:       chosenAction.Definition().Name.String(),
+		ToolCallID: chosenAction.Definition().Name.String(),
+	})
+}
+
 // This is running in the background.
 func (a *Agent) periodicallyRun(timer *time.Timer) {
 	// Remember always to reset the timer - if we don't the agent will stop..

core/agent/agent_test.go

@@ -36,6 +36,10 @@ type TestAction struct {
 	response map[string]string
 }
 
+func (a *TestAction) Plannable() bool {
+	return true
+}
+
 func (a *TestAction) Run(c context.Context, p action.ActionParams) (action.ActionResult, error) {
 	for k, r := range a.response {
 		if strings.Contains(strings.ToLower(p.String()), strings.ToLower(k)) {

core/agent/mcp.go

@@ -26,6 +26,10 @@ type mcpAction struct {
 	toolDescription string
 }
 
+func (a *mcpAction) Plannable() bool {
+	return true
+}
+
 func (m *mcpAction) Run(ctx context.Context, params action.ActionParams) (action.ActionResult, error) {
 	resp, err := m.mcpClient.CallTool(ctx, m.toolName, params)
 	if err != nil {

core/agent/options.go

@@ -26,6 +26,7 @@ type options struct {
 	canStopItself         bool
 	initiateConversations bool
 	forceReasoning        bool
+	canPlan               bool
 	characterfile         string
 	statefile             string
 	context               context.Context
@@ -127,6 +128,11 @@ var EnableInitiateConversations = func(o *options) error {
 	return nil
 }
 
+var EnablePlanning = func(o *options) error {
+	o.canPlan = true
+	return nil
+}
+
 // EnableStandaloneJob is an option to enable the agent
 // to run jobs in the background automatically
 var EnableStandaloneJob = func(o *options) error {

core/state/config.go

@@ -47,6 +47,7 @@ type AgentConfig struct {
 	StandaloneJob         bool   `json:"standalone_job" form:"standalone_job"`
 	RandomIdentity        bool   `json:"random_identity" form:"random_identity"`
 	InitiateConversations bool   `json:"initiate_conversations" form:"initiate_conversations"`
+	CanPlan               bool   `json:"enable_planning" form:"enable_planning"`
 	IdentityGuidance      string `json:"identity_guidance" form:"identity_guidance"`
 	PeriodicRuns          string `json:"periodic_runs" form:"periodic_runs"`
 	PermanentGoal         string `json:"permanent_goal" form:"permanent_goal"`

core/state/pool.go

@@ -332,6 +332,10 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig) error
 		opts = append(opts, CanStopItself)
 	}
 
+	if config.CanPlan {
+		opts = append(opts, EnablePlanning)
+	}
+
 	if config.InitiateConversations {
 		opts = append(opts, EnableInitiateConversations)
 	}

services/actions/browse.go

@@ -66,3 +66,7 @@ func (a *BrowseAction) Definition() action.ActionDefinition {
 		Required: []string{"url"},
 	}
 }
+
+func (a *BrowseAction) Plannable() bool {
+	return true
+}

services/actions/callagents.go

@@ -85,3 +85,7 @@ func (a *CallAgentAction) Definition() action.ActionDefinition {
 		Required: []string{"agent_name", "message"},
 	}
 }
+
+func (a *CallAgentAction) Plannable() bool {
+	return true
+}

services/actions/counter.go

@@ -93,3 +93,6 @@ func (a *CounterAction) Definition() action.ActionDefinition {
 	}
 }
 
+func (a *CounterAction) Plannable() bool {
+	return true
+}

services/actions/genimage.go

@@ -92,3 +92,7 @@ func (a *GenImageAction) Definition() action.ActionDefinition {
 		Required: []string{"prompt"},
 	}
 }
+
+func (a *GenImageAction) Plannable() bool {
+	return true
+}

services/actions/githubissuecloser.go

@@ -116,3 +116,7 @@ func (g *GithubIssuesCloser) Definition() action.ActionDefinition {
 		Required: []string{"issue_number", "repository", "owner"},
 	}
 }
+
+func (a *GithubIssuesCloser) Plannable() bool {
+	return true
+}

services/actions/githubissuecomment.go

@@ -103,3 +103,7 @@ func (g *GithubIssuesCommenter) Definition() action.ActionDefinition {
 		Required: []string{"issue_number", "repository", "owner", "comment"},
 	}
 }
+
+func (a *GithubIssuesCommenter) Plannable() bool {
+	return true
+}

services/actions/githubissuelabeler.go

@@ -118,3 +118,7 @@ func (g *GithubIssuesLabeler) Definition() action.ActionDefinition {
 		Required: []string{"issue_number", "repository", "owner", "label"},
 	}
 }
+
+func (a *GithubIssuesLabeler) Plannable() bool {
+	return true
+}

services/actions/githubissueopener.go

@@ -109,3 +109,7 @@ func (g *GithubIssuesOpener) Definition() action.ActionDefinition {
 		Required: []string{"title", "text", "owner", "repository"},
 	}
 }
+
+func (a *GithubIssuesOpener) Plannable() bool {
+	return true
+}

services/actions/githubissuereader.go

@@ -97,3 +97,7 @@ func (g *GithubIssuesReader) Definition() action.ActionDefinition {
 		Required: []string{"issue_number", "repository", "owner"},
 	}
 }
+
+func (a *GithubIssuesReader) Plannable() bool {
+	return true
+}

services/actions/githubissuesearch.go

@@ -106,3 +106,7 @@ func (g *GithubIssueSearch) Definition() action.ActionDefinition {
 		Required: []string{"query", "repository", "owner"},
 	}
 }
+
+func (a *GithubIssueSearch) Plannable() bool {
+	return true
+}

services/actions/githubrepositorycreateupdatecontent.go

@@ -142,3 +142,7 @@ func (g *GithubRepositoryCreateOrUpdateContent) Definition() action.ActionDefini
 		Required:    []string{"path", "repository", "owner", "content"},
 	}
 }
+
+func (a *GithubRepositoryCreateOrUpdateContent) Plannable() bool {
+	return true
+}

services/actions/githubrepositorygetcontent.go

@@ -107,3 +107,7 @@ func (g *GithubRepositoryGetContent) Definition() action.ActionDefinition {
 		Required: []string{"path", "repository", "owner"},
 	}
 }
+
+func (a *GithubRepositoryGetContent) Plannable() bool {
+	return true
+}

services/actions/githubrepositoryreadme.go

@@ -73,3 +73,7 @@ func (g *GithubRepositoryREADME) Definition() action.ActionDefinition {
 		Required: []string{"repository", "owner"},
 	}
 }
+
+func (a *GithubRepositoryREADME) Plannable() bool {
+	return true
+}

services/actions/scrape.go

@@ -54,3 +54,7 @@ func (a *ScraperAction) Definition() action.ActionDefinition {
 		Required: []string{"url"},
 	}
 }
+
+func (a *ScraperAction) Plannable() bool {
+	return true
+}

services/actions/search.go

@@ -85,3 +85,7 @@ func (a *SearchAction) Definition() action.ActionDefinition {
 		Required: []string{"query"},
 	}
 }
+
+func (a *SearchAction) Plannable() bool {
+	return true
+}

services/actions/sendmail.go

@@ -76,3 +76,7 @@ func (a *SendMailAction) Definition() action.ActionDefinition {
 		Required: []string{"to", "subject", "message"},
 	}
 }
+
+func (a *SendMailAction) Plannable() bool {
+	return true
+}

services/actions/shell.go

@@ -134,3 +134,7 @@ func sshCommand(privateKey, command, user, host string) (string, error) {
 
 	return string(output), nil
 }
+
+func (a *ShellAction) Plannable() bool {
+	return true
+}

services/actions/twitter_post.go

@@ -58,3 +58,7 @@ func (a *PostTweetAction) Definition() action.ActionDefinition {
 		Required: []string{"text"},
 	}
 }
+
+func (a *PostTweetAction) Plannable() bool {
+	return true
+}

services/actions/wikipedia.go

@@ -48,3 +48,7 @@ func (a *WikipediaAction) Definition() action.ActionDefinition {
 		Required: []string{"query"},
 	}
 }
+
+func (a *WikipediaAction) Plannable() bool {
+	return true
+}

webui/views/partials/agent-form.html

@@ -254,6 +254,16 @@
                 </label>
             </div>
     
+            <div class="mb-4">
+                <label for="enable_planning" class="checkbox-label">
+                    <span class="checkbox-custom">
+                        <input type="checkbox" name="enable_planning" id="enable_planning">
+                        <span class="checkmark"></span>
+                    </span>
+                    Enable Planning
+                </label>
+            </div>
+
             <div class="mb-4">
                 <label for="can_stop_itself" class="checkbox-label">
                     <span class="checkbox-custom">

webui/views/settings.html

@@ -336,6 +336,7 @@
             document.getElementById('kb_results').value = config.kb_results || '';
             document.getElementById('standalone_job').checked = config.standalone_job || false;
             document.getElementById('initiate_conversations').checked = config.initiate_conversations || false;
+            document.getElementById('enable_planning').checked = config.enable_planning || false;
             document.getElementById('can_stop_itself').checked = config.can_stop_itself || false;
             document.getElementById('random_identity').checked = config.random_identity || false;
             document.getElementById('long_term_memory').checked = config.long_term_memory || false;