reordering

core/action/custom.go

@@ -0,0 +1,125 @@
+package action
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/mudler/local-agent-framework/pkg/xlog"
+	"github.com/sashabaranov/go-openai/jsonschema"
+	"github.com/traefik/yaegi/interp"
+	"github.com/traefik/yaegi/stdlib"
+)
+
+func NewCustom(config map[string]string, goPkgPath string) (*CustomAction, error) {
+	a := &CustomAction{
+		config:    config,
+		goPkgPath: goPkgPath,
+	}
+
+	if err := a.initializeInterpreter(); err != nil {
+		return nil, err
+	}
+
+	if err := a.callInit(); err != nil {
+		xlog.Error("Error calling custom action init", "error", err)
+	}
+
+	return a, nil
+}
+
+type CustomAction struct {
+	config    map[string]string
+	goPkgPath string
+	i         *interp.Interpreter
+}
+
+func (a *CustomAction) callInit() error {
+	if a.i == nil {
+		return nil
+	}
+
+	v, err := a.i.Eval(fmt.Sprintf("%s.Init", a.config["name"]))
+	if err != nil {
+		return err
+	}
+
+	run := v.Interface().(func() error)
+
+	return run()
+}
+
+func (a *CustomAction) initializeInterpreter() error {
+	if _, exists := a.config["code"]; exists && a.i == nil {
+		unsafe := strings.ToLower(a.config["unsafe"]) == "true"
+		i := interp.New(interp.Options{
+			GoPath:       a.goPkgPath,
+			Unrestricted: unsafe,
+		})
+		if err := i.Use(stdlib.Symbols); err != nil {
+			return err
+		}
+
+		if _, exists := a.config["name"]; !exists {
+			a.config["name"] = "custom"
+		}
+
+		_, err := i.Eval(fmt.Sprintf("package %s\n%s", a.config["name"], a.config["code"]))
+		if err != nil {
+			return err
+		}
+
+		a.i = i
+	}
+
+	return nil
+}
+
+func (a *CustomAction) Run(ctx context.Context, params ActionParams) (string, error) {
+	v, err := a.i.Eval(fmt.Sprintf("%s.Run", a.config["name"]))
+	if err != nil {
+		return "", err
+	}
+
+	run := v.Interface().(func(map[string]interface{}) (string, error))
+
+	return run(params)
+}
+
+func (a *CustomAction) Definition() ActionDefinition {
+
+	v, err := a.i.Eval(fmt.Sprintf("%s.Definition", a.config["name"]))
+	if err != nil {
+		xlog.Error("Error getting custom action definition", "error", err)
+		return ActionDefinition{}
+	}
+
+	properties := v.Interface().(func() map[string][]string)
+
+	v, err = a.i.Eval(fmt.Sprintf("%s.RequiredFields", a.config["name"]))
+	if err != nil {
+		xlog.Error("Error getting custom action definition", "error", err)
+		return ActionDefinition{}
+	}
+
+	requiredFields := v.Interface().(func() []string)
+
+	prop := map[string]jsonschema.Definition{}
+
+	for k, v := range properties() {
+		if len(v) != 2 {
+			xlog.Error("Invalid property definition", "property", k)
+			continue
+		}
+		prop[k] = jsonschema.Definition{
+			Type:        jsonschema.DataType(v[0]),
+			Description: v[1],
+		}
+	}
+	return ActionDefinition{
+		Name:        ActionDefinitionName(a.config["name"]),
+		Description: a.config["description"],
+		Properties:  prop,
+		Required:    requiredFields(),
+	}
+}

core/action/custom_test.go

@@ -0,0 +1,86 @@
+package action_test
+
+import (
+	"context"
+
+	. "github.com/mudler/local-agent-framework/core/action"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+var _ = Describe("Agent custom action", func() {
+	Context("custom action", func() {
+		It("initializes correctly", func() {
+
+			testCode := `
+
+import (
+	"encoding/json"
+)
+type Params struct {
+	Foo string
+}
+
+func Run(config map[string]interface{}) (string, error) {
+
+p := Params{}
+b, err := json.Marshal(config)
+	if err != nil {
+		return "", err
+	}
+if err := json.Unmarshal(b, &p); err != nil {
+	return "", err
+}
+
+return p.Foo, nil
+}
+
+func Definition() map[string][]string {
+return map[string][]string{
+	"foo": []string{
+		"string",
+		"The foo value",
+		},
+	}
+}
+
+func RequiredFields() []string {
+return []string{"foo"}
+}
+
+			`
+
+			customAction, err := NewCustom(
+				map[string]string{
+					"code":        testCode,
+					"name":        "test",
+					"description": "A test action",
+				},
+				"",
+			)
+			Expect(err).ToNot(HaveOccurred())
+
+			definition := customAction.Definition()
+			Expect(definition).To(Equal(ActionDefinition{
+				Properties: map[string]jsonschema.Definition{
+					"foo": {
+						Type:        jsonschema.String,
+						Description: "The foo value",
+					},
+				},
+				Required:    []string{"foo"},
+				Name:        "test",
+				Description: "A test action",
+			}))
+
+			runResult, err := customAction.Run(context.Background(), ActionParams{
+				"Foo": "bar",
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(runResult).To(Equal("bar"))
+
+		})
+	})
+})

core/action/definition.go

@@ -0,0 +1,81 @@
+package action
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/sashabaranov/go-openai"
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+type ActionContext struct {
+	context.Context
+	cancelFunc context.CancelFunc
+}
+
+func (ac *ActionContext) Cancel() {
+	if ac.cancelFunc == nil {
+		ac.cancelFunc()
+	}
+}
+
+func NewContext(ctx context.Context, cancel context.CancelFunc) *ActionContext {
+	return &ActionContext{
+		Context:    ctx,
+		cancelFunc: cancel,
+	}
+}
+
+type ActionParams map[string]interface{}
+
+func (ap ActionParams) Read(s string) error {
+	err := json.Unmarshal([]byte(s), &ap)
+	return err
+}
+
+func (ap ActionParams) String() string {
+	b, _ := json.Marshal(ap)
+	return string(b)
+}
+
+func (ap ActionParams) Unmarshal(v interface{}) error {
+	b, err := json.Marshal(ap)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(b, v); err != nil {
+		return err
+	}
+	return nil
+}
+
+//type ActionDefinition openai.FunctionDefinition
+
+type ActionDefinition struct {
+	Properties  map[string]jsonschema.Definition
+	Required    []string
+	Name        ActionDefinitionName
+	Description string
+}
+
+type ActionDefinitionName string
+
+func (a ActionDefinitionName) Is(name string) bool {
+	return string(a) == name
+}
+
+func (a ActionDefinitionName) String() string {
+	return string(a)
+}
+
+func (a ActionDefinition) ToFunctionDefinition() openai.FunctionDefinition {
+	return openai.FunctionDefinition{
+		Name:        a.Name.String(),
+		Description: a.Description,
+		Parameters: jsonschema.Definition{
+			Type:       jsonschema.Object,
+			Properties: a.Properties,
+			Required:   a.Required,
+		},
+	}
+}

core/action/intention.go

@@ -0,0 +1,45 @@
+package action
+
+import (
+	"context"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+// NewIntention creates a new intention action
+// The inention action is special as it tries to identify
+// a tool to use and a reasoning over to use it
+func NewIntention(s ...string) *IntentAction {
+	return &IntentAction{tools: s}
+}
+
+type IntentAction struct {
+	tools []string
+}
+type IntentResponse struct {
+	Tool      string `json:"tool"`
+	Reasoning string `json:"reasoning"`
+}
+
+func (a *IntentAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *IntentAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        "pick_tool",
+		Description: "Pick a tool",
+		Properties: map[string]jsonschema.Definition{
+			"reasoning": {
+				Type:        jsonschema.String,
+				Description: "A detailed reasoning on why you want to call this tool.",
+			},
+			"tool": {
+				Type:        jsonschema.String,
+				Description: "The tool you want to use",
+				Enum:        a.tools,
+			},
+		},
+		Required: []string{"tool", "reasoning"},
+	}
+}

core/action/newconversation.go

@@ -0,0 +1,37 @@
+package action
+
+import (
+	"context"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+const ConversationActionName = "new_conversation"
+
+func NewConversation() *ConversationAction {
+	return &ConversationAction{}
+}
+
+type ConversationAction struct{}
+
+type ConversationActionResponse struct {
+	Message string `json:"message"`
+}
+
+func (a *ConversationAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *ConversationAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        ConversationActionName,
+		Description: "Use this tool to initiate a new conversation or to notify something.",
+		Properties: map[string]jsonschema.Definition{
+			"message": {
+				Type:        jsonschema.String,
+				Description: "The message to start the conversation",
+			},
+		},
+		Required: []string{"message"},
+	}
+}

core/action/noreply.go

@@ -0,0 +1,24 @@
+package action
+
+import "context"
+
+// StopActionName is the name of the action
+// used by the LLM to stop any further action
+const StopActionName = "stop"
+
+func NewStop() *StopAction {
+	return &StopAction{}
+}
+
+type StopAction struct{}
+
+func (a *StopAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *StopAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        StopActionName,
+		Description: "Use this tool to stop any further action and stop the conversation. You must use this when it looks like there is a conclusion to the conversation or the topic diverged too much from the original conversation. For instance if the user offer his help and you already replied with a message, you can use this tool to stop the conversation.",
+	}
+}

core/action/plan.go

@@ -0,0 +1,53 @@
+package action
+
+import (
+	"context"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+// PlanActionName is the name of the plan action
+// used by the LLM to schedule more actions
+const PlanActionName = "plan"
+
+func NewPlan() *PlanAction {
+	return &PlanAction{}
+}
+
+type PlanAction struct{}
+
+type PlanResult struct {
+	Subtasks []PlanSubtask `json:"subtasks"`
+}
+type PlanSubtask struct {
+	Action    string `json:"action"`
+	Reasoning string `json:"reasoning"`
+}
+
+func (a *PlanAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *PlanAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        PlanActionName,
+		Description: "The assistant for solving complex tasks that involves calling more functions in sequence, replies with the action.",
+		Properties: map[string]jsonschema.Definition{
+			"subtasks": {
+				Type:        jsonschema.Array,
+				Description: "The message to reply with",
+				Properties: map[string]jsonschema.Definition{
+					"action": {
+						Type:        jsonschema.String,
+						Description: "The action to call",
+					},
+					"reasoning": {
+						Type:        jsonschema.String,
+						Description: "The reasoning for calling this action",
+					},
+				},
+			},
+		},
+		Required: []string{"subtasks"},
+	}
+}

core/action/reasoning.go

@@ -0,0 +1,38 @@
+package action
+
+import (
+	"context"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+// NewReasoning creates a new reasoning action
+// The reasoning action is special as it tries to force the LLM
+// to think about what to do next
+func NewReasoning() *ReasoningAction {
+	return &ReasoningAction{}
+}
+
+type ReasoningAction struct{}
+
+type ReasoningResponse struct {
+	Reasoning string `json:"reasoning"`
+}
+
+func (a *ReasoningAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *ReasoningAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        "pick_action",
+		Description: "try to understand what's the best thing to do and pick an action with a reasoning",
+		Properties: map[string]jsonschema.Definition{
+			"reasoning": {
+				Type:        jsonschema.String,
+				Description: "A detailed reasoning on what would you do in this situation.",
+			},
+		},
+		Required: []string{"reasoning"},
+	}
+}

core/action/reply.go

@@ -0,0 +1,40 @@
+package action
+
+import (
+	"context"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+// ReplyActionName is the name of the reply action
+// used by the LLM to reply to the user without
+// any additional processing
+const ReplyActionName = "reply"
+
+func NewReply() *ReplyAction {
+	return &ReplyAction{}
+}
+
+type ReplyAction struct{}
+
+type ReplyResponse struct {
+	Message string `json:"message"`
+}
+
+func (a *ReplyAction) Run(context.Context, ActionParams) (string, error) {
+	return "no-op", nil
+}
+
+func (a *ReplyAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        ReplyActionName,
+		Description: "Use this tool to reply to the user once we have all the informations we need.",
+		Properties: map[string]jsonschema.Definition{
+			"message": {
+				Type:        jsonschema.String,
+				Description: "The message to reply with",
+			},
+		},
+		Required: []string{"message"},
+	}
+}

core/action/state.go

@@ -0,0 +1,93 @@
+package action
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+const StateActionName = "update_state"
+
+func NewState() *StateAction {
+	return &StateAction{}
+}
+
+type StateAction struct{}
+
+// State is the structure
+// that is used to keep track of the current state
+// and the Agent's short memory that it can update
+// Besides a long term memory that is accessible by the agent (With vector database),
+// And a context memory (that is always powered by a vector database),
+// this memory is the shorter one that the LLM keeps across conversation and across its
+// reasoning process's and life time.
+// TODO: A special action is then used to let the LLM itself update its memory
+// periodically during self-processing, and the same action is ALSO exposed
+// during the conversation to let the user put for example, a new goal to the agent.
+type StateResult struct {
+	NowDoing    string   `json:"doing_now"`
+	DoingNext   string   `json:"doing_next"`
+	DoneHistory []string `json:"done_history"`
+	Memories    []string `json:"memories"`
+	Goal        string   `json:"goal"`
+}
+
+func (a *StateAction) Run(context.Context, ActionParams) (string, error) {
+	return "internal state has been updated", nil
+}
+
+func (a *StateAction) Definition() ActionDefinition {
+	return ActionDefinition{
+		Name:        StateActionName,
+		Description: "update the agent state (short memory) with the current state of the conversation.",
+		Properties: map[string]jsonschema.Definition{
+			"goal": {
+				Type:        jsonschema.String,
+				Description: "The current goal of the agent.",
+			},
+			"doing_next": {
+				Type:        jsonschema.String,
+				Description: "The next action the agent will do.",
+			},
+			"done_history": {
+				Type: jsonschema.Array,
+				Items: &jsonschema.Definition{
+					Type: jsonschema.String,
+				},
+				Description: "A list of actions that the agent has done.",
+			},
+			"now_doing": {
+				Type:        jsonschema.String,
+				Description: "The current action the agent is doing.",
+			},
+			"memories": {
+				Type: jsonschema.Array,
+				Items: &jsonschema.Definition{
+					Type: jsonschema.String,
+				},
+				Description: "A list of memories to keep between conversations.",
+			},
+		},
+	}
+}
+
+const fmtT = `=====================
+NowDoing: %s
+DoingNext: %s
+Your current goal is: %s
+You have done: %+v
+You have a short memory with: %+v
+=====================
+`
+
+func (c StateResult) String() string {
+	return fmt.Sprintf(
+		fmtT,
+		c.NowDoing,
+		c.DoingNext,
+		c.Goal,
+		c.DoneHistory,
+		c.Memories,
+	)
+}

core/agent/actions.go

@@ -0,0 +1,316 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/mudler/local-agent-framework/core/action"
+	"github.com/mudler/local-agent-framework/pkg/xlog"
+
+	"github.com/sashabaranov/go-openai"
+)
+
+type ActionState struct {
+	ActionCurrentState
+	Result string
+}
+
+type ActionCurrentState struct {
+	Action    Action
+	Params    action.ActionParams
+	Reasoning string
+}
+
+// Actions is something the agent can do
+type Action interface {
+	Run(ctx context.Context, action action.ActionParams) (string, error)
+	Definition() action.ActionDefinition
+}
+
+type Actions []Action
+
+func (a Actions) ToTools() []openai.Tool {
+	tools := []openai.Tool{}
+	for _, action := range a {
+		tools = append(tools, openai.Tool{
+			Type:     openai.ToolTypeFunction,
+			Function: action.Definition().ToFunctionDefinition(),
+		})
+	}
+	return tools
+}
+
+func (a Actions) Find(name string) Action {
+	for _, action := range a {
+		if action.Definition().Name.Is(name) {
+			return action
+		}
+	}
+	return nil
+}
+
+type decisionResult struct {
+	actionParams action.ActionParams
+	message      string
+	actioName    string
+}
+
+// decision forces the agent to take one of the available actions
+func (a *Agent) decision(
+	ctx context.Context,
+	conversation []openai.ChatCompletionMessage,
+	tools []openai.Tool, toolchoice any) (*decisionResult, error) {
+
+	decision := openai.ChatCompletionRequest{
+		Model:      a.options.LLMAPI.Model,
+		Messages:   conversation,
+		Tools:      tools,
+		ToolChoice: toolchoice,
+	}
+
+	resp, err := a.client.CreateChatCompletion(ctx, decision)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(resp.Choices) != 1 {
+		return nil, fmt.Errorf("no choices: %d", len(resp.Choices))
+	}
+
+	msg := resp.Choices[0].Message
+	if len(msg.ToolCalls) != 1 {
+		return &decisionResult{message: msg.Content}, nil
+	}
+
+	params := action.ActionParams{}
+	if err := params.Read(msg.ToolCalls[0].Function.Arguments); err != nil {
+		return nil, err
+	}
+
+	return &decisionResult{actionParams: params, actioName: msg.ToolCalls[0].Function.Name, message: msg.Content}, nil
+}
+
+type Messages []openai.ChatCompletionMessage
+
+func (m Messages) ToOpenAI() []openai.ChatCompletionMessage {
+	return []openai.ChatCompletionMessage(m)
+}
+
+func (m Messages) String() string {
+	s := ""
+	for _, cc := range m {
+		s += cc.Role + ": " + cc.Content + "\n"
+	}
+	return s
+}
+
+func (m Messages) Exist(content string) bool {
+	for _, cc := range m {
+		if cc.Content == content {
+			return true
+		}
+	}
+	return false
+}
+
+func (a *Agent) generateParameters(ctx context.Context, pickTemplate string, act Action, c []openai.ChatCompletionMessage, reasoning string) (*decisionResult, error) {
+
+	var promptHUD *PromptHUD
+	if a.options.enableHUD {
+		h := a.prepareHUD()
+		promptHUD = &h
+	}
+
+	stateHUD, err := renderTemplate(pickTemplate, promptHUD, a.systemInternalActions(), reasoning)
+	if err != nil {
+		return nil, err
+	}
+
+	// check if there is already a message with the hud in the conversation already, otherwise
+	// add a message at the top with it
+
+	conversation := c
+
+	if !Messages(c).Exist(stateHUD) && a.options.enableHUD {
+		conversation = append([]openai.ChatCompletionMessage{
+			{
+				Role:    "system",
+				Content: stateHUD,
+			},
+		}, conversation...)
+	}
+
+	cc := conversation
+	if a.options.forceReasoning {
+		cc = append(conversation, openai.ChatCompletionMessage{
+			Role:    "system",
+			Content: fmt.Sprintf("The agent decided to use the tool %s with the following reasoning: %s", act.Definition().Name, reasoning),
+		})
+	}
+
+	return a.decision(ctx,
+		cc,
+		a.systemInternalActions().ToTools(),
+		openai.ToolChoice{
+			Type:     openai.ToolTypeFunction,
+			Function: openai.ToolFunction{Name: act.Definition().Name.String()},
+		},
+	)
+}
+
+func (a *Agent) systemInternalActions() Actions {
+	//	defaultActions := append(a.options.userActions, action.NewReply())
+	defaultActions := a.options.userActions
+
+	if a.options.initiateConversations && a.selfEvaluationInProgress { // && self-evaluation..
+		acts := append(defaultActions, action.NewConversation())
+		if a.options.enableHUD {
+			acts = append(acts, action.NewState())
+		}
+		//if a.options.canStopItself {
+		//		acts = append(acts, action.NewStop())
+		//	}
+
+		return acts
+	}
+
+	if a.options.canStopItself {
+		acts := append(defaultActions, action.NewStop())
+		if a.options.enableHUD {
+			acts = append(acts, action.NewState())
+		}
+		return acts
+	}
+
+	if a.options.enableHUD {
+		return append(defaultActions, action.NewState())
+	}
+
+	return defaultActions
+}
+
+func (a *Agent) prepareHUD() PromptHUD {
+	return PromptHUD{
+		Character:     a.Character,
+		CurrentState:  *a.currentState,
+		PermanentGoal: a.options.permanentGoal,
+		ShowCharacter: a.options.showCharacter,
+	}
+}
+
+// pickAction picks an action based on the conversation
+func (a *Agent) pickAction(ctx context.Context, templ string, messages []openai.ChatCompletionMessage) (Action, action.ActionParams, string, error) {
+	c := messages
+
+	if !a.options.forceReasoning {
+		// We also could avoid to use functions here and get just a reply from the LLM
+		// and then use the reply to get the action
+		thought, err := a.decision(ctx,
+			messages,
+			a.systemInternalActions().ToTools(),
+			nil)
+		if err != nil {
+			return nil, nil, "", err
+		}
+
+		xlog.Debug(fmt.Sprintf("thought action Name: %v", thought.actioName))
+		xlog.Debug(fmt.Sprintf("thought message: %v", thought.message))
+
+		// Find the action
+		chosenAction := a.systemInternalActions().Find(thought.actioName)
+		if chosenAction == nil || thought.actioName == "" {
+			xlog.Debug(fmt.Sprintf("no answer"))
+
+			// LLM replied with an answer?
+			//fmt.Errorf("no action found for intent:" + thought.actioName)
+			return nil, nil, thought.message, nil
+		}
+		xlog.Debug(fmt.Sprintf("chosenAction: %v", chosenAction.Definition().Name))
+		return chosenAction, thought.actionParams, thought.message, nil
+	}
+
+	var promptHUD *PromptHUD
+	if a.options.enableHUD {
+		h := a.prepareHUD()
+		promptHUD = &h
+	}
+
+	prompt, err := renderTemplate(templ, promptHUD, a.systemInternalActions(), "")
+	if err != nil {
+		return nil, nil, "", err
+	}
+	// Get the LLM to think on what to do
+	// and have a thought
+	if !Messages(c).Exist(prompt) {
+		c = append([]openai.ChatCompletionMessage{
+			{
+				Role:    "system",
+				Content: prompt,
+			},
+		}, c...)
+	}
+
+	// We also could avoid to use functions here and get just a reply from the LLM
+	// and then use the reply to get the action
+	thought, err := a.decision(ctx,
+		c,
+		Actions{action.NewReasoning()}.ToTools(),
+		action.NewReasoning().Definition().Name)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	reason := ""
+	response := &action.ReasoningResponse{}
+	if thought.actionParams != nil {
+		if err := thought.actionParams.Unmarshal(response); err != nil {
+			return nil, nil, "", err
+		}
+		reason = response.Reasoning
+	}
+	if thought.message != "" {
+		reason = thought.message
+	}
+
+	// From the thought, get the action call
+	// Get all the available actions IDs
+	actionsID := []string{}
+	for _, m := range a.systemInternalActions() {
+		actionsID = append(actionsID, m.Definition().Name.String())
+	}
+	intentionsTools := action.NewIntention(actionsID...)
+
+	//XXX: Why we add the reason here?
+	params, err := a.decision(ctx,
+		append(c, openai.ChatCompletionMessage{
+			Role:    "system",
+			Content: "Given the assistant thought, pick the relevant action: " + reason,
+		}),
+		Actions{intentionsTools}.ToTools(),
+		intentionsTools.Definition().Name)
+	if err != nil {
+		return nil, nil, "", fmt.Errorf("failed to get the action tool parameters: %v", err)
+	}
+
+	actionChoice := action.IntentResponse{}
+
+	if params.actionParams == nil {
+		return nil, nil, params.message, nil
+	}
+
+	err = params.actionParams.Unmarshal(&actionChoice)
+	if err != nil {
+		return nil, nil, "", err
+	}
+
+	if actionChoice.Tool == "" || actionChoice.Tool == "none" {
+		return nil, nil, "", fmt.Errorf("no intent detected")
+	}
+
+	// Find the action
+	chosenAction := a.systemInternalActions().Find(actionChoice.Tool)
+	if chosenAction == nil {
+		return nil, nil, "", fmt.Errorf("no action found for intent:" + actionChoice.Tool)
+	}
+
+	return chosenAction, nil, actionChoice.Reasoning, nil
+}

core/agent/agent.go

@@ -0,0 +1,838 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mudler/local-agent-framework/pkg/xlog"
+
+	"github.com/mudler/local-agent-framework/core/action"
+	"github.com/mudler/local-agent-framework/pkg/llm"
+	"github.com/sashabaranov/go-openai"
+)
+
+const (
+	UserRole      = "user"
+	AssistantRole = "assistant"
+	SystemRole    = "system"
+)
+
+type Agent struct {
+	sync.Mutex
+	options       *options
+	Character     Character
+	client        *openai.Client
+	jobQueue      chan *Job
+	actionContext *action.ActionContext
+	context       *action.ActionContext
+
+	currentReasoning         string
+	currentState             *action.StateResult
+	nextAction               Action
+	nextActionParams         *action.ActionParams
+	currentConversation      Messages
+	selfEvaluationInProgress bool
+	pause                    bool
+
+	newConversations chan openai.ChatCompletionMessage
+}
+
+type RAGDB interface {
+	Store(s string) error
+	Reset() error
+	Search(s string, similarEntries int) ([]string, error)
+	Count() int
+}
+
+func New(opts ...Option) (*Agent, error) {
+	options, err := newOptions(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to set options: %v", err)
+	}
+
+	client := llm.NewClient(options.LLMAPI.APIKey, options.LLMAPI.APIURL, options.timeout)
+
+	c := context.Background()
+	if options.context != nil {
+		c = options.context
+	}
+
+	ctx, cancel := context.WithCancel(c)
+	a := &Agent{
+		jobQueue:     make(chan *Job),
+		options:      options,
+		client:       client,
+		Character:    options.character,
+		currentState: &action.StateResult{},
+		context:      action.NewContext(ctx, cancel),
+	}
+
+	if a.options.statefile != "" {
+		if _, err := os.Stat(a.options.statefile); err == nil {
+			if err = a.LoadState(a.options.statefile); err != nil {
+				return a, fmt.Errorf("failed to load state: %v", err)
+			}
+		}
+	}
+
+	// var programLevel = new(xlog.LevelVar) // Info by default
+	// h := xlog.NewTextHandler(os.Stdout, &xlog.HandlerOptions{Level: programLevel})
+	// xlog = xlog.New(h)
+	//programLevel.Set(a.options.logLevel)
+
+	xlog.Info(
+		"Agent created",
+		"agent", a.Character.Name,
+		"character", a.Character.String(),
+		"state", a.State().String(),
+		"goal", a.options.permanentGoal,
+	)
+
+	return a, nil
+}
+
+// StopAction stops the current action
+// if any. Can be called before adding a new job.
+func (a *Agent) StopAction() {
+	a.Lock()
+	defer a.Unlock()
+	if a.actionContext != nil {
+		xlog.Debug("Stopping current action", "agent", a.Character.Name)
+		a.actionContext.Cancel()
+	}
+}
+
+func (a *Agent) Context() context.Context {
+	return a.context.Context
+}
+
+func (a *Agent) ActionContext() context.Context {
+	return a.actionContext.Context
+}
+
+func (a *Agent) ConversationChannel() chan openai.ChatCompletionMessage {
+	return a.newConversations
+}
+
+// Ask is a pre-emptive, blocking call that returns the response as soon as it's ready.
+// It discards any other computation.
+func (a *Agent) Ask(opts ...JobOption) *JobResult {
+	xlog.Debug("Agent is being asked", "agent", a.Character.Name)
+	defer func() {
+		xlog.Debug("Agent has finished being asked", "agent", a.Character.Name)
+	}()
+
+	//a.StopAction()
+	j := NewJob(
+		append(
+			opts,
+			WithReasoningCallback(a.options.reasoningCallback),
+			WithResultCallback(a.options.resultCallback),
+		)...,
+	)
+	xlog.Debug("Job created", "agent", a.Character.Name, "job", j)
+	a.jobQueue <- j
+	xlog.Debug("Waiting result", "agent", a.Character.Name, "job", j)
+	return j.Result.WaitResult()
+}
+
+func (a *Agent) CurrentConversation() []openai.ChatCompletionMessage {
+	a.Lock()
+	defer a.Unlock()
+	return a.currentConversation
+}
+
+func (a *Agent) SetConversation(conv []openai.ChatCompletionMessage) {
+	a.Lock()
+	defer a.Unlock()
+	a.currentConversation = conv
+}
+
+func (a *Agent) askLLM(ctx context.Context, conversation []openai.ChatCompletionMessage) (openai.ChatCompletionMessage, error) {
+	resp, err := a.client.CreateChatCompletion(ctx,
+		openai.ChatCompletionRequest{
+			Model:    a.options.LLMAPI.Model,
+			Messages: conversation,
+		},
+	)
+	if err != nil {
+		return openai.ChatCompletionMessage{}, err
+	}
+
+	if len(resp.Choices) != 1 {
+		return openai.ChatCompletionMessage{}, fmt.Errorf("no enough choices: %w", err)
+	}
+
+	return resp.Choices[0].Message, nil
+}
+
+func (a *Agent) ResetConversation() {
+	a.Lock()
+	defer a.Unlock()
+
+	xlog.Info("Resetting conversation", "agent", a.Character.Name)
+
+	// store into memory the conversation before pruning it
+	// TODO: Shall we summarize the conversation into a bullet list of highlights
+	// using the LLM instead?
+	if a.options.enableLongTermMemory {
+		xlog.Info("Saving conversation", "agent", a.Character.Name, "conversation size", len(a.currentConversation))
+
+		if a.options.enableSummaryMemory && len(a.currentConversation) > 0 {
+
+			msg, err := a.askLLM(a.context.Context, []openai.ChatCompletionMessage{{
+				Role:    "user",
+				Content: "Summarize the conversation below, keep the highlights as a bullet list:\n" + Messages(a.currentConversation).String(),
+			}})
+			if err != nil {
+				xlog.Error("Error summarizing conversation", "error", err)
+			}
+
+			if err := a.options.ragdb.Store(msg.Content); err != nil {
+				xlog.Error("Error storing into memory", "error", err)
+			}
+		} else {
+			for _, message := range a.currentConversation {
+				if message.Role == "user" {
+					if err := a.options.ragdb.Store(message.Content); err != nil {
+						xlog.Error("Error storing into memory", "error", err)
+					}
+				}
+			}
+		}
+
+	}
+
+	a.currentConversation = []openai.ChatCompletionMessage{}
+}
+
+var ErrContextCanceled = fmt.Errorf("context canceled")
+
+func (a *Agent) Stop() {
+	a.Lock()
+	defer a.Unlock()
+	a.context.Cancel()
+}
+
+func (a *Agent) Pause() {
+	a.Lock()
+	defer a.Unlock()
+	a.pause = true
+}
+
+func (a *Agent) Resume() {
+	a.Lock()
+	defer a.Unlock()
+	a.pause = false
+}
+
+func (a *Agent) Paused() bool {
+	a.Lock()
+	defer a.Unlock()
+	return a.pause
+}
+
+func (a *Agent) Memory() RAGDB {
+	return a.options.ragdb
+}
+
+func (a *Agent) runAction(chosenAction Action, params action.ActionParams) (result string, err error) {
+	for _, action := range a.systemInternalActions() {
+		if action.Definition().Name == chosenAction.Definition().Name {
+			if result, err = action.Run(a.actionContext, params); err != nil {
+				return "", fmt.Errorf("error running action: %w", err)
+			}
+		}
+	}
+
+	xlog.Info("Running action", "action", chosenAction.Definition().Name, "agent", a.Character.Name)
+
+	if chosenAction.Definition().Name.Is(action.StateActionName) {
+		// We need to store the result in the state
+		state := action.StateResult{}
+
+		err = params.Unmarshal(&state)
+		if err != nil {
+			return "", fmt.Errorf("error unmarshalling state of the agent: %w", err)
+		}
+		// update the current state with the one we just got from the action
+		a.currentState = &state
+
+		// update the state file
+		if a.options.statefile != "" {
+			if err := a.SaveState(a.options.statefile); err != nil {
+				return "", err
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func (a *Agent) consumeJob(job *Job, role string) {
+	a.Lock()
+	paused := a.pause
+	a.Unlock()
+
+	if paused {
+		xlog.Info("Agent is paused, skipping job", "agent", a.Character.Name)
+		job.Result.Finish(fmt.Errorf("agent is paused"))
+		return
+	}
+
+	// We are self evaluating if we consume the job as a system role
+	selfEvaluation := role == SystemRole
+
+	a.Lock()
+	// Set the action context
+	ctx, cancel := context.WithCancel(context.Background())
+	a.actionContext = action.NewContext(ctx, cancel)
+	a.selfEvaluationInProgress = selfEvaluation
+	if len(job.conversationHistory) != 0 {
+		a.currentConversation = job.conversationHistory
+	}
+	a.Unlock()
+
+	defer func() {
+		a.Lock()
+		if a.actionContext != nil {
+			a.actionContext.Cancel()
+			a.actionContext = nil
+		}
+		a.Unlock()
+	}()
+
+	if selfEvaluation {
+		defer func() {
+			a.Lock()
+			a.selfEvaluationInProgress = false
+			a.Unlock()
+		}()
+	}
+
+	//if job.Image != "" {
+	// TODO: Use llava to explain the image content
+	//}
+	// Add custom prompts
+	for _, prompt := range a.options.prompts {
+		message := prompt.Render(a)
+		if !Messages(a.currentConversation).Exist(a.options.systemPrompt) {
+			a.currentConversation = append([]openai.ChatCompletionMessage{
+				{
+					Role:    prompt.Role(),
+					Content: message,
+				}}, a.currentConversation...)
+		}
+	}
+
+	// TODO: move to a Promptblock?
+	if a.options.systemPrompt != "" {
+		if !Messages(a.currentConversation).Exist(a.options.systemPrompt) {
+			a.currentConversation = append([]openai.ChatCompletionMessage{
+				{
+					Role:    "system",
+					Content: a.options.systemPrompt,
+				}}, a.currentConversation...)
+		}
+	}
+
+	if job.Text != "" {
+		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+			Role:    role,
+			Content: job.Text,
+		})
+	}
+
+	// TODO: move to a promptblock?
+	// RAG
+	if a.options.enableLongTermMemory && len(a.currentConversation) > 0 {
+		// Walk conversation from bottom to top, and find the first message of the user
+		// to use it as a query to the KB
+		var userMessage string
+		for i := len(a.currentConversation) - 1; i >= 0; i-- {
+			xlog.Info("[Long term memory] Conversation", "role", a.currentConversation[i].Role, "Content", a.currentConversation[i].Content)
+			if a.currentConversation[i].Role == "user" {
+				userMessage = a.currentConversation[i].Content
+				break
+			}
+		}
+		xlog.Info("[Long term memory] User message", "agent", a.Character.Name, "message", userMessage)
+
+		if userMessage != "" {
+			results, err := a.options.ragdb.Search(userMessage, a.options.kbResults)
+			if err != nil {
+				xlog.Info("Error finding similar strings inside KB:", "error", err)
+
+				//	job.Result.Finish(fmt.Errorf("error finding similar strings inside KB: %w", err))
+				//	return
+			}
+
+			if len(results) != 0 {
+
+				formatResults := ""
+				for _, r := range results {
+					formatResults += fmt.Sprintf("- %s \n", r)
+				}
+				xlog.Info("Found similar strings in KB", "agent", a.Character.Name, "results", formatResults)
+
+				// a.currentConversation = append(a.currentConversation,
+				// 	openai.ChatCompletionMessage{
+				// 		Role:    "system",
+				// 		Content: fmt.Sprintf("Given the user input you have the following in memory:\n%s", formatResults),
+				// 	},
+				// )
+				a.currentConversation = append([]openai.ChatCompletionMessage{
+					{
+						Role:    "system",
+						Content: fmt.Sprintf("Given the user input you have the following in memory:\n%s", formatResults),
+					}}, a.currentConversation...)
+			}
+		}
+	} else {
+		xlog.Info("[Long term memory] No conversation available", "agent", a.Character.Name)
+	}
+
+	var pickTemplate string
+	var reEvaluationTemplate string
+
+	if selfEvaluation {
+		pickTemplate = pickSelfTemplate
+		reEvaluationTemplate = reSelfEvalTemplate
+	} else {
+		pickTemplate = pickActionTemplate
+		reEvaluationTemplate = reEvalTemplate
+	}
+
+	// choose an action first
+	var chosenAction Action
+	var reasoning string
+	var actionParams action.ActionParams
+
+	if a.nextAction != nil {
+		// if we are being re-evaluated, we already have the action
+		// and the reasoning. Consume it here and reset it
+		chosenAction = a.nextAction
+		reasoning = a.currentReasoning
+		actionParams = *a.nextActionParams
+		a.currentReasoning = ""
+		a.nextActionParams = nil
+		a.nextAction = nil
+	} else {
+		var err error
+		chosenAction, actionParams, reasoning, err = a.pickAction(ctx, pickTemplate, a.currentConversation)
+		if err != nil {
+			xlog.Error("Error picking action", "error", err)
+			job.Result.Finish(err)
+			return
+		}
+	}
+
+	//xlog.Debug("Picked action", "agent", a.Character.Name, "action", chosenAction.Definition().Name, "reasoning", reasoning)
+	if chosenAction == nil {
+		// If no action was picked up, the reasoning is the message returned by the assistant
+		// so we can consume it as if it was a reply.
+		//job.Result.SetResult(ActionState{ActionCurrentState{nil, nil, "No action to do, just reply"}, ""})
+		//job.Result.Finish(fmt.Errorf("no action to do"))\
+		xlog.Info("No action to do, just reply", "agent", a.Character.Name, "reasoning", reasoning)
+
+		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+			Role:    "assistant",
+			Content: reasoning,
+		})
+		job.Result.SetResponse(reasoning)
+		job.Result.Finish(nil)
+		return
+	}
+
+	if chosenAction.Definition().Name.Is(action.StopActionName) {
+		xlog.Info("LLM decided to stop")
+		job.Result.Finish(nil)
+		return
+	}
+
+	// if we force a reasoning, we need to generate the parameters
+	if a.options.forceReasoning || actionParams == nil {
+		xlog.Info("Generating parameters",
+			"agent", a.Character.Name,
+			"action", chosenAction.Definition().Name,
+			"reasoning", reasoning,
+		)
+
+		params, err := a.generateParameters(ctx, pickTemplate, chosenAction, a.currentConversation, reasoning)
+		if err != nil {
+			job.Result.Finish(fmt.Errorf("error generating action's parameters: %w", err))
+			return
+		}
+		actionParams = params.actionParams
+	}
+
+	xlog.Info(
+		"Generated parameters",
+		"agent", a.Character.Name,
+		"action", chosenAction.Definition().Name,
+		"reasoning", reasoning,
+		"params", actionParams.String(),
+	)
+
+	if actionParams == nil {
+		job.Result.Finish(fmt.Errorf("no parameters"))
+		return
+	}
+
+	if !job.Callback(ActionCurrentState{chosenAction, actionParams, reasoning}) {
+		job.Result.SetResult(ActionState{ActionCurrentState{chosenAction, actionParams, reasoning}, "stopped by callback"})
+		job.Result.Finish(nil)
+		return
+	}
+
+	if selfEvaluation && a.options.initiateConversations &&
+		chosenAction.Definition().Name.Is(action.ConversationActionName) {
+
+		message := action.ConversationActionResponse{}
+		if err := actionParams.Unmarshal(&message); err != nil {
+			job.Result.Finish(fmt.Errorf("error unmarshalling conversation response: %w", err))
+			return
+		}
+
+		a.currentConversation = []openai.ChatCompletionMessage{
+			{
+				Role:    "assistant",
+				Content: message.Message,
+			},
+		}
+		go func() {
+			a.newConversations <- openai.ChatCompletionMessage{
+				Role:    "assistant",
+				Content: message.Message,
+			}
+		}()
+		job.Result.SetResponse("decided to initiate a new conversation")
+		job.Result.Finish(nil)
+		return
+	}
+
+	// If we don't have to reply , run the action!
+	if !chosenAction.Definition().Name.Is(action.ReplyActionName) {
+		result, err := a.runAction(chosenAction, actionParams)
+		if err != nil {
+			//job.Result.Finish(fmt.Errorf("error running action: %w", err))
+			//return
+			// make the LLM aware of the error of running the action instead of stopping the job here
+			result = fmt.Sprintf("Error running tool: %v", err)
+		}
+
+		stateResult := ActionState{ActionCurrentState{chosenAction, actionParams, reasoning}, result}
+		job.Result.SetResult(stateResult)
+		job.CallbackWithResult(stateResult)
+
+		// calling the function
+		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+			Role: "assistant",
+			FunctionCall: &openai.FunctionCall{
+				Name:      chosenAction.Definition().Name.String(),
+				Arguments: actionParams.String(),
+			},
+		})
+
+		// result of calling the function
+		a.currentConversation = append(a.currentConversation, openai.ChatCompletionMessage{
+			Role:       openai.ChatMessageRoleTool,
+			Content:    result,
+			Name:       chosenAction.Definition().Name.String(),
+			ToolCallID: chosenAction.Definition().Name.String(),
+		})
+
+		//a.currentConversation = append(a.currentConversation, messages...)
+		//a.currentConversation = messages
+
+		// given the result, we can now ask OpenAI to complete the conversation or
+		// to continue using another tool given the result
+		followingAction, followingParams, reasoning, err := a.pickAction(ctx, reEvaluationTemplate, a.currentConversation)
+		if err != nil {
+			job.Result.Finish(fmt.Errorf("error picking action: %w", err))
+			return
+		}
+
+		if followingAction != nil &&
+			!followingAction.Definition().Name.Is(action.ReplyActionName) &&
+			!chosenAction.Definition().Name.Is(action.ReplyActionName) {
+			// We need to do another action (?)
+			// The agent decided to do another action
+			// call ourselves again
+			a.currentReasoning = reasoning
+			a.nextAction = followingAction
+			a.nextActionParams = &followingParams
+			job.Text = ""
+			a.consumeJob(job, role)
+			return
+		} else if followingAction == nil {
+			if !a.options.forceReasoning {
+				msg := openai.ChatCompletionMessage{
+					Role:    "assistant",
+					Content: reasoning,
+				}
+
+				a.currentConversation = append(a.currentConversation, msg)
+				job.Result.SetResponse(msg.Content)
+				job.Result.Finish(nil)
+				return
+			}
+		}
+	}
+
+	// At this point can only be a reply action
+
+	// decode the response
+	replyResponse := action.ReplyResponse{}
+
+	if err := actionParams.Unmarshal(&replyResponse); err != nil {
+		job.Result.Finish(fmt.Errorf("error unmarshalling reply response: %w", err))
+		return
+	}
+
+	// If we have already a reply from the action, just return it.
+	// Otherwise generate a full conversation to get a proper message response
+	// if chosenAction.Definition().Name.Is(action.ReplyActionName) {
+	// 	replyResponse := action.ReplyResponse{}
+	// 	if err := params.actionParams.Unmarshal(&replyResponse); err != nil {
+	// 		job.Result.Finish(fmt.Errorf("error unmarshalling reply response: %w", err))
+	// 		return
+	// 	}
+	// 	if replyResponse.Message != "" {
+	// 		job.Result.SetResponse(replyResponse.Message)
+	// 		job.Result.Finish(nil)
+	// 		return
+	// 	}
+	// }
+
+	// If we have a hud, display it
+	if a.options.enableHUD {
+		var promptHUD *PromptHUD
+		if a.options.enableHUD {
+			h := a.prepareHUD()
+			promptHUD = &h
+		}
+
+		prompt, err := renderTemplate(hudTemplate, promptHUD, a.systemInternalActions(), reasoning)
+		if err != nil {
+			job.Result.Finish(fmt.Errorf("error renderTemplate: %w", err))
+			return
+		}
+		if !a.currentConversation.Exist(prompt) {
+			a.currentConversation = append([]openai.ChatCompletionMessage{
+				{
+					Role:    "system",
+					Content: prompt,
+				},
+			}, a.currentConversation...)
+		}
+	}
+
+	// Generate a human-readable response
+	// resp, err := a.client.CreateChatCompletion(ctx,
+	// 	openai.ChatCompletionRequest{
+	// 		Model: a.options.LLMAPI.Model,
+	// 		Messages: append(a.currentConversation,
+	// 			openai.ChatCompletionMessage{
+	// 				Role:    "system",
+	// 				Content: "Assistant thought: " + replyResponse.Message,
+	// 			},
+	// 		),
+	// 	},
+	// )
+
+	if !a.options.forceReasoning {
+		msg := openai.ChatCompletionMessage{
+			Role:    "assistant",
+			Content: replyResponse.Message,
+		}
+
+		a.currentConversation = append(a.currentConversation, msg)
+		job.Result.SetResponse(msg.Content)
+		job.Result.Finish(nil)
+		return
+	}
+
+	msg, err := a.askLLM(ctx, append(a.currentConversation, openai.ChatCompletionMessage{
+		Role:    "system",
+		Content: "The assistant needs to reply without using any tool.",
+	}))
+	if err != nil {
+		job.Result.Finish(err)
+		return
+	}
+
+	// If we didn't got any message, we can use the response from the action
+	if chosenAction.Definition().Name.Is(action.ReplyActionName) && msg.Content == "" ||
+		strings.Contains(msg.Content, "<tool_call>") {
+		xlog.Info("No output returned from conversation, using the action response as a reply " + replyResponse.Message)
+
+		msg = openai.ChatCompletionMessage{
+			Role:    "assistant",
+			Content: replyResponse.Message,
+		}
+	}
+
+	a.currentConversation = append(a.currentConversation, msg)
+	job.Result.SetResponse(msg.Content)
+	job.Result.Finish(nil)
+}
+
+// This is running in the background.
+func (a *Agent) periodicallyRun(timer *time.Timer) {
+	// Remember always to reset the timer - if we don't the agent will stop..
+	defer timer.Reset(a.options.periodicRuns)
+
+	a.StopAction()
+	xlog.Debug("Agent is running periodically", "agent", a.Character.Name)
+
+	// TODO: Would be nice if we have a special action to
+	// contact the user. This would actually make sure that
+	// if the agent wants to initiate a conversation, it can do so.
+	// This would be a special action that would be picked up by the agent
+	// and would be used to contact the user.
+
+	xlog.Info("START -- Periodically run is starting")
+
+	// if len(a.CurrentConversation()) != 0 {
+	// 	// Here the LLM could decide to store some part of the conversation too in the memory
+	// 	evaluateMemory := NewJob(
+	// 		WithText(
+	// 			`Evaluate the current conversation and decide if we need to store some relevant informations from it`,
+	// 		),
+	// 		WithReasoningCallback(a.options.reasoningCallback),
+	// 		WithResultCallback(a.options.resultCallback),
+	// 	)
+	// 	a.consumeJob(evaluateMemory, SystemRole)
+
+	// 	a.ResetConversation()
+	// }
+
+	if !a.options.standaloneJob {
+		a.ResetConversation()
+
+		return
+	}
+
+	// Here we go in a loop of
+	// - asking the agent to do something
+	// - evaluating the result
+	// - asking the agent to do something else based on the result
+
+	//	whatNext := NewJob(WithText("Decide what to do based on the state"))
+	whatNext := NewJob(
+		WithText(innerMonologueTemplate),
+		WithReasoningCallback(a.options.reasoningCallback),
+		WithResultCallback(a.options.resultCallback),
+	)
+	a.consumeJob(whatNext, SystemRole)
+	a.ResetConversation()
+
+	xlog.Info("STOP -- Periodically run is done")
+
+	// Save results from state
+
+	// a.ResetConversation()
+
+	// doWork := NewJob(WithText("Select the tool to use based on your goal and the current state."))
+	// a.consumeJob(doWork, SystemRole)
+
+	// results := []string{}
+	// for _, v := range doWork.Result.State {
+	// 	results = append(results, v.Result)
+	// }
+
+	// a.ResetConversation()
+
+	// // Here the LLM could decide to do something based on the result of our automatic action
+	// evaluateAction := NewJob(
+	// 	WithText(
+	// 		`Evaluate the current situation and decide if we need to execute other tools (for instance to store results into permanent, or short memory).
+	// 		We have done the following actions:
+	// 		` + strings.Join(results, "\n"),
+	// 	))
+	// a.consumeJob(evaluateAction, SystemRole)
+
+	// a.ResetConversation()
+}
+
+func (a *Agent) prepareIdentity() error {
+
+	if a.options.characterfile != "" {
+		if _, err := os.Stat(a.options.characterfile); err == nil {
+			// if there is a file, load the character back
+			if err = a.LoadCharacter(a.options.characterfile); err != nil {
+				return fmt.Errorf("failed to load character: %v", err)
+			}
+		} else {
+			if a.options.randomIdentity {
+				if err = a.generateIdentity(a.options.randomIdentityGuidance); err != nil {
+					return fmt.Errorf("failed to generate identity: %v", err)
+				}
+			}
+
+			// otherwise save it for next time
+			if err = a.SaveCharacter(a.options.characterfile); err != nil {
+				return fmt.Errorf("failed to save character: %v", err)
+			}
+		}
+	} else {
+		if err := a.generateIdentity(a.options.randomIdentityGuidance); err != nil {
+			return fmt.Errorf("failed to generate identity: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (a *Agent) Run() error {
+	// The agent run does two things:
+	// picks up requests from a queue
+	// and generates a response/perform actions
+
+	if err := a.prepareIdentity(); err != nil {
+		return fmt.Errorf("failed to prepare identity: %v", err)
+	}
+
+	// It is also preemptive.
+	// That is, it can interrupt the current action
+	// if another one comes in.
+
+	// If there is no action, periodically evaluate if it has to do something on its own.
+
+	// Expose a REST API to interact with the agent to ask it things
+
+	//todoTimer := time.NewTicker(a.options.periodicRuns)
+	timer := time.NewTimer(a.options.periodicRuns)
+	for {
+		xlog.Debug("Agent is waiting for a job", "agent", a.Character.Name)
+		select {
+		case job := <-a.jobQueue:
+			a.loop(timer, job)
+		case <-a.context.Done():
+			// Agent has been canceled, return error
+			xlog.Warn("Agent has been canceled", "agent", a.Character.Name)
+			return ErrContextCanceled
+		case <-timer.C:
+			a.periodicallyRun(timer)
+		}
+	}
+}
+
+func (a *Agent) loop(timer *time.Timer, job *Job) {
+	// Remember always to reset the timer - if we don't the agent will stop..
+	defer timer.Reset(a.options.periodicRuns)
+	// Consume the job and generate a response
+	// TODO: Give a short-term memory to the agent
+	// stop and drain the timer
+	if !timer.Stop() {
+		<-timer.C
+	}
+	xlog.Debug("Agent is consuming a job", "agent", a.Character.Name, "job", job)
+	a.StopAction()
+	a.consumeJob(job, UserRole)
+}

core/agent/agent_suite_test.go

@@ -0,0 +1,26 @@
+package agent_test
+
+import (
+	"os"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestAgent(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Agent test suite")
+}
+
+var testModel = os.Getenv("TEST_MODEL")
+var apiModel = os.Getenv("API_MODEL")
+
+func init() {
+	if testModel == "" {
+		testModel = "hermes-2-pro-mistral"
+	}
+	if apiModel == "" {
+		apiModel = "http://192.168.68.113:8080"
+	}
+}

core/agent/agent_test.go

@@ -0,0 +1,251 @@
+package agent_test
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/mudler/local-agent-framework/pkg/xlog"
+
+	"github.com/mudler/local-agent-framework/core/action"
+	. "github.com/mudler/local-agent-framework/core/agent"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/sashabaranov/go-openai/jsonschema"
+)
+
+const testActionResult = "In Boston it's 30C today, it's sunny, and humidity is at 98%"
+const testActionResult2 = "In milan it's very hot today, it is 45C and the humidity is at 200%"
+const testActionResult3 = "In paris it's very cold today, it is 2C and the humidity is at 10%"
+
+var _ Action = &TestAction{}
+
+var debugOptions = []JobOption{
+	WithReasoningCallback(func(state ActionCurrentState) bool {
+		xlog.Info("Reasoning", state)
+		return true
+	}),
+	WithResultCallback(func(state ActionState) {
+		xlog.Info("Reasoning", state.Reasoning)
+		xlog.Info("Action", state.Action)
+		xlog.Info("Result", state.Result)
+	}),
+}
+
+type TestAction struct {
+	response  []string
+	responseN int
+}
+
+func (a *TestAction) Run(context.Context, action.ActionParams) (string, error) {
+	res := a.response[a.responseN]
+	a.responseN++
+
+	if len(a.response) == a.responseN {
+		a.responseN = 0
+	}
+
+	return res, nil
+}
+
+func (a *TestAction) Definition() action.ActionDefinition {
+	return action.ActionDefinition{
+		Name:        "get_weather",
+		Description: "get current weather",
+		Properties: map[string]jsonschema.Definition{
+			"location": {
+				Type:        jsonschema.String,
+				Description: "The city and state, e.g. San Francisco, CA",
+			},
+			"unit": {
+				Type: jsonschema.String,
+				Enum: []string{"celsius", "fahrenheit"},
+			},
+		},
+
+		Required: []string{"location"},
+	}
+}
+
+type FakeStoreResultAction struct {
+	TestAction
+}
+
+func (a *FakeStoreResultAction) Definition() action.ActionDefinition {
+	return action.ActionDefinition{
+		Name:        "store_results",
+		Description: "store results permanently. Use this tool after you have a result you want to keep.",
+		Properties: map[string]jsonschema.Definition{
+			"term": {
+				Type:        jsonschema.String,
+				Description: "What to store permanently",
+			},
+		},
+
+		Required: []string{"term"},
+	}
+}
+
+type FakeInternetAction struct {
+	TestAction
+}
+
+func (a *FakeInternetAction) Definition() action.ActionDefinition {
+	return action.ActionDefinition{
+		Name:        "search_internet",
+		Description: "search on internet",
+		Properties: map[string]jsonschema.Definition{
+			"term": {
+				Type:        jsonschema.String,
+				Description: "What to search for",
+			},
+		},
+
+		Required: []string{"term"},
+	}
+}
+
+var _ = Describe("Agent test", func() {
+	Context("jobs", func() {
+		It("pick the correct action", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+				//	WithRandomIdentity(),
+				WithActions(&TestAction{response: []string{testActionResult, testActionResult2, testActionResult3}}),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			go agent.Run()
+			defer agent.Stop()
+			res := agent.Ask(
+				append(debugOptions,
+					WithText("can you get the weather in boston, and afterward of Milano, Italy?"),
+				)...,
+			)
+			Expect(res.Error).ToNot(HaveOccurred())
+			reasons := []string{}
+			for _, r := range res.State {
+
+				reasons = append(reasons, r.Result)
+			}
+			Expect(reasons).To(ContainElement(testActionResult), fmt.Sprint(res))
+			Expect(reasons).To(ContainElement(testActionResult2), fmt.Sprint(res))
+			reasons = []string{}
+
+			res = agent.Ask(
+				append(debugOptions,
+					WithText("Now I want to know the weather in Paris"),
+				)...)
+			for _, r := range res.State {
+
+				reasons = append(reasons, r.Result)
+			}
+			Expect(reasons).ToNot(ContainElement(testActionResult), fmt.Sprint(res))
+			Expect(reasons).ToNot(ContainElement(testActionResult2), fmt.Sprint(res))
+			Expect(reasons).To(ContainElement(testActionResult3), fmt.Sprint(res))
+			// conversation := agent.CurrentConversation()
+			// for _, r := range res.State {
+			// 	reasons = append(reasons, r.Result)
+			// }
+			//			Expect(len(conversation)).To(Equal(10), fmt.Sprint(conversation))
+		})
+		It("pick the correct action", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+
+				//	WithRandomIdentity(),
+				WithActions(&TestAction{response: []string{testActionResult}}),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			go agent.Run()
+			defer agent.Stop()
+			res := agent.Ask(
+				append(debugOptions,
+					WithText("can you get the weather in boston?"))...,
+			)
+			reasons := []string{}
+			for _, r := range res.State {
+				reasons = append(reasons, r.Result)
+			}
+			Expect(reasons).To(ContainElement(testActionResult), fmt.Sprint(res))
+		})
+
+		It("updates the state with internal actions", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+				EnableHUD,
+				//	EnableStandaloneJob,
+				//	WithRandomIdentity(),
+				WithPermanentGoal("I want to learn to play music"),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			go agent.Run()
+			defer agent.Stop()
+
+			result := agent.Ask(
+				WithText("Update your goals such as you want to learn to play the guitar"),
+			)
+			fmt.Printf("%+v\n", result)
+			Expect(result.Error).ToNot(HaveOccurred())
+			Expect(agent.State().Goal).To(ContainSubstring("guitar"), fmt.Sprint(agent.State()))
+		})
+
+		It("it automatically performs things in the background", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+				EnableHUD,
+				EnableStandaloneJob,
+				WithAgentReasoningCallback(func(state ActionCurrentState) bool {
+					xlog.Info("Reasoning", state)
+					return true
+				}),
+				WithAgentResultCallback(func(state ActionState) {
+					xlog.Info("Reasoning", state.Reasoning)
+					xlog.Info("Action", state.Action)
+					xlog.Info("Result", state.Result)
+				}),
+				WithActions(
+					&FakeInternetAction{
+						TestAction{
+							response: []string{
+								"Major cities in italy: Roma, Venice, Milan",
+								"In rome it's 30C today, it's sunny, and humidity is at 98%",
+								"In venice it's very hot today, it is 45C and the humidity is at 200%",
+								"In milan it's very cold today, it is 2C and the humidity is at 10%",
+							},
+						},
+					},
+					&FakeStoreResultAction{
+						TestAction{
+							response: []string{
+								"Result permanently stored",
+							},
+						},
+					},
+				),
+				//WithRandomIdentity(),
+				WithPermanentGoal("get the weather of all the cities in italy and store the results"),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			go agent.Run()
+			defer agent.Stop()
+			Eventually(func() string {
+
+				return agent.State().Goal
+			}, "10m", "10s").Should(ContainSubstring("weather"), fmt.Sprint(agent.State()))
+
+			Eventually(func() string {
+				return agent.State().String()
+			}, "10m", "10s").Should(ContainSubstring("store"), fmt.Sprint(agent.State()))
+
+			// result := agent.Ask(
+			// 	WithText("Update your goals such as you want to learn to play the guitar"),
+			// )
+			// fmt.Printf("%+v\n", result)
+			// Expect(result.Error).ToNot(HaveOccurred())
+			// Expect(agent.State().Goal).To(ContainSubstring("guitar"), fmt.Sprint(agent.State()))
+		})
+	})
+})

core/agent/jobs.go

@@ -0,0 +1,131 @@
+package agent
+
+import (
+	"sync"
+
+	"github.com/sashabaranov/go-openai"
+)
+
+// Job is a request to the agent to do something
+type Job struct {
+	// The job is a request to the agent to do something
+	// It can be a question, a command, or a request to do something
+	// The agent will try to do it, and return a response
+	Text                string
+	Image               string // base64 encoded image
+	Result              *JobResult
+	reasoningCallback   func(ActionCurrentState) bool
+	resultCallback      func(ActionState)
+	conversationHistory []openai.ChatCompletionMessage
+}
+
+// JobResult is the result of a job
+type JobResult struct {
+	sync.Mutex
+	// The result of a job
+	State    []ActionState
+	Response string
+	Error    error
+	ready    chan bool
+}
+
+type JobOption func(*Job)
+
+func WithConversationHistory(history []openai.ChatCompletionMessage) JobOption {
+	return func(j *Job) {
+		j.conversationHistory = history
+	}
+}
+
+func WithReasoningCallback(f func(ActionCurrentState) bool) JobOption {
+	return func(r *Job) {
+		r.reasoningCallback = f
+	}
+}
+
+func WithResultCallback(f func(ActionState)) JobOption {
+	return func(r *Job) {
+		r.resultCallback = f
+	}
+}
+
+// NewJobResult creates a new job result
+func NewJobResult() *JobResult {
+	r := &JobResult{
+		ready: make(chan bool),
+	}
+	return r
+}
+
+func (j *Job) Callback(stateResult ActionCurrentState) bool {
+	if j.reasoningCallback == nil {
+		return true
+	}
+	return j.reasoningCallback(stateResult)
+}
+
+func (j *Job) CallbackWithResult(stateResult ActionState) {
+	if j.resultCallback == nil {
+		return
+	}
+	j.resultCallback(stateResult)
+}
+
+func WithImage(image string) JobOption {
+	return func(j *Job) {
+		j.Image = image
+	}
+}
+
+func WithText(text string) JobOption {
+	return func(j *Job) {
+		j.Text = text
+	}
+}
+
+// NewJob creates a new job
+// It is a request to the agent to do something
+// It has a JobResult to get the result asynchronously
+// To wait for a Job result, use JobResult.WaitResult()
+func NewJob(opts ...JobOption) *Job {
+	j := &Job{
+		Result: NewJobResult(),
+	}
+	for _, o := range opts {
+		o(j)
+	}
+	return j
+}
+
+// SetResult sets the result of a job
+func (j *JobResult) SetResult(text ActionState) {
+	j.Lock()
+	defer j.Unlock()
+
+	j.State = append(j.State, text)
+}
+
+// SetResult sets the result of a job
+func (j *JobResult) Finish(e error) {
+	j.Lock()
+	defer j.Unlock()
+
+	j.Error = e
+	close(j.ready)
+}
+
+// SetResult sets the result of a job
+func (j *JobResult) SetResponse(response string) {
+	j.Lock()
+	defer j.Unlock()
+
+	j.Response = response
+}
+
+// WaitResult waits for the result of a job
+func (j *JobResult) WaitResult() *JobResult {
+	<-j.ready
+	j.Lock()
+	defer j.Unlock()
+	return j
+}

core/agent/options.go

@@ -0,0 +1,270 @@
+package agent
+
+import (
+	"context"
+	"strings"
+	"time"
+)
+
+type Option func(*options) error
+type llmOptions struct {
+	APIURL string
+	APIKey string
+	Model  string
+}
+
+type options struct {
+	LLMAPI                                                                                       llmOptions
+	character                                                                                    Character
+	randomIdentityGuidance                                                                       string
+	randomIdentity                                                                               bool
+	userActions                                                                                  Actions
+	enableHUD, standaloneJob, showCharacter, enableKB, enableSummaryMemory, enableLongTermMemory bool
+
+	canStopItself         bool
+	initiateConversations bool
+	forceReasoning        bool
+	characterfile         string
+	statefile             string
+	context               context.Context
+	permanentGoal         string
+	timeout               string
+	periodicRuns          time.Duration
+	kbResults             int
+	ragdb                 RAGDB
+
+	prompts []PromptBlock
+
+	systemPrompt string
+
+	// callbacks
+	reasoningCallback func(ActionCurrentState) bool
+	resultCallback    func(ActionState)
+}
+
+type PromptBlock interface {
+	Render(a *Agent) string
+	Role() string
+}
+
+func defaultOptions() *options {
+	return &options{
+		periodicRuns: 15 * time.Minute,
+		LLMAPI: llmOptions{
+			APIURL: "http://localhost:8080",
+			Model:  "gpt-4",
+		},
+		character: Character{
+			Name:       "",
+			Age:        "",
+			Occupation: "",
+			Hobbies:    []string{},
+			MusicTaste: []string{},
+		},
+	}
+}
+
+func newOptions(opts ...Option) (*options, error) {
+	options := defaultOptions()
+	for _, o := range opts {
+		if err := o(options); err != nil {
+			return nil, err
+		}
+	}
+	return options, nil
+}
+
+var EnableHUD = func(o *options) error {
+	o.enableHUD = true
+	return nil
+}
+
+var EnableForceReasoning = func(o *options) error {
+	o.forceReasoning = true
+	return nil
+}
+
+var EnableKnowledgeBase = func(o *options) error {
+	o.enableKB = true
+	o.kbResults = 5
+	return nil
+}
+
+var CanStopItself = func(o *options) error {
+	o.canStopItself = true
+	return nil
+}
+
+func WithTimeout(timeout string) Option {
+	return func(o *options) error {
+		o.timeout = timeout
+		return nil
+	}
+}
+
+func EnableKnowledgeBaseWithResults(results int) Option {
+	return func(o *options) error {
+		o.enableKB = true
+		o.kbResults = results
+		return nil
+	}
+}
+
+var EnableInitiateConversations = func(o *options) error {
+	o.initiateConversations = true
+	return nil
+}
+
+// EnableStandaloneJob is an option to enable the agent
+// to run jobs in the background automatically
+var EnableStandaloneJob = func(o *options) error {
+	o.standaloneJob = true
+	return nil
+}
+
+var EnablePersonality = func(o *options) error {
+	o.showCharacter = true
+	return nil
+}
+
+var EnableSummaryMemory = func(o *options) error {
+	o.enableSummaryMemory = true
+	return nil
+}
+
+var EnableLongTermMemory = func(o *options) error {
+	o.enableLongTermMemory = true
+	return nil
+}
+
+func WithRAGDB(db RAGDB) Option {
+	return func(o *options) error {
+		o.ragdb = db
+		return nil
+	}
+}
+
+func WithSystemPrompt(prompt string) Option {
+	return func(o *options) error {
+		o.systemPrompt = prompt
+		return nil
+	}
+}
+
+func WithLLMAPIURL(url string) Option {
+	return func(o *options) error {
+		o.LLMAPI.APIURL = url
+		return nil
+	}
+}
+
+func WithStateFile(path string) Option {
+	return func(o *options) error {
+		o.statefile = path
+		return nil
+	}
+}
+
+func WithCharacterFile(path string) Option {
+	return func(o *options) error {
+		o.characterfile = path
+		return nil
+	}
+}
+
+// WithPrompts adds additional block prompts to the agent
+// to be rendered internally in the conversation
+// when processing the conversation to the LLM
+func WithPrompts(prompts ...PromptBlock) Option {
+	return func(o *options) error {
+		o.prompts = prompts
+		return nil
+	}
+}
+
+func WithLLMAPIKey(key string) Option {
+	return func(o *options) error {
+		o.LLMAPI.APIKey = key
+		return nil
+	}
+}
+
+func WithPermanentGoal(goal string) Option {
+	return func(o *options) error {
+		o.permanentGoal = goal
+		return nil
+	}
+}
+
+func WithPeriodicRuns(duration string) Option {
+	return func(o *options) error {
+		t, err := time.ParseDuration(duration)
+		if err != nil {
+			o.periodicRuns, _ = time.ParseDuration("10m")
+		}
+		o.periodicRuns = t
+		return nil
+	}
+}
+
+func WithContext(ctx context.Context) Option {
+	return func(o *options) error {
+		o.context = ctx
+		return nil
+	}
+}
+
+func WithAgentReasoningCallback(cb func(ActionCurrentState) bool) Option {
+	return func(o *options) error {
+		o.reasoningCallback = cb
+		return nil
+	}
+}
+
+func WithAgentResultCallback(cb func(ActionState)) Option {
+	return func(o *options) error {
+		o.resultCallback = cb
+		return nil
+	}
+}
+
+func WithModel(model string) Option {
+	return func(o *options) error {
+		o.LLMAPI.Model = model
+		return nil
+	}
+}
+
+func WithCharacter(c Character) Option {
+	return func(o *options) error {
+		o.character = c
+		return nil
+	}
+}
+
+func FromFile(path string) Option {
+	return func(o *options) error {
+		c, err := Load(path)
+		if err != nil {
+			return err
+		}
+		o.character = *c
+		return nil
+	}
+}
+
+func WithRandomIdentity(guidance ...string) Option {
+	return func(o *options) error {
+		o.randomIdentityGuidance = strings.Join(guidance, "")
+		o.randomIdentity = true
+		o.showCharacter = true
+		return nil
+	}
+}
+
+func WithActions(actions ...Action) Option {
+	return func(o *options) error {
+		o.userActions = actions
+		return nil
+	}
+}

core/agent/state.go

@@ -0,0 +1,125 @@
+package agent
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/local-agent-framework/core/action"
+	"github.com/mudler/local-agent-framework/pkg/llm"
+)
+
+// PromptHUD contains
+// all information that should be displayed to the LLM
+// in the prompts
+type PromptHUD struct {
+	Character     Character          `json:"character"`
+	CurrentState  action.StateResult `json:"current_state"`
+	PermanentGoal string             `json:"permanent_goal"`
+	ShowCharacter bool               `json:"show_character"`
+}
+
+type Character struct {
+	Name       string   `json:"name"`
+	Age        any      `json:"age"`
+	Occupation string   `json:"job_occupation"`
+	Hobbies    []string `json:"hobbies"`
+	MusicTaste []string `json:"favorites_music_genres"`
+	Sex        string   `json:"sex"`
+}
+
+func Load(path string) (*Character, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var c Character
+	err = json.Unmarshal(data, &c)
+	if err != nil {
+		return nil, err
+	}
+	return &c, nil
+}
+
+func (a *Agent) State() action.StateResult {
+	return *a.currentState
+}
+
+func (a *Agent) LoadState(path string) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, a.currentState)
+}
+
+func (a *Agent) LoadCharacter(path string) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, &a.Character)
+}
+
+func (a *Agent) SaveState(path string) error {
+	os.MkdirAll(filepath.Dir(path), 0755)
+	data, err := json.Marshal(a.currentState)
+	if err != nil {
+		return err
+	}
+	os.WriteFile(path, data, 0644)
+	return nil
+}
+
+func (a *Agent) SaveCharacter(path string) error {
+	os.MkdirAll(filepath.Dir(path), 0755)
+	data, err := json.Marshal(a.Character)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, data, 0644)
+}
+
+func (a *Agent) generateIdentity(guidance string) error {
+	if guidance == "" {
+		guidance = "Generate a random character for roleplaying."
+	}
+	err := llm.GenerateJSONFromStruct(a.context.Context, a.client, guidance, a.options.LLMAPI.Model, &a.options.character)
+	a.Character = a.options.character
+	if err != nil {
+		return fmt.Errorf("failed to generate JSON from structure: %v", err)
+	}
+
+	if !a.validCharacter() {
+		return fmt.Errorf("generated character is not valid ( guidance: %s ): %v", guidance, a.Character.String())
+	}
+	return nil
+}
+
+func (a *Agent) validCharacter() bool {
+	return a.Character.Name != "" &&
+		a.Character.Age != "" &&
+		a.Character.Occupation != "" &&
+		len(a.Character.Hobbies) != 0 &&
+		len(a.Character.MusicTaste) != 0
+}
+
+const fmtT = `=====================
+Name: %s
+Age: %s
+Occupation: %s
+Hobbies: %v
+Music taste: %v
+=====================`
+
+func (c *Character) String() string {
+	return fmt.Sprintf(
+		fmtT,
+		c.Name,
+		c.Age,
+		c.Occupation,
+		c.Hobbies,
+		c.MusicTaste,
+	)
+}

core/agent/state_test.go

@@ -0,0 +1,43 @@
+package agent_test
+
+import (
+	. "github.com/mudler/local-agent-framework/core/agent"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Agent test", func() {
+
+	Context("identity", func() {
+		It("generates all the fields with random data", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+				WithRandomIdentity(),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(agent.Character.Name).ToNot(BeEmpty())
+			Expect(agent.Character.Age).ToNot(BeZero())
+			Expect(agent.Character.Occupation).ToNot(BeEmpty())
+			Expect(agent.Character.Hobbies).ToNot(BeEmpty())
+			Expect(agent.Character.MusicTaste).ToNot(BeEmpty())
+		})
+		It("detect an invalid character", func() {
+			_, err := New(WithRandomIdentity())
+			Expect(err).To(HaveOccurred())
+		})
+		It("generates all the fields", func() {
+			agent, err := New(
+				WithLLMAPIURL(apiModel),
+				WithModel(testModel),
+				WithRandomIdentity("An old man with a long beard, a wizard, who lives in a tower."),
+			)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(agent.Character.Name).ToNot(BeEmpty())
+			Expect(agent.Character.Age).ToNot(BeZero())
+			Expect(agent.Character.Occupation).ToNot(BeEmpty())
+			Expect(agent.Character.Hobbies).ToNot(BeEmpty())
+			Expect(agent.Character.MusicTaste).ToNot(BeEmpty())
+		})
+	})
+})

core/agent/templates.go

@@ -0,0 +1,103 @@
+package agent
+
+import (
+	"bytes"
+	"html/template"
+	"time"
+
+	"github.com/mudler/local-agent-framework/core/action"
+	"github.com/sashabaranov/go-openai"
+)
+
+func renderTemplate(templ string, hud *PromptHUD, actions Actions, reasoning string) (string, error) {
+	// prepare the prompt
+	prompt := bytes.NewBuffer([]byte{})
+
+	promptTemplate, err := template.New("pickAction").Parse(templ)
+	if err != nil {
+		return "", err
+	}
+
+	// Get all the actions definitions
+	definitions := []action.ActionDefinition{}
+	for _, m := range actions {
+		definitions = append(definitions, m.Definition())
+	}
+
+	err = promptTemplate.Execute(prompt, struct {
+		HUD       *PromptHUD
+		Actions   []action.ActionDefinition
+		Reasoning string
+		Messages  []openai.ChatCompletionMessage
+		Time      string
+	}{
+		Actions:   definitions,
+		HUD:       hud,
+		Reasoning: reasoning,
+		Time:      time.Now().Format(time.RFC3339),
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return prompt.String(), nil
+}
+
+const innerMonologueTemplate = `"This is not a typical conversation between an assistant and an user.
+You are thinking out loud by yourself now, and you are evaluating the current situation.
+Considering the goal and the persistent goal (if you have one) do an action or decide to plan something for later on. If possible for you, you might also decide to engage a conversation with the user by notifying him."`
+
+const hudTemplate = `{{with .HUD }}{{if .ShowCharacter}}The assistant acts like an human, has a character and the replies and actions might be influenced by it.
+{{if .Character.Name}}This is the assistant name: {{.Character.Name}}
+{{end}}{{if .Character.Age}}This is the assistant age: {{.Character.Age}}
+{{end}}{{if .Character.Occupation}}This is the assistant job: {{.Character.Occupation}}
+{{end}}{{if .Character.Hobbies}}This is the assistant's hobbies: {{.Character.Hobbies}}
+{{end}}{{if .Character.MusicTaste}}This is the assistant's music taste: {{.Character.MusicTaste}}
+{{end}}
+{{end}}
+
+This is your current state:
+NowDoing: {{if .CurrentState.NowDoing}}{{.CurrentState.NowDoing}}{{else}}Nothing{{end}}
+DoingNext: {{if .CurrentState.DoingNext}}{{.CurrentState.DoingNext}}{{else}}Nothing{{end}}
+Your permanent goal is: {{if .PermanentGoal}}{{.PermanentGoal}}{{else}}Nothing{{end}}
+Your current goal is: {{if .CurrentState.Goal}}{{.CurrentState.Goal}}{{else}}Nothing{{end}}
+You have done: {{range .CurrentState.DoneHistory}}{{.}} {{end}}
+You have a short memory with: {{range .CurrentState.Memories}}{{.}} {{end}}{{end}}
+Current time: is {{.Time}}`
+
+const pickSelfTemplate = `You can take any of the following tools: 
+
+{{range .Actions -}}
+- {{.Name}}: {{.Description }}
+{{ end }}
+
+To finish your session, use the "reply" tool with your answer.
+
+Act like as a fully autonomous smart AI agent having a character, the character and your state is defined in the message above.
+You are now self-evaluating what to do next based on the state in the previous message. 
+For example, if the permanent goal is to "make a sandwich", you might want to "get the bread" first, and update the state afterwards by calling two tools in sequence.
+You can update the short-term goal, the current action, the next action, the history of actions, and the memories.
+You can't ask things to the user as you are thinking by yourself. You are autonomous.
+
+{{if .Reasoning}}Reasoning: {{.Reasoning}}{{end}}
+` + hudTemplate
+
+const reSelfEvalTemplate = pickSelfTemplate + `
+
+We already have called other tools. Evaluate the current situation and decide if we need to execute other tools.`
+
+const pickActionTemplate = hudTemplate + `
+When you have to pick a tool in the reasoning explain how you would use the tools you'd pick from: 
+
+{{range .Actions -}}
+- {{.Name}}: {{.Description }}
+{{ end }}
+To answer back to the user, use the "reply" or the "answer" tool.
+Given the text below, decide which action to take and explain the detailed reasoning behind it. For answering without picking a choice, reply with 'none'.
+
+{{if .Reasoning}}Reasoning: {{.Reasoning}}{{end}}
+`
+
+const reEvalTemplate = pickActionTemplate + `
+
+We already have called other tools. Evaluate the current situation and decide if we need to execute other tools or answer back with a result.`