feat(evaluation): add deep evaluation mechanism (#145)

* feat(evaluation): add deep evaluation mechanism

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* consider whole conversation when evaluating

Signed-off-by: mudler <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: mudler <mudler@localai.io>

core/state/config.go

@@ -74,6 +74,8 @@ type AgentConfig struct {
 	SummaryLongTermMemory bool   `json:"summary_long_term_memory" form:"summary_long_term_memory"`
 	ParallelJobs          int    `json:"parallel_jobs" form:"parallel_jobs"`
 	StripThinkingTags     bool   `json:"strip_thinking_tags" form:"strip_thinking_tags"`
+	EnableEvaluation      bool   `json:"enable_evaluation" form:"enable_evaluation"`
+	MaxEvaluationLoops    int    `json:"max_evaluation_loops" form:"max_evaluation_loops"`
 }
 
 type AgentConfigMeta struct {
@@ -309,6 +311,24 @@ func NewAgentConfigMeta(
 				HelpText:     "Remove content between <thinking></thinking> and <think></think> tags from agent responses",
 				Tags:         config.Tags{Section: "ModelSettings"},
 			},
+			{
+				Name:         "enable_evaluation",
+				Label:        "Enable Evaluation",
+				Type:         "checkbox",
+				DefaultValue: false,
+				HelpText:     "Enable automatic evaluation of agent responses to ensure they meet user requirements",
+				Tags:         config.Tags{Section: "AdvancedSettings"},
+			},
+			{
+				Name:         "max_evaluation_loops",
+				Label:        "Max Evaluation Loops",
+				Type:         "number",
+				DefaultValue: 2,
+				Min:          1,
+				Step:         1,
+				HelpText:     "Maximum number of evaluation loops to perform when addressing gaps in responses",
+				Tags:         config.Tags{Section: "AdvancedSettings"},
+			},
 		},
 		MCPServers: []config.Field{
 			{

core/state/pool.go

@@ -247,7 +247,7 @@ func createAgentAvatar(APIURL, APIKey, model, imageModel, avatarDir string, agen
 		ImagePrompt string `json:"image_prompt"`
 	}
 
-	err := llm.GenerateTypedJSON(
+	err := llm.GenerateTypedJSONWithGuidance(
 		context.Background(),
 		llm.NewClient(APIKey, APIURL, "10m"),
 		"Generate a prompt that I can use to create a random avatar for the bot '"+agent.Name+"', the description of the bot is: "+agent.Description,
@@ -561,6 +561,13 @@ func (a *AgentPool) startAgentWithConfig(name string, config *AgentConfig, obs O
 		opts = append(opts, WithParallelJobs(config.ParallelJobs))
 	}
 
+	if config.EnableEvaluation {
+		opts = append(opts, EnableEvaluation())
+		if config.MaxEvaluationLoops > 0 {
+			opts = append(opts, WithMaxEvaluationLoops(config.MaxEvaluationLoops))
+		}
+	}
+
 	xlog.Info("Starting agent", "name", name, "config", config)
 
 	agent, err := New(opts...)