stuff

.dockerignore

@@ -0,0 +1,2 @@
+models/
+db/

.env

@@ -0,0 +1,8 @@
+DEBUG=true
+MODELS_PATH=/models
+GALLERIES=[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]
+PRELOAD_MODELS=[{"id":"model-gallery@stablediffusion"},{"id":"model-gallery@voice-en-us-kathleen-low"},{"url": "github:go-skynet/model-gallery/base.yaml", "name": "all-MiniLM-L6-v2", "overrides": {"embeddings": true, "backend":"huggingface-embeddings", "parameters": {"model": "all-MiniLM-L6-v2"}}}, {"id": "huggingface@thebloke/wizardlm-13b-v1.0-uncensored-ggml/wizardlm-13b-v1.0-uncensored.ggmlv3.q4_0.bin", "name": "functions", "overrides": { "context_size": 2048, "template": {"chat": "", "completion": "" }, "roles": { "assistant": "ASSISTANT:",  "system": "SYSTEM:",  "assistant_function_call": "FUNCTION_CALL:",  "function": "FUNCTION CALL RESULT:" }, "parameters": { "temperature": 0.1, "top_k": 40, "top_p": 0.95, "rope_freq_base": 10000.0, "rope_freq_scale": 1.0 }, "function": { "disable_no_action": true }, "mmap": true, "f16": true }},{"id": "huggingface@thebloke/wizardlm-13b-v1.0-uncensored-ggml/wizardlm-13b-v1.0-uncensored.ggmlv3.q4_k_m.bin", "name":"gpt-4", "overrides": { "context_size": 2048, "mmap": true, "f16": true, "parameters": { "temperature": 0.1, "top_k": 40, "top_p": 0.95, "rope_freq_base": 10000.0, "rope_freq_scale": 1.0 }}}]
+OPENAI_API_KEY=sk---
+OPENAI_API_BASE=http://api:8080
+IMAGE_PATH=/tmp
+THREADS=14

.gitignore

@@ -0,0 +1,2 @@
+db/
+models/

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.10-bullseye
+WORKDIR /app
+COPY ./requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install package dependencies
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+        alsa-utils \
+        libsndfile1-dev && \
+    apt-get clean
+
+COPY . /app
+
+ENTRYPOINT [ "python", "./main.py" ];

README.md

@@ -1,2 +1,36 @@
 # miniAGI
-100% Local, mini-AGI
+
+From the [LocalAI](https://localai.io) author, miniAGI. 100% Local AI assistant.
+
+Note: this is a fun project, not a serious one. It's a toy, not a tool. Be warned!
+
+## What is miniAGI?
+
+It is a dead simple experiment to show how to tie the various LocalAI functionalities to create a virtual assistant that can do tasks. It is simple on purpose, trying to be minimalistic and easy to understand and customize.
+
+## Quick start
+
+No frills, just run docker-compose and start chatting with your virtual assistant:
+
+```bash
+docker-compose run --build -i --rm miniagi
+```
+
+## Roadmap
+
+- [x] 100% Local, with Local AI. NO API KEYS NEEDED!
+- [x] Create a simple virtual assistant
+- [x] Make the virtual assistant do functions like store long-term memory and autonomously search between them when needed
+- [] Create the assistant avatar with Stable Diffusion
+- [] Give it a voice (push to talk or wakeword)
+- [] Get voice input
+- [] Make a REST API (OpenAI compliant?) so can be plugged by e.g. a third party service
+- [] Take a system prompt so can act with a "character" (e.g. "answer in rick and morty style")
+
+## Development
+
+Run docker-compose with main.py checked-out:
+
+```bash
+docker-compose run -v main.py:/app/main.py -i --rm miniagi
+```

docker-compose.yaml

@@ -0,0 +1,24 @@
+version: "3.9"
+services:
+  api:
+    image: quay.io/go-skynet/local-ai:master
+    ports:
+      - 8090:8080
+    env_file:
+      - .env
+    volumes:
+      - ./models:/models:cached
+    command: ["/usr/bin/local-ai" ]
+  miniagi:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    devices:
+    - /dev/snd
+    depends_on:
+      api:
+        condition: service_healthy
+    volumes:
+      - ./db:/app/db
+    env_file:
+      - .env

main.py

@@ -1,6 +1,15 @@
 import openai
-from langchain.embeddings import HuggingFaceEmbeddings
+#from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.embeddings import LocalAIEmbeddings
 import uuid
+import requests
+from ascii_magic import AsciiArt
+
+# these three lines swap the stdlib sqlite3 lib with the pysqlite3 package for chroma
+__import__('pysqlite3')
+import sys
+sys.modules['sqlite3'] = sys.modules.pop('pysqlite3')
+
 from langchain.vectorstores import Chroma
 from chromadb.config import Settings
 import json
@@ -8,16 +17,120 @@ import os
 
 FUNCTIONS_MODEL = os.environ.get("FUNCTIONS_MODEL", "functions")
 LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-4")
+VOICE_MODEL= os.environ.get("TTS_MODEL","en-us-kathleen-low.onnx")
+DEFAULT_SD_MODEL = os.environ.get("DEFAULT_SD_MODEL", "stablediffusion")
+DEFAULT_SD_PROMPT = os.environ.get("DEFAULT_SD_PROMPT", "floating hair, portrait, ((loli)), ((one girl)), cute face, hidden hands, asymmetrical bangs, beautiful detailed eyes, eye shadow, hair ornament, ribbons, bowties, buttons, pleated skirt, (((masterpiece))), ((best quality)), colorful|((part of the head)), ((((mutated hands and fingers)))), deformed, blurry, bad anatomy, disfigured, poorly drawn face, mutation, mutated, extra limb, ugly, poorly drawn hands, missing limb, blurry, floating limbs, disconnected limbs, malformed hands, blur, out of focus, long neck, long body, Octane renderer, lowres, bad anatomy, bad hands, text")
 
-embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+#embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+embeddings = LocalAIEmbeddings(model="all-MiniLM-L6-v2")
 
 chroma_client = Chroma(collection_name="memories", persist_directory="db", embedding_function=embeddings)
 
+
+# Function to create images with OpenAI
+def create_image(input_text=DEFAULT_SD_PROMPT, model=DEFAULT_SD_MODEL):
+    response = openai.Image.create(
+        prompt=input_text,
+        n=1,
+        size="128x128",
+        api_base=os.environ.get("OPENAI_API_BASE", "http://api:8080")+"/v1"
+    )
+    image_url = response['data'][0]['url']
+    # convert the image to ascii art
+    my_art = AsciiArt.from_url(image_url)
+    my_art.to_terminal()
+
+def tts(input_text, model=VOICE_MODEL):
+    # strip newlines from text
+    input_text = input_text.replace("\n", ".")
+    # Create a temp file to store the audio output
+    output_file_path = '/tmp/output.wav'
+    # get from OPENAI_API_BASE env var
+    url = os.environ.get("OPENAI_API_BASE", "http://api:8080") + '/tts'
+    headers = {'Content-Type': 'application/json'}
+    data = {
+        "input": input_text,
+        "model": model
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(data))
+
+    if response.status_code == 200:
+        with open(output_file_path, 'wb') as f:
+            f.write(response.content)
+        print('Audio file saved successfully:', output_file_path)
+    else:
+        print('Request failed with status code', response.status_code)
+
+    # Use aplay to play the audio
+    os.system('aplay ' + output_file_path)
+    # remove the audio file
+    os.remove(output_file_path)
+
+def calculate_plan(user_input):
+    print("--> Calculating plan ")
+    print(user_input)
+    messages = [
+            {"role": "user",
+             "content": f"""Transcript of AI assistant responding to user requests. Replies with a plan to achieve the user's goal.
+
+Request: {user_input}
+Function call: """
+             }
+        ]
+    functions = [
+        {
+        "name": "plan",
+        "description": """Decide to do an action.""",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "subtasks": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "subtask list"
+                },
+                "thought": {
+                    "type": "string",
+                    "description": "reasoning behind the planning"
+                },
+            },
+            "required": ["plan"]
+        }
+        },    
+    ]
+    response = openai.ChatCompletion.create(
+        #model="gpt-3.5-turbo",
+        model=FUNCTIONS_MODEL,
+        messages=messages,
+        functions=functions,
+        max_tokens=200,
+        stop=None,
+        temperature=0.5,
+        #function_call="auto"
+        function_call={"name": "plan"},
+    )
+    response_message = response["choices"][0]["message"]
+    if response_message.get("function_call"):
+        function_name = response.choices[0].message["function_call"].name
+        function_parameters = response.choices[0].message["function_call"].arguments
+        # read the json from the string
+        res = json.loads(function_parameters)
+        print(">>> function name: "+function_name)
+        print(">>> function parameters: "+function_parameters)
+        return res
+    return {"action": "none"}
+
 def needs_to_do_action(user_input):
     messages = [
-         #   {"role": "system", "content": "You are a helpful assistant."},
             {"role": "user",
              "content": f"""Transcript of AI assistant responding to user requests. Replies with the action to perform, including reasoning, and the confidence interval from 0 to 100.
+For saving a memory, the assistant replies with the action "save_memory" and the string to save. 
+For searching a memory, the assistant replies with the action "search_memory" and the query to search. 
+For generating a plan for complex tasks, the assistant replies with the action "generate_plan" and a list of plans to execute.
+For replying to the user, the assistant replies with the action "reply" and the reply to the user.
 
 Request: {user_input}
 Function call: """
@@ -36,7 +149,7 @@ Function call: """
             },
             "action": {
                 "type": "string",
-                "enum": ["save_memory", "search_memory", "reply"],
+                "enum": ["save_memory", "search_memory", "reply", "generate_plan"],
                 "description": "user intent"
             },
             "reasoning": {
@@ -67,8 +180,8 @@ Function call: """
         res = json.loads(function_parameters)
         print(">>> function name: "+function_name)
         print(">>> function parameters: "+function_parameters)
-        return res["action"]
-    return "reply"
+        return res
+    return {"action": "reply"}
 
 ### Agent capabilities
 def save(memory):
@@ -86,11 +199,15 @@ def search(query):
     print(res)
     return res
 
+function_result = {}
 def process_functions(user_input, action=""):
     messages = [
          #   {"role": "system", "content": "You are a helpful assistant."},
             {"role": "user",
              "content": f"""Transcript of AI assistant responding to user requests.
+For saving a memory, the assistant replies with the action "save_memory" and the string to save. 
+For searching a memory, the assistant replies with the action "search_memory" and the query to search to find information stored previously. 
+For replying to the user, the assistant replies with the action "reply" and the reply to the user directly when there is nothing to do.
 
 Request: {user_input}
 Function call: """
@@ -107,11 +224,14 @@ Function call: """
 
         available_functions = {
             "save_memory": save,
+            "generate_plan": calculate_plan,
             "search_memory": search,
         }
 
         function_to_call = available_functions[function_name]
         function_result = function_to_call(function_parameters)
+        print("==> function result: ")
+        print(function_result)
         messages = [
          #   {"role": "system", "content": "You are a helpful assistant."},
             {
@@ -130,22 +250,10 @@ Function call: """
             {
                 "role": "function",
                 "name": function_name,
-                "content": f'{{"result": {str(function_result)}}}'
-            }
-        )
-        response = openai.ChatCompletion.create(
-            model=LLM_MODEL,
-            messages=messages,
-            max_tokens=200,
-            stop=None,
-            temperature=0.5,
-        )
-        messages.append(
-            {
-                "role": "assistant",
-                "content": response.choices[0].message["content"],
+                "content": f'{{"result": "{str(function_result)}"}}'
             }
         )
+
     return messages
 
 def get_completion(messages, action=""):
@@ -161,7 +269,7 @@ def get_completion(messages, action=""):
         "parameters": {
             "type": "object",
             "properties": {
-            "string": {
+            "thought": {
                 "type": "string",
                 "description": "information to save"
             },
@@ -179,6 +287,10 @@ def get_completion(messages, action=""):
                 "type": "string",
                 "description": "The query to be used to search informations"
             },
+            "reasoning": {
+                "type": "string",
+                "description": "reasoning behind the intent"
+            },
             },
             "required": ["query"]
         }
@@ -197,17 +309,70 @@ def get_completion(messages, action=""):
 
     return response
 
+create_image()
 conversation_history = []
 while True:
     user_input = input("> ")
-    action = needs_to_do_action(user_input) 
-    if action != "reply":
-        print("==> needs to do action: "+action)
-        responses = process_functions(user_input, action=action)
+
+    try:
+        action = needs_to_do_action(user_input) 
+    except Exception as e:
+        print("==> error: ")
+        print(e)
+        action = {"action": "reply"}
+
+    if action["action"] != "reply":
+        print("==> needs to do action: ")
+        print(action)
+        responses = process_functions(user_input+"\nReasoning: "+action["reasoning"], action=action["action"])
+        response = openai.ChatCompletion.create(
+            model=LLM_MODEL,
+            messages=responses,
+            max_tokens=200,
+            stop=None,
+            temperature=0.5,
+        )
+        responses.append(
+            {
+                "role": "assistant",
+                "content": response.choices[0].message["content"],
+            }
+        )
         # add responses to conversation history by extending the list
         conversation_history.extend(responses)
         # print the latest response from the conversation history
-        print(conversation_history[-1])
+        print(conversation_history[-1]["content"])
+        tts(conversation_history[-1]["content"])
+
+    # If its a plan, we need to execute it
+    elif action["action"] == "generate_plan":
+        print("==> needs to do planify: ")
+        print(action)
+        responses = process_functions(user_input+"\nReasoning: "+action["reasoning"], action=action["action"])
+        # cycle subtasks and execute functions
+        for subtask in function_result["subtasks"]:
+            print("==> subtask: ")
+            print(subtask)
+            subtask_response = process_functions(subtask,)
+            responses.extend(subtask_response)
+        response = openai.ChatCompletion.create(
+            model=LLM_MODEL,
+            messages=responses,
+            max_tokens=200,
+            stop=None,
+            temperature=0.5,
+        )
+        responses.append(
+            {
+                "role": "assistant",
+                "content": response.choices[0].message["content"],
+            }
+        )
+        # add responses to conversation history by extending the list
+        conversation_history.extend(responses)
+        # print the latest response from the conversation history
+        print(conversation_history[-1]["content"])
+        tts(conversation_history[-1]["content"])
     else:
         print("==> no action needed")
         # construct the message and add it to the conversation history
@@ -226,5 +391,7 @@ while True:
         # add the response to the conversation history by extending the list
         conversation_history.append({ "role": "assistant", "content": response.choices[0].message["content"]})
         # print the latest response from the conversation history
-        print(conversation_history[-1])
+        print(conversation_history[-1]["content"])
+        tts(conversation_history[-1]["content"])
+
        

requirements.txt

@@ -0,0 +1,6 @@
+langchain
+openai
+chromadb
+pysqlite3-binary
+requests
+ascii-magic