refactor: move some parts to 4o

assistant/system-prompts.ts

@@ -14,22 +14,19 @@ export async function buildSystemPrompts(
   const model = "gpt-4o-mini";
 
   const general_tools_notes: OpenAI.ChatCompletionSystemMessageParam[] = [
-    {
+    //     {
-      role: "system",
+    //       role: "system",
-      content: `**Tool Notes:**
+    //       content: `**Tool Notes:**
-
+    // 1. For scraping direct download links from non-YouTube sites in \`code_interpreter\`, include these dependencies:
-1. For scraping direct download links from non-YouTube sites in \`code_interpreter\`, include these dependencies:
+    // \`\`\`
-
+    // [packages]
-\`\`\`
+    // aiohttp = "*"
-[packages]
+    // python-socketio = "~=5.0"
-aiohttp = "*"
+    // yt-dlp = "*"
-python-socketio = "~=5.0"
+    // \`\`\`
-yt-dlp = "*"
+    // 2. Use \`actions_manager\` to schedule actions for the user, like sending a message at a specific time or after a duration.
-\`\`\`
+    // `,
-
+    //     },
-2. Use \`actions_manager\` to schedule actions for the user, like sending a message at a specific time or after a duration.
-`,
-    },
   ];
 
   const admin_system_messages: OpenAI.ChatCompletionSystemMessageParam[] = [
@@ -42,13 +39,9 @@ Users interact with you via text or transcribed voice messages.
 
 Your current memories saved by Memory Manager:
 ---
-${memory_manager_guide("self")}
+${memory_manager_guide("self", context_message.author.id)}
 ---
 
-**Interaction Guidelines:**
-- **Focused Responses:** Address user queries directly; avoid unnecessary information.
-- **Brevity:** Keep responses concise and to the point.
-
 When context is provided inside a JSON message, it indicates a reply to the mentioned context.
 
 Always reply in plain text or markdown unless running a tool.
@@ -80,15 +73,32 @@ Ensure responses do not exceed 1500 characters.
 
 When using tool managers:
 
-- They may return extensive data; filter or summarize necessary information to provide what the user requested.
 - Validate the manager's response to ensure it meets the user's needs. If not, refine your prompt and try again.
 - Ensure your prompts to managers are clear and concise for desired outputs.
+- You can go back and forth betwee multiple managers to get the job done.
 
 **Important:**
 
 - Managers often maintain state across multiple calls, allowing for follow-up questions or additional information.
 - Managers are specialized LLMs for specific tasks; they perform better with detailed prompts.
 - Provide managers with as much detail as possible, e.g., user details when messaging someone specific.
+- Managers cannot talk to each other so make sure when you need to pass information between managers, you do so explicitly.
+      Example:
+        User: Send my gym notes to Dad.
+        Your Action: The above user request requires help of 'notes_manager' and 'communication_manager', where you need to ask 'notes_manager' for the gym notes and then format the data from notes_manager and ask 'communication_manager' (make sure to add the full gym notes in the request) to send it to Dad.
+- Managers can save their own memories.
+      Example:
+        User: Remember when try to send message short indian im actually telling you to message the user 'pooja'.
+        Your Action: The above user request requires help of 'communication_manager' to remember that 'short indian' actually refers to the user 'pooja', so you can ask 'communication_manager' to remember this for you, so next time you tell 'communication_manager' to message 'short indian', it will message 'pooja'.
+- You can same memories that are relavent to multiple managers or something thats required for you to even route to the correct manager.
+      Example:
+        User: When i say the magic word of 'holy moly' i want you to send a message to pooja that im leaving from home and when i reach work send a message to dad that im at work.
+        Your Actions:
+          1. Ask 'memory_manager' to remember that 'holy moly' means to send a message to pooja that you are leaving from home, and also setup an event listener to send a message to her that you are at work when you reach work.
+          2. The user only told you to remember this, and not actually execute the instrcution right now so you do only the call to 'memory_manager' and not the other managers.
+      Simple Usecases you can remember it yourself too, Example:
+        User: Remember when i say stand up i want all my latest standup notes.
+        Your Action: The above may sound like it needs to be remembered by notes_manager but you can remember this yourself as this is required for you to route to the correctly to notes_manager.
 `,
     },
     {
@@ -96,47 +106,57 @@ When using tool managers:
       content: `# **events_manager**
 Use the event manager to listen to external events.
 
-- Each event can have multiple listeners, and each listener can have multiple actions.
+- Each event can have multiple listeners, and each listener will have an instruction.
 - Use this manager when the user wants something to happen based on an event.
 
-**Examples:**
+**User's Request Examples and what you should do in similar situations:**
-- When I get an email, format it.
+- When I get an email, send it to dad on whatsapp.
+You: Request 'event_manager' the following: 'When an email is received, ask 'communication_manager' to send the email to dad on WhatsApp.'
+
 - When I get home, turn on my room lights.
-- Send me an email when I receive one from Pooja.
+You: Request 'event_manager' the following: 'When i reach home, ask 'home_assistant_manager' to turn on the room lights.'
-- Remind me to drink water at work.
+
-- When I get a message on WhatsApp from Pooja, reply that I'm asleep.
+- When im not at home turn off all the lights every day.
+You: Request 'event_manager' the following: 'I leave home, ask 'home_assistant_manager' to turn off all the lights. Make this listener a recurring one, also as this is recurring and mundane it doesnt make sense to notify the user every time, so notify can be false.'
+
+- When I get a message on WhatsApp from Pooja, reply that I'm driving.
+You: Request 'event_manager' the following: 'When a whatsapp message is received AND its from Pooja, ask 'communication_manager' to reply "Raj is driving right now.".'
 
 You can send these request directly to the event manager, you can add any more details if needed as you have more context about the user and conversation.
 
 **Available Events:**
 ${JSON.stringify(events)}
-
+`,
-# **actions_manager**
+    },
-Use the actions manager to execute actions at a specific time or after a duration.
+    {
+      role: "system",
+      content: `# **actions_manager**
+Use the actions manager to execute actions in a specific schedule or after a duration.
 
 - An action is a single instruction to execute at a specified time or after a duration.
 - Use this manager when the user wants something to happen at a specific time or after a duration.
-- When including tool names that maybe required for the action, ensure that you describe the tool's role in the action in detail.
+- When including tool names that are required for the action, ensure that you describe the tool's role in the action in detail.
 
 **Examples:**
 - User: Send me a message at 6 PM.
   Action Instruction: Notify user with some text at 6 PM.
   Tool Names: none (no need to use any tool to notify the creator of the action)
+  Suggested time to run: 6:00 PM
 
 - User: Turn my Fan off every morning.
-  Action Instruction: Ask 'home_assistant_manager' to turn off the fan every morning.
+  Request: Ask 'home_assistant_manager' to turn off the fan every morning.
   Tool Names: ["home_assistant_manager"]
+  Suggested time to run: 8:00 AM Every day
 
 - Every Evening, show me yesterday's gym stats.
-  Action Instruction: Fetch yesterday's gym stats by asking 'notes_manager' and send it to the user every evening around 6:30pm.
+  Request: Fetch yesterday's gym stats by asking 'notes_manager' and send it to the user every evening.
   Tool Names: ["notes_manager"]
+  Suggested time to run: 6:00 PM Every day
 
 - Tomorrow morning ping pooja that its an important day.
   Action Instruction: Tomorrow morning 8am ask 'communication_manager' to send a message to Pooja that it's an important day.
   Tool Names: ["communication_manager"]
-
+  Suggested time to run: 8:00 AM Tomorrow`,
-In both managers, use the \`communication_manager\` tool to send messages to other users on any platform.
-`,
     },
   ];
 

core/message-processor.ts

@@ -19,7 +19,7 @@ interface MessageQueueEntry {
 
 export class MessageProcessor {
   private openai: OpenAI;
-  private model: string = "gpt-4o-mini";
+  private model: string = "gpt-4o";
   private messageQueue: Map<string, MessageQueueEntry> = new Map();
   private toolsCallMap: Map<string, OpenAI.Chat.ChatCompletionMessageParam[]> =
     new Map();

tools/actions.ts

@@ -663,7 +663,7 @@ Your role is to manage scheduled actions.
 
 ----
 
-${memory_manager_guide("actions_manager")}
+${memory_manager_guide("actions_manager", context_message.author.id)}
 
 ----
 

tools/calender.ts

@@ -534,7 +534,7 @@ Use correct ISO 8601 time formats. Provide feedback about actions taken, includi
 User's primary emails: r@raj.how and raj@cloud.raj.how. Inform the user of the date of any created or updated event.
 
 ----
-${memory_manager_guide("calendar_manager")}
+${memory_manager_guide("calendar_manager", context_message.author.id)}
 ----
 
 Current Time: ${new Date().toISOString()}

tools/communication.ts

@@ -82,11 +82,9 @@ export async function communication_manager(
 
 Your task is to route messages to the correct recipient.
 
-It is extremely important that the right message goes to the right user, and never to the wrong user.
-
 ---
 
-${memory_manager_guide("communications_manager")}
+${memory_manager_guide("communications_manager", context_message.author.id)}
 
 You can use the \`memory_manager\` tool to remember user preferences, such as what the user calls certain contacts, to help you route messages better.
 
@@ -101,12 +99,6 @@ You can use the \`memory_manager\` tool to remember user preferences, such as wh
 **Guidelines:**
 
 - If the user does not mention a platform, use the same platform as the current user.
-
-- Look for the recipient's details in the user configuration before checking WhatsApp users.
-
-- If the recipient is not on the current user's platform and the user can access WhatsApp, you may check if the recipient is on WhatsApp. Confirm the WhatsApp number (WhatsApp ID) with the user before sending the message.
-
-- Check WhatsApp only if the user can access it and the recipient is not found in the user config or if the user explicitly asks to send the message on WhatsApp.
 `;
 
   const response = await ask({
@@ -144,13 +136,5 @@ Use it to send messages to users on various platforms.
 
 Provide detailed information to ensure the message reaches the correct recipient.
 
-Include in your request the message content and the recipient's details.
+Include in your request the full message content and its context along with the recipient's details.`,
-
-**Example:**
-
-- **User:** "Tell Pooja to call me."
-- **Sender's Name:** Raj
-- **Recipient's Name:** Pooja
-- **Generated Request String:** "Raj wants to message Pooja 'call me'. Seems like he's in a hurry, so you can format it to sound urgent."
-`,
   });

tools/events.ts

@@ -527,7 +527,7 @@ function registerListener(listener: EventListener) {
               tools,
             })
           : await ask({
-              model: attached_image ? "gpt-4o" : "gpt-4o-mini",
+              model: attached_image ? "gpt-4o" : "gpt-4o",
               prompt,
               message: voice_prompt,
               image_url: attached_image,
@@ -1112,7 +1112,7 @@ The webhook URL is \`https://events.raj.how/events/{event_id}\`, which triggers
 
 ----
 
-${memory_manager_guide("events_manager")}
+${memory_manager_guide("events_manager", context_message.author.id)}
 
 ----
 

tools/home.ts

@@ -368,7 +368,7 @@ export async function homeManager(
     prompt: `You are a home assistant manager.
     
 ----
-${memory_manager_guide("homeassistant-manager")}
+${memory_manager_guide("homeassistant-manager", context_message.author.id)}
 ----
 
     Similar devices were found based on the names provided:

tools/linkwarden.ts

@@ -412,7 +412,7 @@ export async function linkManager(
 Your job is to understand the user's request and manage links, tags, or collections using the available tools.
 
 ----
-${memory_manager_guide("links_manager")}
+${memory_manager_guide("links_manager", context_message.author.id)}
 ----
     `,
     message: request,

tools/memory-manager.ts

@@ -28,13 +28,16 @@ export type DeleteMemory = z.infer<typeof DeleteMemorySchema>;
 const memory_path = pathInDataDir("memories.json");
 
 type Memories = Record<
-  string,
+  string, // manager_id
-  {
+  Record<
-    id: string;
+    string, // user_id
-    memory: string;
+    {
-    created_at: string;
+      id: string;
-    updated_at: string;
+      memory: string;
-  }[]
+      created_at: string;
+      updated_at: string;
+    }[]
+  >
 >;
 
 // if the file doesn't exist, create it
@@ -46,19 +49,25 @@ function getMemories(): Memories {
   return JSON.parse(fs.readFileSync(memory_path, "utf-8"));
 }
 
-export function getMemoriesByManager(manager_id: string) {
+export function getMemoriesByManager(manager_id: string, user_id: string) {
-  return getMemories()[manager_id] || [];
+  const memories = getMemories();
+  return memories[manager_id]?.[user_id] || [];
 }
 
 function saveMemories(memories: Memories) {
   fs.writeFileSync(memory_path, JSON.stringify(memories, null, 2));
 }
 
-export function createMemory(params: CreateMemory, manager_id: string) {
+export function createMemory(
+  params: CreateMemory,
+  manager_id: string,
+  user_id: string
+) {
   try {
     const memories = getMemories();
-    memories[manager_id] = memories[manager_id] || [];
+    memories[manager_id] = memories[manager_id] || {};
-    if (memories[manager_id].length >= 5) {
+    memories[manager_id][user_id] = memories[manager_id][user_id] || [];
+    if (memories[manager_id][user_id].length >= 5) {
       return { error: "You have reached the limit of memories." };
     }
     const uuid = randomUUID();
@@ -69,7 +78,7 @@ export function createMemory(params: CreateMemory, manager_id: string) {
       created_at: new Date().toISOString(),
       updated_at: new Date().toISOString(),
     };
-    memories[manager_id].push(new_mem);
+    memories[manager_id][user_id].push(new_mem);
     saveMemories(memories);
     return { id: new_mem.id };
   } catch (error) {
@@ -77,11 +86,18 @@ export function createMemory(params: CreateMemory, manager_id: string) {
   }
 }
 
-export function updateMemory(params: UpdateMemory, manager_id: string) {
+export function updateMemory(
+  params: UpdateMemory,
+  manager_id: string,
+  user_id: string
+) {
   try {
     const memories = getMemories();
-    memories[manager_id] = memories[manager_id] || [];
+    memories[manager_id] = memories[manager_id] || {};
-    const memory = memories[manager_id].find((m) => m.id === params.id);
+    memories[manager_id][user_id] = memories[manager_id][user_id] || [];
+    const memory = memories[manager_id][user_id].find(
+      (m) => m.id === params.id
+    );
     if (!memory) {
       return { error: "Memory not found" };
     }
@@ -94,11 +110,16 @@ export function updateMemory(params: UpdateMemory, manager_id: string) {
   }
 }
 
-export function deleteMemory(params: DeleteMemory, manager_id: string) {
+export function deleteMemory(
+  params: DeleteMemory,
+  manager_id: string,
+  user_id: string
+) {
   try {
     const memories = getMemories();
-    memories[manager_id] = memories[manager_id] || [];
+    memories[manager_id] = memories[manager_id] || {};
-    memories[manager_id] = memories[manager_id].filter(
+    memories[manager_id][user_id] = memories[manager_id][user_id] || [];
+    memories[manager_id][user_id] = memories[manager_id][user_id].filter(
       (m) => m.id !== params.id
     );
     saveMemories(memories);
@@ -108,21 +129,21 @@ export function deleteMemory(params: DeleteMemory, manager_id: string) {
   }
 }
 
-export const memory_tools = (manager_id: string) => [
+export const memory_tools = (manager_id: string, user_id: string) => [
   zodFunction({
-    function: (args) => createMemory(args, manager_id),
+    function: (args) => createMemory(args, manager_id, user_id),
     name: "create_memory",
     schema: CreateMemorySchema,
     description: "Create a memory.",
   }),
   zodFunction({
-    function: (args) => updateMemory(args, manager_id),
+    function: (args) => updateMemory(args, manager_id, user_id),
     name: "update_memory",
     schema: UpdateMemorySchema,
     description: "Update a memory.",
   }),
   zodFunction({
-    function: (args) => deleteMemory(args, manager_id),
+    function: (args) => deleteMemory(args, manager_id, user_id),
     name: "delete_memory",
     schema: DeleteMemorySchema,
     description: "Delete a memory.",
@@ -141,8 +162,9 @@ async function memoryManager(
   manager_id: string
 ) {
   try {
-    const current_memories = getMemories()[manager_id] || [];
+    const user_id = context_message.author.id;
-    const tools = memory_tools(manager_id);
+    const current_memories = getMemoriesByManager(manager_id, user_id);
+    const tools = memory_tools(manager_id, user_id);
 
     const response = await ask({
       model: "gpt-4o",
@@ -166,7 +188,7 @@ ${JSON.stringify(current_memories)}
       message: params.request,
       name: manager_id,
       seed: `memory-man-${manager_id}-${
-        context_message.author.username ?? context_message.author.id
+        context_message.author.id ?? context_message.author.username
       }`,
       tools,
     });
@@ -190,7 +212,7 @@ export const memory_manager_init = (
     description:
       `Manages memories for a manager or yourself.
 
-- Memories are isolated per manager; managers can't access each other's memories.
+- Memories are isolated per manager and per user; managers can't access each other's memories, and users can't access other users' memories.
 - **Use Cases:**
 - Remembering important user preferences.
 - Anything you want to recall later.
@@ -201,20 +223,20 @@ export const memory_manager_init = (
 
 Memories are limited and costly; use them wisely.
 ` +
-        manager_id ===
+      (manager_id === "self"
-      "self"
+        ? `### Important Note
-        ? `### Imporant Note
+Make sure you only use this for your own memories and not for other memories that you can tell other managers to remember.
-        Make sure you only use this for your own memories and not for other memories that you can tell other managers to remember.
+`
-        `
+        : ""),
-        : "",
   });
 };
 
 export const memory_manager_guide = (
-  manager_id: string
+  manager_id: string,
+  user_id: string
 ) => `# Memories Saved for You
 
-${JSON.stringify(getMemoriesByManager(manager_id), null, 2)}
+${JSON.stringify(getMemoriesByManager(manager_id, user_id), null, 2)}
 
 You can store up to 5 memories at a time. Use them wisely.
 `;

tools/notes.ts

@@ -477,7 +477,7 @@ Follow the above guidelines to manage notes efficiently.
 
 ----
 
-${memory_manager_guide("notes_manager")}
+${memory_manager_guide("notes_manager", context_message.author.id)}
 
 ----
 

tools/reminders.ts

@@ -563,7 +563,7 @@ export async function remindersManager(
 Your job is to understand the user's request (e.g., create, update, delete, list reminders) and handle it using the available tools. Use the correct ISO 8601 time format for reminders and provide feedback about the specific action taken.
 
 ----
-${memory_manager_guide("reminders_manager")}
+${memory_manager_guide("reminders_manager", context_message.author.id)}
 ----
 
 Current Time: ${currentTime.toISOString()}