diff --git a/deno.lock b/deno.lock
index 12dc7ff..ad38b83 100644
--- a/deno.lock
+++ b/deno.lock
@@ -2758,9 +2758,74 @@
     }
   },
   "redirects": {
+    "https://deno.land/std/fs/mod.ts": "https://deno.land/std@0.224.0/fs/mod.ts",
     "https://esm.sh/@types/estree@1.0.5": "https://esm.sh/v135/@types/estree@1.0.5/index.d.ts"
   },
   "remote": {
+    "https://deno.land/std@0.224.0/assert/assert.ts": "09d30564c09de846855b7b071e62b5974b001bb72a4b797958fe0660e7849834",
+    "https://deno.land/std@0.224.0/assert/assertion_error.ts": "ba8752bd27ebc51f723702fac2f54d3e94447598f54264a6653d6413738a8917",
+    "https://deno.land/std@0.224.0/fs/_create_walk_entry.ts": "5d9d2aaec05bcf09a06748b1684224d33eba7a4de24cf4cf5599991ca6b5b412",
+    "https://deno.land/std@0.224.0/fs/_get_file_info_type.ts": "da7bec18a7661dba360a1db475b826b18977582ce6fc9b25f3d4ee0403fe8cbd",
+    "https://deno.land/std@0.224.0/fs/_is_same_path.ts": "709c95868345fea051c58b9e96af95cff94e6ae98dfcff2b66dee0c212c4221f",
+    "https://deno.land/std@0.224.0/fs/_is_subdir.ts": "c68b309d46cc8568ed83c000f608a61bbdba0943b7524e7a30f9e450cf67eecd",
+    "https://deno.land/std@0.224.0/fs/_to_path_string.ts": "29bfc9c6c112254961d75cbf6ba814d6de5349767818eb93090cecfa9665591e",
+    "https://deno.land/std@0.224.0/fs/copy.ts": "7ab12a16adb65d155d4943c88081ca16ce3b0b5acada64c1ce93800653678039",
+    "https://deno.land/std@0.224.0/fs/empty_dir.ts": "e400e96e1d2c8c558a5a1712063bd43939e00619c1d1cc29959babc6f1639418",
+    "https://deno.land/std@0.224.0/fs/ensure_dir.ts": "51a6279016c65d2985f8803c848e2888e206d1b510686a509fa7cc34ce59d29f",
+    "https://deno.land/std@0.224.0/fs/ensure_file.ts": "67608cf550529f3d4aa1f8b6b36bf817bdc40b14487bf8f60e61cbf68f507cf3",
+    "https://deno.land/std@0.224.0/fs/ensure_link.ts": "5c98503ebfa9cc05e2f2efaa30e91e60b4dd5b43ebbda82f435c0a5c6e3ffa01",
+    "https://deno.land/std@0.224.0/fs/ensure_symlink.ts": "cafe904cebacb9a761977d6dbf5e3af938be946a723bb394080b9a52714fafe4",
+    "https://deno.land/std@0.224.0/fs/eol.ts": "18c4ac009d0318504c285879eb7f47942643f13619e0ff070a0edc59353306bd",
+    "https://deno.land/std@0.224.0/fs/exists.ts": "3d38cb7dcbca3cf313be343a7b8af18a87bddb4b5ca1bd2314be12d06533b50f",
+    "https://deno.land/std@0.224.0/fs/expand_glob.ts": "2e428d90acc6676b2aa7b5c78ef48f30641b13f1fe658e7976c9064fb4b05309",
+    "https://deno.land/std@0.224.0/fs/mod.ts": "c25e6802cbf27f3050f60b26b00c2d8dba1cb7fcdafe34c66006a7473b7b34d4",
+    "https://deno.land/std@0.224.0/fs/move.ts": "ca205d848908d7f217353bc5c623627b1333490b8b5d3ef4cab600a700c9bd8f",
+    "https://deno.land/std@0.224.0/fs/walk.ts": "cddf87d2705c0163bff5d7767291f05b0f46ba10b8b28f227c3849cace08d303",
+    "https://deno.land/std@0.224.0/path/_common/assert_path.ts": "dbdd757a465b690b2cc72fc5fb7698c51507dec6bfafce4ca500c46b76ff7bd8",
+    "https://deno.land/std@0.224.0/path/_common/basename.ts": "569744855bc8445f3a56087fd2aed56bdad39da971a8d92b138c9913aecc5fa2",
+    "https://deno.land/std@0.224.0/path/_common/constants.ts": "dc5f8057159f4b48cd304eb3027e42f1148cf4df1fb4240774d3492b5d12ac0c",
+    "https://deno.land/std@0.224.0/path/_common/dirname.ts": "684df4aa71a04bbcc346c692c8485594fc8a90b9408dfbc26ff32cf3e0c98cc8",
+    "https://deno.land/std@0.224.0/path/_common/from_file_url.ts": "d672bdeebc11bf80e99bf266f886c70963107bdd31134c4e249eef51133ceccf",
+    "https://deno.land/std@0.224.0/path/_common/glob_to_reg_exp.ts": "6cac16d5c2dc23af7d66348a7ce430e5de4e70b0eede074bdbcf4903f4374d8d",
+    "https://deno.land/std@0.224.0/path/_common/normalize.ts": "684df4aa71a04bbcc346c692c8485594fc8a90b9408dfbc26ff32cf3e0c98cc8",
+    "https://deno.land/std@0.224.0/path/_common/normalize_string.ts": "33edef773c2a8e242761f731adeb2bd6d683e9c69e4e3d0092985bede74f4ac3",
+    "https://deno.land/std@0.224.0/path/_common/strip_trailing_separators.ts": "7024a93447efcdcfeaa9339a98fa63ef9d53de363f1fbe9858970f1bba02655a",
+    "https://deno.land/std@0.224.0/path/_os.ts": "8fb9b90fb6b753bd8c77cfd8a33c2ff6c5f5bc185f50de8ca4ac6a05710b2c15",
+    "https://deno.land/std@0.224.0/path/basename.ts": "7ee495c2d1ee516ffff48fb9a93267ba928b5a3486b550be73071bc14f8cc63e",
+    "https://deno.land/std@0.224.0/path/constants.ts": "0c206169ca104938ede9da48ac952de288f23343304a1c3cb6ec7625e7325f36",
+    "https://deno.land/std@0.224.0/path/dirname.ts": "85bd955bf31d62c9aafdd7ff561c4b5fb587d11a9a5a45e2b01aedffa4238a7c",
+    "https://deno.land/std@0.224.0/path/from_file_url.ts": "911833ae4fd10a1c84f6271f36151ab785955849117dc48c6e43b929504ee069",
+    "https://deno.land/std@0.224.0/path/glob_to_regexp.ts": "7f30f0a21439cadfdae1be1bf370880b415e676097fda584a63ce319053b5972",
+    "https://deno.land/std@0.224.0/path/is_absolute.ts": "4791afc8bfd0c87f0526eaa616b0d16e7b3ab6a65b62942e50eac68de4ef67d7",
+    "https://deno.land/std@0.224.0/path/is_glob.ts": "a65f6195d3058c3050ab905705891b412ff942a292bcbaa1a807a74439a14141",
+    "https://deno.land/std@0.224.0/path/join.ts": "ae2ec5ca44c7e84a235fd532e4a0116bfb1f2368b394db1c4fb75e3c0f26a33a",
+    "https://deno.land/std@0.224.0/path/join_globs.ts": "5b3bf248b93247194f94fa6947b612ab9d3abd571ca8386cf7789038545e54a0",
+    "https://deno.land/std@0.224.0/path/normalize.ts": "4155743ccceeed319b350c1e62e931600272fad8ad00c417b91df093867a8352",
+    "https://deno.land/std@0.224.0/path/posix/_util.ts": "1e3937da30f080bfc99fe45d7ed23c47dd8585c5e473b2d771380d3a6937cf9d",
+    "https://deno.land/std@0.224.0/path/posix/basename.ts": "d2fa5fbbb1c5a3ab8b9326458a8d4ceac77580961b3739cd5bfd1d3541a3e5f0",
+    "https://deno.land/std@0.224.0/path/posix/constants.ts": "93481efb98cdffa4c719c22a0182b994e5a6aed3047e1962f6c2c75b7592bef1",
+    "https://deno.land/std@0.224.0/path/posix/dirname.ts": "76cd348ffe92345711409f88d4d8561d8645353ac215c8e9c80140069bf42f00",
+    "https://deno.land/std@0.224.0/path/posix/from_file_url.ts": "951aee3a2c46fd0ed488899d024c6352b59154c70552e90885ed0c2ab699bc40",
+    "https://deno.land/std@0.224.0/path/posix/glob_to_regexp.ts": "76f012fcdb22c04b633f536c0b9644d100861bea36e9da56a94b9c589a742e8f",
+    "https://deno.land/std@0.224.0/path/posix/is_absolute.ts": "cebe561ad0ae294f0ce0365a1879dcfca8abd872821519b4fcc8d8967f888ede",
+    "https://deno.land/std@0.224.0/path/posix/join.ts": "7fc2cb3716aa1b863e990baf30b101d768db479e70b7313b4866a088db016f63",
+    "https://deno.land/std@0.224.0/path/posix/join_globs.ts": "a9475b44645feddceb484ee0498e456f4add112e181cb94042cdc6d47d1cdd25",
+    "https://deno.land/std@0.224.0/path/posix/normalize.ts": "baeb49816a8299f90a0237d214cef46f00ba3e95c0d2ceb74205a6a584b58a91",
+    "https://deno.land/std@0.224.0/path/posix/normalize_glob.ts": "9c87a829b6c0f445d03b3ecadc14492e2864c3ebb966f4cea41e98326e4435c6",
+    "https://deno.land/std@0.224.0/path/posix/resolve.ts": "08b699cfeee10cb6857ccab38fa4b2ec703b0ea33e8e69964f29d02a2d5257cf",
+    "https://deno.land/std@0.224.0/path/resolve.ts": "a6f977bdb4272e79d8d0ed4333e3d71367cc3926acf15ac271f1d059c8494d8d",
+    "https://deno.land/std@0.224.0/path/windows/_util.ts": "d5f47363e5293fced22c984550d5e70e98e266cc3f31769e1710511803d04808",
+    "https://deno.land/std@0.224.0/path/windows/basename.ts": "6bbc57bac9df2cec43288c8c5334919418d784243a00bc10de67d392ab36d660",
+    "https://deno.land/std@0.224.0/path/windows/constants.ts": "5afaac0a1f67b68b0a380a4ef391bf59feb55856aa8c60dfc01bd3b6abb813f5",
+    "https://deno.land/std@0.224.0/path/windows/dirname.ts": "33e421be5a5558a1346a48e74c330b8e560be7424ed7684ea03c12c21b627bc9",
+    "https://deno.land/std@0.224.0/path/windows/from_file_url.ts": "ced2d587b6dff18f963f269d745c4a599cf82b0c4007356bd957cb4cb52efc01",
+    "https://deno.land/std@0.224.0/path/windows/glob_to_regexp.ts": "e45f1f89bf3fc36f94ab7b3b9d0026729829fabc486c77f414caebef3b7304f8",
+    "https://deno.land/std@0.224.0/path/windows/is_absolute.ts": "4a8f6853f8598cf91a835f41abed42112cebab09478b072e4beb00ec81f8ca8a",
+    "https://deno.land/std@0.224.0/path/windows/join.ts": "8d03530ab89195185103b7da9dfc6327af13eabdcd44c7c63e42e27808f50ecf",
+    "https://deno.land/std@0.224.0/path/windows/join_globs.ts": "a9475b44645feddceb484ee0498e456f4add112e181cb94042cdc6d47d1cdd25",
+    "https://deno.land/std@0.224.0/path/windows/normalize.ts": "78126170ab917f0ca355a9af9e65ad6bfa5be14d574c5fb09bb1920f52577780",
+    "https://deno.land/std@0.224.0/path/windows/normalize_glob.ts": "9c87a829b6c0f445d03b3ecadc14492e2864c3ebb966f4cea41e98326e4435c6",
+    "https://deno.land/std@0.224.0/path/windows/resolve.ts": "8dae1dadfed9d46ff46cc337c9525c0c7d959fb400a6308f34595c45bdca1972",
     "https://deno.land/x/esbuild@v0.24.0/wasm.js": "5cd1dd0c40214d06bd86177b4ffebfbb219a22114f78c14c23606f7ad216c174"
   },
   "workspace": {
diff --git a/src/app.ts b/src/app.ts
index 0dfac89..c106f9c 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -30,7 +30,7 @@ export async function runApp(config: {
     config.projectPath,
     config.ipfs,
     config.cacheDir,
-    config.forceReload,
+    config.forceReload
   );
 
   const sandbox = await getDefaultSandbox(loader, config.toolTimeout);
@@ -39,7 +39,7 @@ export async function runApp(config: {
     config.host,
     sandbox,
     loader,
-    config.openAiApiKey,
+    config.openAiApiKey
   );
 
   const runnerHost = new RunnerHost(() => {
diff --git a/src/http.ts b/src/http.ts
index e154ff9..bb770c7 100644
--- a/src/http.ts
+++ b/src/http.ts
@@ -14,6 +14,8 @@ const Message = Type.Object({
     Type.Literal("assistant"),
     Type.Literal("tool"),
   ]),
+  id: Type.Optional(Type.String()),
+  conversation_id: Type.Optional(Type.String()),
 });
 
 const CompletionChoice = Type.Object({
@@ -56,6 +58,7 @@ const ChatResponse = Type.Object({
 
 const ChatChunkResponse = Type.Object({
   id: Type.String(),
+  conversation_id: Type.Optional(Type.String()),
   model: Type.String(),
   choices: Type.Array(CompletionChunkChoice),
   created: Type.Number({ description: "Unix timestamp in seconds" }),
@@ -73,13 +76,13 @@ export function http(
   runnerHost: RunnerHost,
   port: number,
   streamKeepAlive: number = 0,
-  onReady?: Promise<unknown>,
+  onReady?: Promise<unknown>
 ): Deno.HttpServer<Deno.NetAddr> {
   const app = new Hono();
 
   // The ready status should change once the project is fully loaded, including the vector DB
   let ready = false;
-  onReady?.then(() => ready = true);
+  onReady?.then(() => (ready = true));
 
   app.use("*", cors());
 
@@ -120,11 +123,13 @@ export function http(
       if (req.stream) {
         return streamSSE(c, async (stream) => {
           // Send empty data to keep the connection alive in browsers, they have a default timeout of 1m
-          const interval = streamKeepAlive && setInterval(async () => {
-            const empty = createChatChunkResponse("", "", new Date());
-            await stream.writeSSE({ data: JSON.stringify(empty) });
-            await stream.sleep(20);
-          }, streamKeepAlive);
+          const interval =
+            streamKeepAlive &&
+            setInterval(async () => {
+              const empty = createChatChunkResponse("", "", new Date());
+              await stream.writeSSE({ data: JSON.stringify(empty) });
+              await stream.sleep(20);
+            }, streamKeepAlive);
 
           const chatRes = await runner.promptMessages(req.messages);
 
@@ -140,6 +145,8 @@ export function http(
               chatRes.model,
               chatRes.created_at,
               last ? "stop" : null,
+              chatRes.message.id,
+              chatRes.message.conversation_id
             );
             await stream.writeSSE({ data: JSON.stringify(res) });
             await stream.sleep(20);
@@ -149,7 +156,7 @@ export function http(
               const res_space = createChatChunkResponse(
                 " ",
                 chatRes.model,
-                chatRes.created_at,
+                chatRes.created_at
               );
               await stream.writeSSE({ data: JSON.stringify(res_space) });
               await stream.sleep(20);
@@ -161,17 +168,19 @@ export function http(
       const chatRes = await runner.promptMessages(req.messages);
 
       const response: ChatResponse = {
-        id: "0",
+        id: crypto.randomUUID(),
         model: chatRes.model,
-        choices: [{
-          index: 0,
-          message: {
-            content: chatRes.message.content,
-            role: "assistant",
+        choices: [
+          {
+            index: 0,
+            message: {
+              content: chatRes.message.content,
+              role: "assistant",
+            },
+            logprobs: null,
+            finish_reason: chatRes.done_reason,
           },
-          logprobs: null,
-          finish_reason: chatRes.done_reason,
-        }],
+        ],
         created: new Date(chatRes.created_at).getTime() / 1000,
         object: "chat.completion",
         usage: {
@@ -204,18 +213,23 @@ function createChatChunkResponse(
   model: string,
   createdAt: Date,
   finish_reason: string | null = null,
+  id?: string,
+  conversation_id?: string
 ): ChatChunkResponse {
   const res: ChatChunkResponse = {
-    id: "0",
+    id: id ?? "0",
+    conversation_id: conversation_id ?? "0",
     object: "chat.completion.chunk",
     model,
     created: new Date(createdAt).getTime() / 1000,
-    choices: [{
-      index: 0,
-      delta: { role: "assistant", content: message },
-      logprobs: null,
-      finish_reason,
-    }],
+    choices: [
+      {
+        index: 0,
+        delta: { role: "assistant", content: message },
+        logprobs: null,
+        finish_reason,
+      },
+    ],
   };
   Value.Assert(ChatChunkResponse, res);
   return res;
diff --git a/src/loader.ts b/src/loader.ts
index 000409d..ccce88f 100644
--- a/src/loader.ts
+++ b/src/loader.ts
@@ -12,7 +12,9 @@ import { getLogger } from "./logger.ts";
 const logger = await getLogger("loader");
 
 export const getOSTempDir = () =>
-  Deno.env.get("TMPDIR") || Deno.env.get("TMP") || Deno.env.get("TEMP") ||
+  Deno.env.get("TMPDIR") ||
+  Deno.env.get("TMP") ||
+  Deno.env.get("TEMP") ||
   "/tmp";
 
 async function loadJson(path: string): Promise<unknown> {
@@ -58,13 +60,12 @@ async function loadManfiest(path: string): Promise<ProjectManifest> {
  */
 async function extractArchive(
   readable: ReadableStream<Uint8Array>,
-  dest: string,
+  dest: string
 ): Promise<string | undefined> {
   let first: string | undefined;
-  for await (
-    const entry of readable.pipeThrough(new DecompressionStream("gzip"))
-      .pipeThrough(new UntarStream())
-  ) {
+  for await (const entry of readable
+    .pipeThrough(new DecompressionStream("gzip"))
+    .pipeThrough(new UntarStream())) {
     const path = resolve(dest, entry.path);
     if (!first) {
       first = path;
@@ -92,7 +93,7 @@ export async function pullContent(
   fileName: string,
   tmpDir?: string,
   force?: boolean,
-  workingPath?: string,
+  workingPath?: string
 ): Promise<[string, Source]> {
   if (CIDReg.test(path)) {
     const cid = path.replace("ipfs://", "");
@@ -155,7 +156,7 @@ export async function pullContent(
         }
 
         const tmp = resolve(
-          fromFileUrlSafe(tmpDir ?? await Deno.makeTempDir()),
+          fromFileUrlSafe(tmpDir ?? (await Deno.makeTempDir()))
         );
         try {
           const p = await extractArchive(res.body, tmp);
@@ -173,7 +174,7 @@ export async function pullContent(
 
   const localPath = resolve(fromFileUrlSafe(workingPath ?? ""), path);
   if (localPath.endsWith(".gz")) {
-    const tmp = resolve(fromFileUrlSafe(tmpDir ?? await Deno.makeTempDir()));
+    const tmp = resolve(fromFileUrlSafe(tmpDir ?? (await Deno.makeTempDir())));
 
     const archive = await Deno.open(localPath);
 
@@ -187,10 +188,7 @@ export async function pullContent(
   }
 
   // File urls are used to avoid imports being from the same package registry as the framework is run from
-  return [
-    toFileUrlString(localPath),
-    "local",
-  ];
+  return [toFileUrlString(localPath), "local"];
 }
 
 export class Loader {
@@ -201,7 +199,7 @@ export class Loader {
     readonly projectPath: string,
     ipfs: IPFSClient,
     readonly tmpDir?: string,
-    force?: boolean,
+    force?: boolean
   ) {
     this.#ipfs = ipfs;
     this.#force = force ?? false;
@@ -211,7 +209,7 @@ export class Loader {
     path: string,
     fileName: string,
     tmpDir = this.tmpDir,
-    workingPath?: string,
+    workingPath?: string
   ): Promise<[string, Source]> {
     return await pullContent(
       path,
@@ -219,7 +217,7 @@ export class Loader {
       fileName,
       tmpDir,
       this.#force,
-      workingPath,
+      workingPath
     );
   }
 
@@ -234,7 +232,7 @@ export class Loader {
       this.projectPath,
       "manifest.json",
       undefined,
-      Deno.cwd(),
+      Deno.cwd()
     );
 
     logger.debug(`getManifest [${source}] ${manifestPath}`);
@@ -254,9 +252,10 @@ export class Loader {
       manifest.entry,
       "project.ts",
       dirname(manifestPath),
-      manifestSource == "local" ? dirname(this.projectPath) : undefined,
+      manifestSource == "local" ? dirname(this.projectPath) : undefined
     );
     logger.debug(`getProject [${source}] ${projectPath}`);
+
     return [projectPath, source];
   }
 
@@ -275,7 +274,7 @@ export class Loader {
       manifest.vectorStorage.path,
       "db.gz",
       dirname(manifestPath),
-      manifestSource == "local" ? dirname(this.projectPath) : undefined,
+      manifestSource == "local" ? dirname(this.projectPath) : undefined
     );
     logger.debug(`getVectorDb [${res[1]}] ${res[0]}`);
     return res;
diff --git a/src/project/project.ts b/src/project/project.ts
index 0d5d166..6b49db2 100644
--- a/src/project/project.ts
+++ b/src/project/project.ts
@@ -25,33 +25,45 @@ export const VectorConfig = Type.Object({
 export const ProjectManifest = Type.Object({
   specVersion: Type.Literal("0.0.1"),
   model: Type.String({ description: "The Ollama LLM model to be used" }),
-  embeddingsModel: Type.Optional(Type.String({
-    description: "The Ollama LLM model to be used for vector embeddings",
-  })),
+  embeddingsModel: Type.Optional(
+    Type.String({
+      description: "The Ollama LLM model to be used for vector embeddings",
+    })
+  ),
   entry: Type.String({
     description: "File path to the project entrypoint",
   }),
-  vectorStorage: Type.Optional(Type.Object({
-    type: Type.String({
-      description:
-        "The type of vector storage, currently only lancedb is supported.",
-    }),
-    path: Type.String({ description: "The path to the db" }),
-  })),
-  endpoints: Type.Optional(Type.Array(Type.String({
-    description: "Allowed endpoints the tools are allowed to make requests to",
-  }))),
+  vectorStorage: Type.Optional(
+    Type.Object({
+      type: Type.String({
+        description:
+          "The type of vector storage, currently only lancedb is supported.",
+      }),
+      path: Type.String({ description: "The path to the db" }),
+    })
+  ),
+  endpoints: Type.Optional(
+    Type.Array(
+      Type.String({
+        description:
+          "Allowed endpoints the tools are allowed to make requests to",
+      })
+    )
+  ),
   config: Type.Optional(Type.Any()), // TODO how can this be a JSON Schema type?
 });
 
 export const Project = Type.Object({
   tools: Type.Array(FunctionToolType),
   systemPrompt: Type.String(),
+  onResponse: Type.Optional(
+    Type.Function([Type.Object({})], Type.Promise(Type.Void()))
+  ),
 });
 
 export const ProjectEntry = Type.Function(
   [Type.Any()],
-  Type.Union([Project, Type.Promise(Project)]),
+  Type.Union([Project, Type.Promise(Project)])
 );
 
 export type FunctionToolType = Static<typeof FunctionToolType>;
@@ -62,7 +74,7 @@ export type ProjectEntry = Static<typeof ProjectEntry>;
 export async function loadProject(
   manifest: ProjectManifest,
   entry: unknown,
-  config?: Record<string, string>,
+  config?: Record<string, string>
 ): Promise<Project> {
   try {
     Value.Assert(ProjectEntry, entry);
diff --git a/src/runners/openai.ts b/src/runners/openai.ts
index 0cdc92e..ee2c6f9 100644
--- a/src/runners/openai.ts
+++ b/src/runners/openai.ts
@@ -1,4 +1,4 @@
-import type { ChatResponse, Message, Tool } from "ollama";
+import type { ChatResponse, Tool } from "ollama";
 import type { IRunner, IRunnerFactory } from "./runner.ts";
 import OpenAI from "openai";
 import type { IChatStorage } from "../chatStorage/chatStorage.ts";
@@ -8,6 +8,7 @@ import { LogPerformance, Memoize } from "../decorators.ts";
 import type { Loader } from "../loader.ts";
 import { Context } from "../context/context.ts";
 import { getLogger } from "../logger.ts";
+import type { Message } from "./types.ts";
 
 const logger = await getLogger("runner:openai");
 export class OpenAIRunnerFactory implements IRunnerFactory {
@@ -172,13 +173,18 @@ export class OpenAIRunner implements IRunner {
 
     // Convert respons to Ollama ChatResponse
     const choice = completion.choices[0];
+    const newMessage = {
+      content: choice.message.content ?? "",
+      role: choice.message.role,
+      id: crypto.randomUUID(),
+      conversation_id:
+        messages.find((i) => i.conversation_id)?.conversation_id ??
+        crypto.randomUUID(),
+    };
     const res: ChatResponse = {
       model: completion.model,
       created_at: new Date(completion.created * 1000),
-      message: {
-        content: choice.message.content ?? "",
-        role: choice.message.role,
-      },
+      message: newMessage,
       done: true,
       done_reason: choice.finish_reason,
       total_duration: 0,
@@ -189,6 +195,12 @@ export class OpenAIRunner implements IRunner {
       eval_duration: 0,
     };
 
+    try {
+      this.sandbox?.onResponse?.([...messages, newMessage]);
+    } catch {
+      // pass, maybe throw better?
+    }
+
     return res;
   }
 }
diff --git a/src/runners/runner.ts b/src/runners/runner.ts
index a7244b6..4139bfb 100644
--- a/src/runners/runner.ts
+++ b/src/runners/runner.ts
@@ -1,4 +1,4 @@
-import { type ChatResponse, type Message, Ollama } from "ollama";
+import { Ollama } from "ollama";
 import type { IChatStorage } from "../chatStorage/index.ts";
 import type { GenerateEmbedding } from "../embeddings/lance/writer.ts";
 import OpenAI from "openai";
@@ -7,6 +7,7 @@ import type { ISandbox } from "../sandbox/sandbox.ts";
 import type { Loader } from "../loader.ts";
 import { OpenAIRunnerFactory } from "./openai.ts";
 import { OllamaRunnerFactory } from "./ollama.ts";
+import type { Message, ChatResponse } from "./types.ts";
 
 export interface IRunner {
   prompt(message: string): Promise<string>;
@@ -18,7 +19,7 @@ export interface IRunnerFactory {
 }
 
 async function runForModels<T>(
-  runners: Record<string, () => Promise<T>>,
+  runners: Record<string, () => Promise<T>>
 ): Promise<T> {
   const errors: Record<string, unknown> = {};
   for (const [name, fn] of Object.entries(runners)) {
@@ -30,31 +31,25 @@ async function runForModels<T>(
   }
 
   throw new Error(`All options failed to run:
-\t${
-    Object.entries(errors).map(([name, error]) => `${name} error: ${error}`)
-      .join("\n\t")
-  }`);
+\t${Object.entries(errors)
+    .map(([name, error]) => `${name} error: ${error}`)
+    .join("\n\t")}`);
 }
 
 export function createRunner(
   endpoint: string,
   sandbox: ISandbox,
   loader: Loader,
-  openAiApiKey?: string,
+  openAiApiKey?: string
 ): Promise<IRunnerFactory> {
   return runForModels<IRunnerFactory>({
-    Ollama: () =>
-      OllamaRunnerFactory.create(
-        endpoint,
-        sandbox,
-        loader,
-      ),
+    Ollama: () => OllamaRunnerFactory.create(endpoint, sandbox, loader),
     OpenAI: () =>
       OpenAIRunnerFactory.create(
         endpoint === DEFAULT_LLM_HOST ? undefined : endpoint,
         openAiApiKey,
         sandbox,
-        loader,
+        loader
       ),
   });
 }
@@ -62,7 +57,7 @@ export function createRunner(
 export function getGenerateFunction(
   endpoint: string,
   model: string,
-  apiKey?: string,
+  apiKey?: string
 ): Promise<GenerateEmbedding> {
   return runForModels<GenerateEmbedding>({
     Ollama: async () => {
@@ -77,9 +72,7 @@ export function getGenerateFunction(
         // https://github.com/ollama/ollama/issues/651
         if (dimensions != undefined && embeddings[0].length != dimensions) {
           throw new Error(
-            `Dimensions mismatch, expected:"${dimensions}" received:"${
-              embeddings[0].length
-            }"`,
+            `Dimensions mismatch, expected:"${dimensions}" received:"${embeddings[0].length}"`
           );
         }
         return embeddings;
diff --git a/src/runners/types.ts b/src/runners/types.ts
new file mode 100644
index 0000000..e83c94c
--- /dev/null
+++ b/src/runners/types.ts
@@ -0,0 +1,13 @@
+import {
+  type Message as OllamaMessage,
+  ChatResponse as OllamaChatResponse,
+} from "ollama";
+
+export interface Message extends OllamaMessage {
+  id?: string;
+  conversation_id?: string;
+}
+
+export interface ChatResponse extends OllamaChatResponse {
+  message: Message;
+}
diff --git a/src/sandbox/sandbox.ts b/src/sandbox/sandbox.ts
index 57d46aa..cff294c 100644
--- a/src/sandbox/sandbox.ts
+++ b/src/sandbox/sandbox.ts
@@ -1,4 +1,4 @@
-import type { Tool } from "ollama";
+import type { Message, Tool } from "ollama";
 import type { IContext } from "../context/types.ts";
 import type { ProjectManifest } from "../project/project.ts";
 
@@ -13,4 +13,6 @@ export interface ISandbox {
   getTools(): Promise<Tool[]>;
 
   runTool(toolName: string, args: unknown, ctx: IContext): Promise<string>;
+
+  onResponse?: (message: Message[]) => Promise<void>;
 }
diff --git a/src/sandbox/webWorker/messages.ts b/src/sandbox/webWorker/messages.ts
index f09e2e7..2a77606 100644
--- a/src/sandbox/webWorker/messages.ts
+++ b/src/sandbox/webWorker/messages.ts
@@ -1,8 +1,12 @@
-import type { Tool } from "ollama";
+import type { Message, Tool } from "ollama";
 import * as rpc from "vscode-jsonrpc";
 import type { ProjectManifest } from "../../project/project.ts";
 
-export type IProjectJson = { tools: Tool[]; systemPrompt: string };
+export type IProjectJson = {
+  tools: Tool[];
+  systemPrompt: string;
+  onResponse?: { manifest: string };
+};
 
 // Framework -> Sandbox
 export const Load = new rpc.RequestType<string, void, string>("load");
@@ -13,9 +17,15 @@ export const Init = new rpc.RequestType2<
   string
 >("init");
 export const CallTool = new rpc.RequestType2<string, unknown, string, void>(
-  "call_tool",
+  "call_tool"
 );
 
+export const CallOnResponse = new rpc.RequestType<
+  Message[],
+  void | undefined,
+  void
+>("call_on_response");
+
 // Sandbox -> Framework
 export const CtxVectorSearch = new rpc.RequestType2<
   string,
diff --git a/src/sandbox/webWorker/webWorker.ts b/src/sandbox/webWorker/webWorker.ts
index 5c4e35e..d1900f8 100644
--- a/src/sandbox/webWorker/webWorker.ts
+++ b/src/sandbox/webWorker/webWorker.ts
@@ -5,6 +5,7 @@ import {
 } from "vscode-jsonrpc/browser.js";
 import {
   CallTool,
+  CallOnResponse,
   CtxComputeQueryEmbedding,
   CtxVectorSearch,
   Init,
@@ -19,7 +20,7 @@ import { loadProject } from "../../project/project.ts";
 
 const conn = rpc.createMessageConnection(
   new BrowserMessageReader(self),
-  new BrowserMessageWriter(self),
+  new BrowserMessageWriter(self)
 );
 
 let entrypoint: unknown;
@@ -33,10 +34,11 @@ const context = {
 } satisfies IContext;
 
 function toJsonProject(): IProjectJson {
-  const { tools, ...rest } = project;
+  const { tools, systemPrompt } = project;
   return {
-    ...rest,
+    systemPrompt,
     tools: tools.map((t) => t.toTool()),
+    onResponse: { manifest: "" },
   };
 }
 
@@ -51,7 +53,6 @@ conn.onRequest(Init, async (manifest, config) => {
 
   try {
     project ??= await loadProject(manifest, entrypoint, config);
-
     return toJsonProject();
   } catch (e: unknown) {
     if (e instanceof Error) {
@@ -75,4 +76,12 @@ conn.onRequest(CallTool, (toolName, args) => {
   return tool.call(args, context);
 });
 
+conn.onRequest(CallOnResponse, (message) => {
+  if (!project) {
+    throw new Error("Project is not initialized");
+  }
+
+  return project.onResponse?.(message);
+});
+
 conn.listen();
diff --git a/src/sandbox/webWorker/webWorkerSandbox.ts b/src/sandbox/webWorker/webWorkerSandbox.ts
index 880aeeb..4a4b02c 100644
--- a/src/sandbox/webWorker/webWorkerSandbox.ts
+++ b/src/sandbox/webWorker/webWorkerSandbox.ts
@@ -1,4 +1,4 @@
-import type { Tool } from "ollama";
+import type { Message, Tool } from "ollama";
 import * as rpc from "vscode-jsonrpc";
 import {
   BrowserMessageReader,
@@ -6,6 +6,7 @@ import {
 } from "vscode-jsonrpc/browser.js";
 import type { ISandbox } from "../sandbox.ts";
 import {
+  CallOnResponse,
   CallTool,
   CtxComputeQueryEmbedding,
   CtxVectorSearch,
@@ -56,7 +57,7 @@ const LOCAL_PERMISSIONS: Deno.PermissionOptionsObject = {
 
 function getPermisionsForSource(
   source: Source,
-  projectDir: string,
+  projectDir: string
 ): Deno.PermissionOptionsObject {
   switch (source) {
     case "local":
@@ -65,7 +66,7 @@ function getPermisionsForSource(
       return IPFS_PERMISSIONS(projectDir);
     default:
       throw new Error(
-        `Unable to set permissions for unknown source: ${source}`,
+        `Unable to set permissions for unknown source: ${source}`
       );
   }
 }
@@ -74,33 +75,24 @@ async function workerFactory(
   manifest: ProjectManifest,
   entryPath: string,
   config: Record<string, string>,
-  permissions: Deno.PermissionOptionsObject,
+  permissions: Deno.PermissionOptionsObject
 ): Promise<[Worker, rpc.MessageConnection, IProjectJson]> {
-  const w = new Worker(
-    import.meta.resolve("./webWorker.ts"),
-    {
-      type: "module",
-      deno: {
-        permissions: permissions,
-      },
+  const w = new Worker(import.meta.resolve("./webWorker.ts"), {
+    type: "module",
+    deno: {
+      permissions: permissions,
     },
-  );
+  });
 
   // Setup a JSON RPC for interaction to the worker
   const conn = rpc.createMessageConnection(
     new BrowserMessageReader(w),
-    new BrowserMessageWriter(w),
+    new BrowserMessageWriter(w)
   );
-
   conn.listen();
 
   await conn.sendRequest(Load, entryPath);
-
-  const pJson = await conn.sendRequest(
-    Init,
-    manifest,
-    config,
-  );
+  const pJson = await conn.sendRequest(Init, manifest, config);
 
   return [w, conn, pJson];
 }
@@ -117,7 +109,7 @@ export class WebWorkerSandbox implements ISandbox {
    */
   public static async create(
     loader: Loader,
-    timeout: number,
+    timeout: number
   ): Promise<WebWorkerSandbox> {
     const [manifestPath, manifest, source] = await loader.getManifest();
     const config = loadRawConfigFromEnv(manifest.config);
@@ -133,20 +125,14 @@ export class WebWorkerSandbox implements ISandbox {
     ];
 
     const [entryPath] = await loader.getProject();
-
     const initProjectWorker = () =>
-      workerFactory(
-        manifest,
-        entryPath,
-        config as Record<string, string>,
-        {
-          ...permissions,
-          env: false,
-          net: hostnames,
-          run: false,
-          write: false,
-        },
-      );
+      workerFactory(manifest, entryPath, config as Record<string, string>, {
+        ...permissions,
+        env: false,
+        net: hostnames,
+        run: false,
+        write: false,
+      });
 
     const [_worker, _conn, { tools, systemPrompt }] = await initProjectWorker();
 
@@ -155,7 +141,7 @@ export class WebWorkerSandbox implements ISandbox {
       systemPrompt,
       tools,
       initProjectWorker,
-      timeout,
+      timeout
     );
   }
 
@@ -164,7 +150,7 @@ export class WebWorkerSandbox implements ISandbox {
     readonly systemPrompt: string,
     tools: Tool[],
     initWorker: () => ReturnType<typeof workerFactory>,
-    readonly timeout: number = 100,
+    readonly timeout: number = 100
   ) {
     this.#tools = tools;
     this.#initWorker = initWorker;
@@ -178,7 +164,7 @@ export class WebWorkerSandbox implements ISandbox {
   async runTool(
     toolName: string,
     args: unknown,
-    ctx: IContext,
+    ctx: IContext
   ): Promise<string> {
     // Create a worker just for the tool call, this is so we can terminate if it exceeds the timeout.
     const [worker, conn] = await this.#initWorker();
@@ -206,4 +192,17 @@ export class WebWorkerSandbox implements ISandbox {
       worker.terminate();
     });
   }
+
+  async onResponse(message: Message[]) {
+    const [worker, conn] = await this.#initWorker();
+
+    return Promise.race([
+      timeout(this.timeout).then(() => {
+        throw new Error(`Timeout calling onResponse`);
+      }),
+      conn.sendRequest(CallOnResponse, message),
+    ]).finally(() => {
+      worker.terminate();
+    });
+  }
 }