Merge pull request #16 from subquery/fixes-1024

Fixes 2024/10/24

Dockerfile

@@ -1,13 +1,15 @@
-FROM denoland/deno:2.0.0-rc.10
+FROM denoland/deno:2.0.1
 
 WORKDIR /ai-app
 
-# Prefer not to run as root.
-USER deno
+# TODO bring this back, the deno user is very restrictive so currently doens't allow using the tmp dir
+# RUN mkdir /.cache && chown deno /.cache
+# USER deno
+# TODO set cacheDir flag on entrypoint once the above works
 
 COPY . .
 RUN deno cache ./src/index.ts
 
 ENTRYPOINT ["./src/index.ts"]
 
-CMD ["-p","/app"]
+CMD ["-p","/ai-app"]

README.md

@@ -11,11 +11,11 @@ Detailed documentation is
 
 Install the latest:
 
-`deno install -g -f --allow-env --allow-net --allow-import --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options --no-prompt -n subql-ai jsr:@subql/ai-app-framework/cli`
+`deno install -g -f --allow-env --allow-net --allow-import --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options -n subql-ai jsr:@subql/ai-app-framework/cli`
 
 Install a specific version:
 
-`deno install -g -f --allow-env --allow-net --allow-import --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options --no-prompt -n subql-ai jsr:@subql/ai-app-framework@<version>/cli`
+`deno install -g -f --allow-env --allow-net --allow-import --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options -n subql-ai jsr:@subql/ai-app-framework@<version>/cli`
 
 `NOTE: These permissions can change, for the most upto date permissions see the top of ./src/index.ts`
 

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/ai-app-framework",
-  "version": "1.0.0-14",
+  "version": "1.0.0-15",
   "exports": {
     "./cli": "./src/index.ts",
     ".": "./src/mod.ts"
@@ -39,6 +39,7 @@
     "mdast-util-to-string": "npm:mdast-util-to-string@^4.0.0",
     "micromark-extension-mdxjs": "npm:micromark-extension-mdxjs@^3.0.0",
     "ollama": "npm:ollama@^0.5.9",
+    "openai": "npm:openai@^4.68.3",
     "ora": "npm:ora@^8.1.0",
     "pino": "npm:pino@^9.5.0",
     "pino-pretty": "npm:pino-pretty@^11.3.0",

src/app.ts

@@ -21,13 +21,15 @@ export async function runApp(config: {
   ipfs: IPFSClient;
   forceReload?: boolean;
   toolTimeout: number;
+  streamKeepAlive: number;
+  cacheDir?: string;
 }): Promise<void> {
   const model = new Ollama({ host: config.host });
 
   const loader = new Loader(
     config.projectPath,
     config.ipfs,
-    undefined,
+    config.cacheDir,
     config.forceReload,
   );
 
@@ -58,7 +60,7 @@ export async function runApp(config: {
       break;
     case "http":
     default:
-      http(runnerHost, config.port);
+      http(runnerHost, config.port, config.streamKeepAlive);
   }
 }
 

src/http.ts

@@ -72,6 +72,7 @@ export type ChatChunkResponse = Static<typeof ChatChunkResponse>;
 export function http(
   runnerHost: RunnerHost,
   port: number,
+  streamKeepAlive: number = 0,
   onReady?: Promise<unknown>,
 ): Deno.HttpServer<Deno.NetAddr> {
   const app = new Hono();
@@ -114,12 +115,23 @@ export function http(
       }
 
       const runner = await runnerHost.getAnonymousRunner();
-      const chatRes = await runner.promptMessages(req.messages);
 
       // Mock streaming, current Ollama doesn't support streaming with tools. See https://github.com/subquery/subql-ai-app-framework/issues/3
       if (req.stream) {
-        const parts = chatRes.message.content.split(" ");
         return streamSSE(c, async (stream) => {
+          // Send empty data to keep the connection alive in browsers, they have a default timeout of 1m
+          const interval = streamKeepAlive && setInterval(async () => {
+            const empty = createChatChunkResponse("", "", new Date());
+            await stream.writeSSE({ data: JSON.stringify(empty) });
+            await stream.sleep(20);
+          }, streamKeepAlive);
+
+          const chatRes = await runner.promptMessages(req.messages);
+
+          // Stop sending empty data and send the actual response
+          interval && clearInterval(interval);
+
+          const parts = chatRes.message.content.split(" ");
           for (const [i, part] of parts.entries()) {
             const last = i == parts.length - 1;
 
@@ -146,6 +158,8 @@ export function http(
         });
       }
 
+      const chatRes = await runner.promptMessages(req.messages);
+
       const response: ChatResponse = {
         id: "0",
         model: chatRes.model,

src/index.ts

@@ -41,6 +41,11 @@ const sharedArgs = {
       "A bearer authentication token to be used with the ipfs endpoint",
     type: "string",
   },
+  cacheDir: {
+    description:
+      "The location to cache data from ipfs. Default is a temp directory",
+    type: "string",
+  },
 } satisfies Record<string, Options>;
 
 const debugArgs = {
@@ -106,6 +111,12 @@ yargs(Deno.args)
         type: "number",
         default: 10_000, // 10s
       },
+      streamKeepAlive: {
+        description:
+          "The interval in MS to send empty data in stream responses to keep the connection alive. Only wokrs with http interface. Use 0 to disable.",
+        type: "number",
+        default: 5_000, // 5s
+      },
     },
     async (argv) => {
       try {
@@ -124,6 +135,8 @@ yargs(Deno.args)
           ipfs: ipfsFromArgs(argv),
           forceReload: argv.forceReload,
           toolTimeout: argv.toolTimeout,
+          streamKeepAlive: argv.streamKeepAlive,
+          cacheDir: argv.cacheDir,
         });
       } catch (e) {
         console.log(e);
@@ -212,10 +225,15 @@ yargs(Deno.args)
         type: "string",
         default: `http://localhost:${DEFAULT_PORT}`,
       },
+      stream: {
+        description: "Stream responses",
+        type: "boolean",
+        default: true,
+      },
     },
     async (argv) => {
       const { httpCli } = await import("./subcommands/httpCli.ts");
-      await httpCli(argv.host);
+      await httpCli(argv.host, argv.stream);
     },
   )
   .command(
@@ -284,6 +302,7 @@ yargs(Deno.args)
       }
     },
   )
+  .strict()
   // .fail(() => {}) // Disable logging --help if theres an error with a command // TODO need to fix so it only logs when error is with yargs
   .help()
   .argv;

src/loader.ts

@@ -1,28 +1,23 @@
 import { dirname } from "@std/path/dirname";
 import { resolve } from "@std/path/resolve";
-import { fromFileUrl } from "@std/path/from-file-url";
-import { toFileUrl } from "@std/path/to-file-url";
 import { CIDReg, type IPFSClient } from "./ipfs.ts";
-
 import { UntarStream } from "@std/tar";
 import { ensureDir, exists } from "@std/fs";
-import type { Source } from "./util.ts";
+import { fromFileUrlSafe, type Source, toFileUrlString } from "./util.ts";
 import { ProjectManifest } from "./project/project.ts";
 import { Value } from "@sinclair/typebox/value";
 import { Memoize, SpinnerLog } from "./decorators.ts";
 import { getLogger } from "./logger.ts";
 
 const logger = await getLogger("loader");
 
-const toFileUrlString = (input: string) => toFileUrl(input).toString();
-
 export const getOSTempDir = () =>
   Deno.env.get("TMPDIR") || Deno.env.get("TMP") || Deno.env.get("TEMP") ||
   "/tmp";
 
 async function loadJson(path: string): Promise<unknown> {
   const decoder = new TextDecoder();
-  const normalPath = path.startsWith("file://") ? fromFileUrl(path) : path;
+  const normalPath = fromFileUrlSafe(path);
   const data = await Deno.readFile(normalPath);
   const raw = decoder.decode(data);
 
@@ -74,7 +69,7 @@ export async function pullContent(
 ): Promise<[string, Source]> {
   if (CIDReg.test(path)) {
     const cid = path.replace("ipfs://", "");
-    const tmp = resolve(tmpDir ?? getOSTempDir(), cid);
+    const tmp = resolve(fromFileUrlSafe(tmpDir ?? getOSTempDir()), cid);
     await ensureDir(tmp);
 
     if (fileName.endsWith(".gz")) {
@@ -123,10 +118,10 @@ export async function pullContent(
   }
 
   // File urls are used to avoid imports being from the same package registry as the framework is run from
-  workingPath = workingPath?.startsWith("file://")
-    ? fromFileUrl(workingPath)
-    : workingPath;
-  return [toFileUrlString(resolve(workingPath ?? "", path)), "local"];
+  return [
+    toFileUrlString(resolve(fromFileUrlSafe(workingPath ?? ""), path)),
+    "local",
+  ];
 }
 
 export class Loader {
@@ -213,6 +208,7 @@ export class Loader {
       dirname(manifestPath),
       manifestSource == "local" ? dirname(this.projectPath) : undefined,
     );
+    logger.debug(`getVectorDb [${res[1]}] ${res[0]}`);
     return res;
   }
 }