diff --git a/apps/web/app/api/user/settings/route.ts b/apps/web/app/api/user/settings/route.ts index c0215683..885eb69c 100644 --- a/apps/web/app/api/user/settings/route.ts +++ b/apps/web/app/api/user/settings/route.ts @@ -30,6 +30,8 @@ async function saveAISettings(options: SaveSettingsBody) { return Model.CLAUDE_3_5_SONNET_BEDROCK; case Provider.GOOGLE: return options.aiModel || Model.GEMINI_1_5_PRO; + case Provider.GROQ: + return options.aiModel || Model.GROQ_LLAMA_3_3_70B; case Provider.OLLAMA: return Model.OLLAMA; default: diff --git a/apps/web/app/api/user/settings/validation.ts b/apps/web/app/api/user/settings/validation.ts index 2760b591..bad0f781 100644 --- a/apps/web/app/api/user/settings/validation.ts +++ b/apps/web/app/api/user/settings/validation.ts @@ -7,6 +7,7 @@ export const saveSettingsBody = z Provider.ANTHROPIC, Provider.OPEN_AI, Provider.GOOGLE, + Provider.GROQ, ...(Provider.OLLAMA ? [Provider.OLLAMA] : []), ]), aiModel: z.string(), diff --git a/apps/web/package.json b/apps/web/package.json index f1e9e42f..2f88bed9 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -16,6 +16,7 @@ "@ai-sdk/amazon-bedrock": "^1.0.6", "@ai-sdk/anthropic": "^1.0.6", "@ai-sdk/google": "^1.0.12", + "@ai-sdk/groq": "^1.0.10", "@ai-sdk/openai": "^1.0.11", "@asteasolutions/zod-to-openapi": "^7.3.0", "@auth/core": "^0.37.4", diff --git a/apps/web/utils/llms/config.ts b/apps/web/utils/llms/config.ts index 49cfaf76..d2b872c4 100644 --- a/apps/web/utils/llms/config.ts +++ b/apps/web/utils/llms/config.ts @@ -6,6 +6,7 @@ export const Provider = { OPEN_AI: "openai", ANTHROPIC: "anthropic", GOOGLE: "google", + GROQ: "groq", ...(supportsOllama ? { OLLAMA: "ollama" } : {}), }; @@ -18,6 +19,7 @@ export const Model = { CLAUDE_3_5_SONNET_ANTHROPIC: "claude-3-5-sonnet-20241022", GEMINI_1_5_PRO: "gemini-1.5-pro-latest", GEMINI_1_5_FLASH: "gemini-1.5-flash-latest", + GROQ_LLAMA_3_3_70B: "llama-3.3-70b-versatile", ...(supportsOllama ? { OLLAMA: env.NEXT_PUBLIC_OLLAMA_MODEL } : {}), }; @@ -25,6 +27,7 @@ export const providerOptions: { label: string; value: string }[] = [ { label: "OpenAI", value: Provider.OPEN_AI }, { label: "Anthropic", value: Provider.ANTHROPIC }, { label: "Google", value: Provider.GOOGLE }, + { label: "Groq", value: Provider.GROQ }, ...(supportsOllama && Provider.OLLAMA ? [{ label: "Ollama", value: Provider.OLLAMA }] : []), @@ -52,6 +55,12 @@ export const modelOptions: Record = value: Model.GEMINI_1_5_FLASH, }, ], + [Provider.GROQ]: [ + { + label: "Groq Llama 3.3 70B", + value: Model.GROQ_LLAMA_3_3_70B, + }, + ], ...(Provider.OLLAMA && Model.OLLAMA ? { [Provider.OLLAMA]: [{ label: "Ollama", value: Model.OLLAMA }], diff --git a/apps/web/utils/llms/index.ts b/apps/web/utils/llms/index.ts index 807bf399..e37e60af 100644 --- a/apps/web/utils/llms/index.ts +++ b/apps/web/utils/llms/index.ts @@ -11,6 +11,7 @@ import { createOpenAI } from "@ai-sdk/openai"; import { createAnthropic } from "@ai-sdk/anthropic"; import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createGroq } from "@ai-sdk/groq"; import { createOllama } from "ollama-ai-provider"; import { env } from "@/env"; import { saveAiUsage } from "@/utils/usage"; @@ -82,6 +83,17 @@ function getModel({ aiProvider, aiModel, aiApiKey }: UserAIFields) { }; } + if (provider === Provider.GROQ) { + if (!aiApiKey) throw new Error("Groq API key is not set"); + + const model = aiModel || Model.GROQ_LLAMA_3_3_70B; + return { + provider: Provider.GROQ, + model, + llmModel: createGroq({ apiKey: aiApiKey })(model), + }; + } + if (provider === Provider.OLLAMA && env.NEXT_PUBLIC_OLLAMA_MODEL) { return { provider: Provider.OLLAMA, diff --git a/apps/web/utils/usage.ts b/apps/web/utils/usage.ts index 8ed0e645..3423feca 100644 --- a/apps/web/utils/usage.ts +++ b/apps/web/utils/usage.ts @@ -101,6 +101,11 @@ const costs: Record< input: 0.075 / 1_000_000, output: 0.3 / 1_000_000, }, + // https://groq.com/pricing + "llama-3.3-70b-versatile": { + input: 0.59 / 1_000_000, + output: 0.79 / 1_000_000, + }, }; // returns cost in cents diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dbd3b748..6381d276 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -93,6 +93,9 @@ importers: '@ai-sdk/google': specifier: ^1.0.12 version: 1.0.12(zod@3.24.1) + '@ai-sdk/groq': + specifier: ^1.0.10 + version: 1.0.10(zod@3.24.1) '@ai-sdk/openai': specifier: ^1.0.11 version: 1.0.11(zod@3.24.1) @@ -668,6 +671,12 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/groq@1.0.10': + resolution: {integrity: sha512-FU2UT0+cz2VsaI7M0JgynC/3tXLspBBPBMauV36FfkFSeHscI1CCQlzQoTngp4mqXbfD+tJFN8gEg8qYCkawzw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + '@ai-sdk/openai@1.0.11': resolution: {integrity: sha512-qI9s7Slma5i5bB4yYVlFdcG3PNDwdqivPT1Dr8adDX92nSSpILjgFIooS5yys9sXjvvcfOi/WXbDvVhLSRRlvg==} engines: {node: '>=18'} @@ -692,6 +701,15 @@ packages: zod: optional: true + '@ai-sdk/provider-utils@2.0.6': + resolution: {integrity: sha512-nB0rPwIBSCk0UkfdkprAxQ45ZjfKlk+Ts5zvIBQkJ5SnTCL9meg6bW65aomQrxhdvtqZML2jjaWTI8/l6AIVlQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider@1.0.2': resolution: {integrity: sha512-YYtP6xWQyaAf5LiWLJ+ycGTOeBLWrED7LUrvc+SQIWhGaneylqbaGsyQL7VouQUeQ4JZ1qKYZuhmi3W56HADPA==} engines: {node: '>=18'} @@ -700,6 +718,10 @@ packages: resolution: {integrity: sha512-WiuJEpHTrltOIzv3x2wx4gwksAHW0h6nK3SoDzjqCOJLu/2OJ1yASESTIX+f07ChFykHElVoP80Ol/fe9dw6tQ==} engines: {node: '>=18'} + '@ai-sdk/provider@1.0.4': + resolution: {integrity: sha512-lJi5zwDosvvZER3e/pB8lj1MN3o3S7zJliQq56BRr4e9V3fcRyFtwP0JRxaRS5vHYX3OJ154VezVoQNrk0eaKw==} + engines: {node: '>=18'} + '@ai-sdk/react@1.0.7': resolution: {integrity: sha512-j2/of4iCNq+r2Bjx0O9vdRhn5C/02t2Esenis71YtnsoynPz74eQlJ3N0RYYPheThiJes50yHdfdVdH9ulxs1A==} engines: {node: '>=18'} @@ -12480,6 +12502,12 @@ snapshots: '@ai-sdk/provider-utils': 2.0.5(zod@3.24.1) zod: 3.24.1 + '@ai-sdk/groq@1.0.10(zod@3.24.1)': + dependencies: + '@ai-sdk/provider': 1.0.4 + '@ai-sdk/provider-utils': 2.0.6(zod@3.24.1) + zod: 3.24.1 + '@ai-sdk/openai@1.0.11(zod@3.24.1)': dependencies: '@ai-sdk/provider': 1.0.3 @@ -12504,6 +12532,15 @@ snapshots: optionalDependencies: zod: 3.24.1 + '@ai-sdk/provider-utils@2.0.6(zod@3.24.1)': + dependencies: + '@ai-sdk/provider': 1.0.4 + eventsource-parser: 3.0.0 + nanoid: 3.3.8 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.24.1 + '@ai-sdk/provider@1.0.2': dependencies: json-schema: 0.4.0 @@ -12512,6 +12549,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@1.0.4': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/react@1.0.7(react@18.3.1)(zod@3.24.1)': dependencies: '@ai-sdk/provider-utils': 2.0.5(zod@3.24.1)