feat: add support to images

.eslintrc.json

@@ -34,6 +34,7 @@
     "sort/object-properties": "error",
     "sort/type-properties": "error",
     "unicorn/no-null": "off",
+    "unicorn/no-array-reduce": "off",
     "unused-imports/no-unused-imports": "error"
   }
 }

README.md

@@ -53,18 +53,18 @@ const model = ollama('phi3');
 ### Tested models and capabilities
 
 This provider is capable of generating and streaming text and objects. It does not
-support image input and function calling (tools). Object generation may fail depending 
+support function calling (tools). Object generation may fail depending 
 on the model used and the schema used.
 
 At least it has been verified to work on the following models:
 
-| Model      | Image input | Object generation  | Tool usage | Tool streaming |
-|------------|-------------|--------------------|------------|----------------|
-| llama2     | :x:         | :white_check_mark: | :x:        | :x:            | 
-| llama3     | :x:         | :white_check_mark: | :x:        | :x:            | 
-| llava      | :x:         | :white_check_mark: | :x:        | :x:            | 
-| mistral    | :x:         | :white_check_mark: | :x:        | :x:            | 
-| mixtral    | :x:         | :white_check_mark: | :x:        | :x:            | 
-| openhermes | :x:         | :white_check_mark: | :x:        | :x:            | 
-| phi3       | :x:         | :white_check_mark: | :x:        | :x:            | 
+| Model      | Image input        | Object generation  | Tool usage | Tool streaming |
+|------------|--------------------|--------------------|------------|----------------|
+| llama2     | :x:                | :white_check_mark: | :x:        | :x:            | 
+| llama3     | :x:                | :white_check_mark: | :x:        | :x:            | 
+| llava      | :white_check_mark: | :white_check_mark: | :x:        | :x:            | 
+| mistral    | :x:                | :white_check_mark: | :x:        | :x:            | 
+| mixtral    | :x:                | :white_check_mark: | :x:        | :x:            | 
+| openhermes | :x:                | :white_check_mark: | :x:        | :x:            | 
+| phi3       | :x:                | :white_check_mark: | :x:        | :x:            | 
 

src/convert-to-ollama-chat-messages.test.ts

@@ -0,0 +1,81 @@
+import {
+  LanguageModelV1Prompt,
+  UnsupportedFunctionalityError,
+} from '@ai-sdk/provider'
+import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'
+import { describe, expect, it } from 'vitest'
+
+import { convertToOllamaChatMessages } from '@/convert-to-ollama-chat-messages'
+
+describe('convertToOllamaChatMessages', () => {
+  it('should convert LanguageModelV1Prompt to OllamaChatPrompt', () => {
+    // Arrange
+    const prompt: LanguageModelV1Prompt = [
+      { content: 'System message', role: 'system' },
+      {
+        content: [
+          { text: 'User ', type: 'text' },
+          { text: 'text ', type: 'text' },
+          { text: 'message', type: 'text' },
+          { image: new Uint8Array([0, 1, 2]), type: 'image' },
+          { image: new Uint8Array([3, 4, 5]), type: 'image' },
+        ],
+        role: 'user',
+      },
+      {
+        content: [{ text: 'Assistant text message', type: 'text' }],
+        role: 'assistant',
+      },
+    ]
+
+    // Act
+    const result = convertToOllamaChatMessages(prompt)
+
+    // Assert
+    const expectedResult = [
+      { content: 'System message', role: 'system' },
+      {
+        content: 'User text message',
+        images: [
+          convertUint8ArrayToBase64(new Uint8Array([0, 1, 2])),
+          convertUint8ArrayToBase64(new Uint8Array([3, 4, 5])),
+        ],
+        role: 'user',
+      },
+      { content: 'Assistant text message', role: 'assistant' },
+    ]
+    expect(result).toEqual(expectedResult)
+  })
+
+  test('should throw UnsupportedFunctionalityError for image URL', () => {
+    // Arrange
+    const prompt: LanguageModelV1Prompt = [
+      {
+        content: [
+          { image: new URL('https://example.com/image.png'), type: 'image' },
+        ],
+        role: 'user',
+      },
+    ]
+
+    // Act
+    const act = () => convertToOllamaChatMessages(prompt)
+
+    // Assert
+    expect(act).throws(UnsupportedFunctionalityError)
+  })
+
+  test('should throw Error for unsupported role', () => {
+    // Arrange
+    const prompt: LanguageModelV1Prompt = [
+      // @ts-expect-error role checking
+      { content: 'Invalid role message', role: 'invalid' },
+    ]
+
+    // Act
+    const act = () => convertToOllamaChatMessages(prompt)
+
+    // Assert
+    expect(act).throws(Error, 'Unsupported role: invalid')
+  })
+})

src/convert-to-ollama-chat-messages.ts

@@ -1,4 +1,8 @@
-import { LanguageModelV1Prompt } from '@ai-sdk/provider'
+import {
+  LanguageModelV1Prompt,
+  UnsupportedFunctionalityError,
+} from '@ai-sdk/provider'
+import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'
 
 import { OllamaChatPrompt } from '@/ollama-chat-prompt'
 
@@ -16,15 +20,29 @@ export function convertToOllamaChatMessages(
 
       case 'user': {
         messages.push({
-          content: content
-            .map((part) => {
-              switch (part.type) {
-                case 'text': {
-                  return part.text
-                }
+          ...content.reduce<{ content: string; images?: string[] }>(
+            (previous, current) => {
+              if (current.type === 'text') {
+                previous.content += current.text
+              } else if (
+                current.type === 'image' &&
+                current.image instanceof URL
+              ) {
+                throw new UnsupportedFunctionalityError({
+                  functionality: 'image-part',
+                })
+              } else if (
+                current.type === 'image' &&
+                current.image instanceof Uint8Array
+              ) {
+                previous.images = previous.images || []
+                previous.images.push(convertUint8ArrayToBase64(current.image))
               }
-            })
-            .join(''),
+
+              return previous
+            },
+            { content: '' },
+          ),
           role: 'user',
         })
         break