From e07dfa2a2cf03bf82699366647fb5a6475c2ff85 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 16:16:36 -0800 Subject: [PATCH 01/25] feat (provider/fireworks): Add image model support. --- .changeset/poor-pets-obey.md | 5 + .../ai-core/src/e2e/feature-test-suite.ts | 28 +++- examples/ai-core/src/e2e/fireworks.test.ts | 1 + .../ai-core/src/generate-image/fireworks.ts | 17 ++ packages/fireworks/src/fireworks-config.ts | 8 + .../src/fireworks-image-model.test.ts | 154 ++++++++++++++++++ .../fireworks/src/fireworks-image-model.ts | 122 ++++++++++++++ packages/fireworks/src/fireworks-provider.ts | 63 ++++--- .../src/test/binary-test-server.ts | 67 ++++++++ packages/provider-utils/src/test/index.ts | 1 + 10 files changed, 440 insertions(+), 26 deletions(-) create mode 100644 .changeset/poor-pets-obey.md create mode 100644 examples/ai-core/src/generate-image/fireworks.ts create mode 100644 packages/fireworks/src/fireworks-config.ts create mode 100644 packages/fireworks/src/fireworks-image-model.test.ts create mode 100644 packages/fireworks/src/fireworks-image-model.ts create mode 100644 packages/provider-utils/src/test/binary-test-server.ts diff --git a/.changeset/poor-pets-obey.md b/.changeset/poor-pets-obey.md new file mode 100644 index 000000000000..96cd0ecb6ca7 --- /dev/null +++ b/.changeset/poor-pets-obey.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/fireworks': patch +--- + +feat (provider/fireworks): Add image model support. diff --git a/examples/ai-core/src/e2e/feature-test-suite.ts b/examples/ai-core/src/e2e/feature-test-suite.ts index 13820446de62..34701b772334 100644 --- a/examples/ai-core/src/e2e/feature-test-suite.ts +++ b/examples/ai-core/src/e2e/feature-test-suite.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { + experimental_generateImage as generateImage, generateText, generateObject, streamText, @@ -10,12 +11,17 @@ import { } from 'ai'; import fs from 'fs'; import { describe, expect, it, vi } from 'vitest'; -import type { EmbeddingModelV1, LanguageModelV1 } from '@ai-sdk/provider'; +import type { + EmbeddingModelV1, + ImageModelV1, + LanguageModelV1, +} from '@ai-sdk/provider'; export interface ModelVariants { invalidModel?: LanguageModelV1; languageModels?: LanguageModelV1[]; embeddingModels?: EmbeddingModelV1[]; + imageModels?: ImageModelV1[]; } export interface TestSuiteOptions { @@ -369,5 +375,25 @@ export function createFeatureTestSuite({ ); } }); + + describe.each(createModelObjects(models.imageModels))( + 'Image Model: $modelId', + ({ model }) => { + it('should generate an image', async () => { + const result = await generateImage({ + model, + prompt: 'A cute cartoon cat', + }); + + // Verify we got a base64 string back + expect(result.image.base64).toBeTruthy(); + expect(typeof result.image.base64).toBe('string'); + + // Check the decoded length is reasonable (at least 10KB) + const decoded = Buffer.from(result.image.base64, 'base64'); + expect(decoded.length).toBeGreaterThan(10 * 1024); + }); + }, + ); }; } diff --git a/examples/ai-core/src/e2e/fireworks.test.ts b/examples/ai-core/src/e2e/fireworks.test.ts index af100d555956..880b062b5c47 100644 --- a/examples/ai-core/src/e2e/fireworks.test.ts +++ b/examples/ai-core/src/e2e/fireworks.test.ts @@ -21,6 +21,7 @@ createFeatureTestSuite({ embeddingModels: [ provider.textEmbeddingModel('nomic-ai/nomic-embed-text-v1.5'), ], + imageModels: [provider.image('accounts/fireworks/models/flux-1-dev-fp8')], }, timeout: 10000, customAssertions: { diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts new file mode 100644 index 000000000000..15e356750ce3 --- /dev/null +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -0,0 +1,17 @@ +import 'dotenv/config'; +import { fireworks } from '@ai-sdk/fireworks'; +import { experimental_generateImage as generateImage } from 'ai'; +import fs from 'fs'; + +async function main() { + const { image } = await generateImage({ + model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), + prompt: 'A burrito launched through a tunnel', + }); + + const filename = `image-${Date.now()}.png`; + fs.writeFileSync(filename, image.uint8Array); + console.log(`Image saved to ${filename}`); +} + +main().catch(console.error); diff --git a/packages/fireworks/src/fireworks-config.ts b/packages/fireworks/src/fireworks-config.ts new file mode 100644 index 000000000000..0d7f037a57c9 --- /dev/null +++ b/packages/fireworks/src/fireworks-config.ts @@ -0,0 +1,8 @@ +import { FetchFunction } from '@ai-sdk/provider-utils'; + +export type FireworksConfig = { + provider: string; + url: ({ path }: { path: string }) => string; + headers: () => Record; + fetch?: FetchFunction; +}; diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts new file mode 100644 index 000000000000..abeb437f30aa --- /dev/null +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -0,0 +1,154 @@ +import { APICallError } from '@ai-sdk/provider'; +import { BinaryTestServer } from '@ai-sdk/provider-utils/test'; +import { FireworksImageModel } from './fireworks-image-model'; +import { describe, it, expect } from 'vitest'; + +const prompt = 'A cute baby sea otter'; + +const model = new FireworksImageModel( + 'accounts/fireworks/models/flux-1-dev-fp8', + { + provider: 'fireworks', + baseURL: 'https://api.example.com', + headers: { 'api-key': 'test-key' }, + }, +); + +describe('FireworksImageModel', () => { + describe('doGenerate', () => { + const server = new BinaryTestServer( + 'https://api.example.com/workflows/accounts/fireworks/models/flux-1-dev-fp8/text_to_image', + ); + + server.setupTestEnvironment(); + + function prepareBinaryResponse() { + const mockImageBuffer = Buffer.from('mock-image-data'); + server.responseBody = mockImageBuffer; + } + + it('should pass the correct parameters', async () => { + prepareBinaryResponse(); + + await model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: { fireworks: { additional_param: 'value' } }, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + prompt, + additional_param: 'value', + }); + }); + + it('should pass headers', async () => { + prepareBinaryResponse(); + + const modelWithHeaders = new FireworksImageModel( + 'accounts/fireworks/models/flux-1-dev-fp8', + { + provider: 'fireworks', + baseURL: 'https://api.example.com', + headers: { + 'Custom-Provider-Header': 'provider-header-value', + }, + }, + ); + + await modelWithHeaders.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + headers: { + 'Custom-Request-Header': 'request-header-value', + }, + }); + + const requestHeaders = await server.getRequestHeaders(); + + expect(requestHeaders).toStrictEqual({ + 'content-type': 'application/json', + 'custom-provider-header': 'provider-header-value', + 'custom-request-header': 'request-header-value', + }); + }); + + it('should return base64 encoded image', async () => { + const mockImageBuffer = Buffer.from('mock-image-data'); + server.responseBody = mockImageBuffer; + + const result = await model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + }); + + expect(result.images).toHaveLength(1); + expect(result.images[0]).toBe(mockImageBuffer.toString('base64')); + }); + + it('should handle empty response body', async () => { + server.responseBody = null; + + await expect( + model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + }), + ).rejects.toThrow(APICallError); + + await expect( + model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + }), + ).rejects.toMatchObject({ + message: 'Response body is empty', + statusCode: 200, + url: 'https://api.example.com/workflows/accounts/fireworks/models/flux-1-dev-fp8/text_to_image', + requestBodyValues: { + prompt: 'A cute baby sea otter', + }, + }); + }); + + it('should handle API errors', async () => { + server.responseStatus = 400; + server.responseBody = Buffer.from('Bad Request'); + + await expect( + model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + }), + ).rejects.toThrow(APICallError); + + await expect( + model.doGenerate({ + prompt, + n: 1, + size: undefined, + providerOptions: {}, + }), + ).rejects.toMatchObject({ + message: 'Bad Request', + statusCode: 400, + url: 'https://api.example.com/workflows/accounts/fireworks/models/flux-1-dev-fp8/text_to_image', + requestBodyValues: { + prompt: 'A cute baby sea otter', + }, + responseBody: 'Bad Request', + }); + }); + }); +}); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts new file mode 100644 index 000000000000..289bb2a2b352 --- /dev/null +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -0,0 +1,122 @@ +import { APICallError, ImageModelV1 } from '@ai-sdk/provider'; +import { + Resolvable, + postJsonToApi, + combineHeaders, + resolve, + extractResponseHeaders, + ResponseHandler, +} from '@ai-sdk/provider-utils'; + +// https://fireworks.ai/models?type=image +export type FireworksImageModelId = + | 'accounts/fireworks/models/flux-1-dev-fp8' + | 'accounts/fireworks/models/flux-1-schnell-fp8' + | (string & {}); + +interface FireworksImageModelConfig { + provider: string; + baseURL: string; + headers?: Resolvable>; + fetch?: typeof fetch; +} + +const createBinaryResponseHandler = + (): ResponseHandler => + async ({ response, url, requestBodyValues }) => { + const responseHeaders = extractResponseHeaders(response); + + if (!response.body) { + throw new APICallError({ + message: 'Response body is empty', + url, + requestBodyValues, + statusCode: response.status, + responseHeaders, + responseBody: undefined, + }); + } + + try { + const buffer = await response.arrayBuffer(); + return { + responseHeaders, + value: buffer, + }; + } catch (error) { + throw new APICallError({ + message: 'Failed to read response as array buffer', + url, + requestBodyValues, + statusCode: response.status, + responseHeaders, + responseBody: undefined, + cause: error, + }); + } + }; + +const statusCodeErrorResponseHandler: ResponseHandler = async ({ + response, + url, + requestBodyValues, +}) => { + const responseHeaders = extractResponseHeaders(response); + const responseBody = await response.text(); + + return { + responseHeaders, + value: new APICallError({ + message: response.statusText, + url, + requestBodyValues: requestBodyValues as Record, + statusCode: response.status, + responseHeaders, + responseBody, + }), + }; +}; + +export class FireworksImageModel implements ImageModelV1 { + readonly specificationVersion = 'v1'; + + get provider(): string { + return this.config.provider; + } + + constructor( + readonly modelId: FireworksImageModelId, + private config: FireworksImageModelConfig, + ) {} + + async doGenerate({ + prompt, + n, + size, + providerOptions, + headers, + abortSignal, + }: Parameters[0]): Promise< + Awaited> + > { + const url = `${this.config.baseURL}/workflows/${this.modelId}/text_to_image`; + const body = { + prompt, + ...(providerOptions.fireworks ?? {}), + }; + + const { value: response } = await postJsonToApi({ + url, + headers: combineHeaders(await resolve(this.config.headers), headers), + body, + failedResponseHandler: statusCodeErrorResponseHandler, + successfulResponseHandler: createBinaryResponseHandler(), + abortSignal: abortSignal, + fetch: this.config.fetch, + }); + + return { + images: [Buffer.from(response).toString('base64')], + }; + } +} diff --git a/packages/fireworks/src/fireworks-provider.ts b/packages/fireworks/src/fireworks-provider.ts index d28ddca8922b..52363ae7b326 100644 --- a/packages/fireworks/src/fireworks-provider.ts +++ b/packages/fireworks/src/fireworks-provider.ts @@ -4,7 +4,11 @@ import { OpenAICompatibleEmbeddingModel, ProviderErrorStructure, } from '@ai-sdk/openai-compatible'; -import { EmbeddingModelV1, LanguageModelV1 } from '@ai-sdk/provider'; +import { + EmbeddingModelV1, + ImageModelV1, + LanguageModelV1, +} from '@ai-sdk/provider'; import { FetchFunction, loadApiKey, @@ -23,6 +27,11 @@ import { FireworksEmbeddingModelId, FireworksEmbeddingSettings, } from './fireworks-embedding-settings'; +import { + FireworksImageModel, + FireworksImageModelId, +} from './fireworks-image-model'; +import { FireworksConfig } from './fireworks-config'; export type FireworksErrorData = z.infer; @@ -87,14 +96,19 @@ Creates a text embedding model for text generation. modelId: FireworksEmbeddingModelId, settings?: FireworksEmbeddingSettings, ): EmbeddingModelV1; + + /** + * Creates a model for image generation. + */ + image(modelId: FireworksImageModelId): ImageModelV1; } +const defaultBaseURL = 'https://api.fireworks.ai/inference/v1'; + export function createFireworks( options: FireworksProviderSettings = {}, ): FireworksProvider { - const baseURL = withoutTrailingSlash( - options.baseURL ?? 'https://api.fireworks.ai/inference/v1', - ); + const baseURL = withoutTrailingSlash(options.baseURL ?? defaultBaseURL); const getHeaders = () => ({ Authorization: `Bearer ${loadApiKey({ apiKey: options.apiKey, @@ -104,20 +118,11 @@ export function createFireworks( ...options.headers, }); - interface CommonModelConfig { - provider: `fireworks.${string}`; - url: ({ path }: { path: string }) => string; - headers: () => Record; - fetch?: FetchFunction; - errorStructure?: ProviderErrorStructure; - } - - const getCommonModelConfig = (modelType: string): CommonModelConfig => ({ + const createCommonModelConfig = (modelType: string): FireworksConfig => ({ provider: `fireworks.${modelType}`, url: ({ path }) => `${baseURL}${path}`, headers: getHeaders, fetch: options.fetch, - errorStructure: fireworksErrorStructure, }); const createChatModel = ( @@ -125,7 +130,8 @@ export function createFireworks( settings: FireworksChatSettings = {}, ) => { return new OpenAICompatibleChatLanguageModel(modelId, settings, { - ...getCommonModelConfig('chat'), + ...createCommonModelConfig('chat'), + errorStructure: fireworksErrorStructure, defaultObjectGenerationMode: 'json', }); }; @@ -134,21 +140,27 @@ export function createFireworks( modelId: FireworksCompletionModelId, settings: FireworksCompletionSettings = {}, ) => - new OpenAICompatibleCompletionLanguageModel( - modelId, - settings, - getCommonModelConfig('completion'), - ); + new OpenAICompatibleCompletionLanguageModel(modelId, settings, { + ...createCommonModelConfig('completion'), + errorStructure: fireworksErrorStructure, + }); const createTextEmbeddingModel = ( modelId: FireworksEmbeddingModelId, settings: FireworksEmbeddingSettings = {}, ) => - new OpenAICompatibleEmbeddingModel( - modelId, - settings, - getCommonModelConfig('embedding'), - ); + new OpenAICompatibleEmbeddingModel(modelId, settings, { + ...createCommonModelConfig('embedding'), + errorStructure: fireworksErrorStructure, + }); + + const createImageModel = (modelId: FireworksImageModelId) => + new FireworksImageModel(modelId, { + ...createCommonModelConfig('image'), + // Image model urls can vary in structure across model types, so we don't + // rely on the common config url function. + baseURL: baseURL ?? defaultBaseURL, + }); const provider = ( modelId: FireworksChatModelId, @@ -158,6 +170,7 @@ export function createFireworks( provider.completionModel = createCompletionModel; provider.chatModel = createChatModel; provider.textEmbeddingModel = createTextEmbeddingModel; + provider.image = createImageModel; return provider as FireworksProvider; } diff --git a/packages/provider-utils/src/test/binary-test-server.ts b/packages/provider-utils/src/test/binary-test-server.ts new file mode 100644 index 000000000000..ead68dd94609 --- /dev/null +++ b/packages/provider-utils/src/test/binary-test-server.ts @@ -0,0 +1,67 @@ +import { HttpResponse, http } from 'msw'; +import { SetupServer, setupServer } from 'msw/node'; + +export class BinaryTestServer { + readonly server: SetupServer; + + responseBody: Buffer | null = null; + responseHeaders: Record = {}; + responseStatus = 200; + + request: Request | undefined; + + constructor(url: string) { + this.server = setupServer( + http.post(url, ({ request }) => { + this.request = request; + + if (this.responseBody === null) { + return new HttpResponse(null, { status: this.responseStatus }); + } + + return new HttpResponse(this.responseBody, { + status: this.responseStatus, + headers: this.responseHeaders, + }); + }), + ); + } + + async getRequestBodyJson() { + expect(this.request).toBeDefined(); + return JSON.parse(await this.request!.text()); + } + + async getRequestHeaders() { + expect(this.request).toBeDefined(); + const requestHeaders = this.request!.headers; + + // convert headers to object for easier comparison + const headersObject: Record = {}; + requestHeaders.forEach((value, key) => { + headersObject[key] = value; + }); + + return headersObject; + } + + async getRequestUrlSearchParams() { + expect(this.request).toBeDefined(); + return new URL(this.request!.url).searchParams; + } + + async getRequestUrl() { + expect(this.request).toBeDefined(); + return new URL(this.request!.url).toString(); + } + + setupTestEnvironment() { + beforeAll(() => this.server.listen()); + beforeEach(() => { + this.responseBody = null; + this.request = undefined; + }); + afterEach(() => this.server.resetHandlers()); + afterAll(() => this.server.close()); + } +} diff --git a/packages/provider-utils/src/test/index.ts b/packages/provider-utils/src/test/index.ts index d88f31089f86..2af39f2f44e9 100644 --- a/packages/provider-utils/src/test/index.ts +++ b/packages/provider-utils/src/test/index.ts @@ -1,3 +1,4 @@ +export * from './binary-test-server'; export * from './convert-array-to-async-iterable'; export * from './convert-array-to-readable-stream'; export * from './convert-async-iterable-to-array'; From fb89179e8e3c25cea5f9fcd705d7244315975d3c Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 17:21:01 -0800 Subject: [PATCH 02/25] avoid use of Buffer.from --- packages/fireworks/src/fireworks-image-model.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 289bb2a2b352..e152171658d4 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -115,8 +115,16 @@ export class FireworksImageModel implements ImageModelV1 { fetch: this.config.fetch, }); + // Preserve serverless runtime compatibility by avoiding the use of Buffer. + const bytes = new Uint8Array(response); + const binaryString = bytes.reduce( + (str, byte) => str + String.fromCharCode(byte), + '', + ); + const base64String = btoa(binaryString); + return { - images: [Buffer.from(response).toString('base64')], + images: [base64String], }; } } From a1fa940ad0767e333d17603376d894cf24506762 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 17:22:58 -0800 Subject: [PATCH 03/25] add provider-utils to changeset --- .changeset/poor-pets-obey.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/poor-pets-obey.md b/.changeset/poor-pets-obey.md index 96cd0ecb6ca7..cd1d577fba9f 100644 --- a/.changeset/poor-pets-obey.md +++ b/.changeset/poor-pets-obey.md @@ -1,5 +1,6 @@ --- '@ai-sdk/fireworks': patch +'@ai-sdk/provider-utils': patch --- feat (provider/fireworks): Add image model support. From 35c602e619e00b52e502a7abf78414a63ad2e602 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 17:58:06 -0800 Subject: [PATCH 04/25] use existing uint8 utils --- packages/fireworks/src/fireworks-image-model.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index e152171658d4..b803ce08d2c7 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -7,6 +7,7 @@ import { extractResponseHeaders, ResponseHandler, } from '@ai-sdk/provider-utils'; +import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; // https://fireworks.ai/models?type=image export type FireworksImageModelId = @@ -115,16 +116,8 @@ export class FireworksImageModel implements ImageModelV1 { fetch: this.config.fetch, }); - // Preserve serverless runtime compatibility by avoiding the use of Buffer. - const bytes = new Uint8Array(response); - const binaryString = bytes.reduce( - (str, byte) => str + String.fromCharCode(byte), - '', - ); - const base64String = btoa(binaryString); - return { - images: [base64String], + images: [convertUint8ArrayToBase64(new Uint8Array(response))], }; } } From f2c2c49c0cc15cca1ee90a4340ee495243dfa344 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 18:04:40 -0800 Subject: [PATCH 05/25] throw on n > 1 or size specified --- .../src/fireworks-image-model.test.ts | 31 +++++++++++++++++++ .../fireworks/src/fireworks-image-model.ts | 18 ++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts index abeb437f30aa..d19be7f87888 100644 --- a/packages/fireworks/src/fireworks-image-model.test.ts +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -2,6 +2,7 @@ import { APICallError } from '@ai-sdk/provider'; import { BinaryTestServer } from '@ai-sdk/provider-utils/test'; import { FireworksImageModel } from './fireworks-image-model'; import { describe, it, expect } from 'vitest'; +import { UnsupportedFunctionalityError } from '@ai-sdk/provider'; const prompt = 'A cute baby sea otter'; @@ -150,5 +151,35 @@ describe('FireworksImageModel', () => { responseBody: 'Bad Request', }); }); + + it('should throw error when requesting multiple images', async () => { + await expect( + model.doGenerate({ + prompt, + n: 2, + size: undefined, + providerOptions: {}, + }), + ).rejects.toThrowError( + new UnsupportedFunctionalityError({ + functionality: 'multiple images', + }), + ); + }); + + it('should throw error when specifying image size', async () => { + await expect( + model.doGenerate({ + prompt, + n: 1, + size: '512x512', + providerOptions: {}, + }), + ).rejects.toThrowError( + new UnsupportedFunctionalityError({ + functionality: 'image size', + }), + ); + }); }); }); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index b803ce08d2c7..382a06b652c4 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -1,4 +1,8 @@ -import { APICallError, ImageModelV1 } from '@ai-sdk/provider'; +import { + APICallError, + ImageModelV1, + UnsupportedFunctionalityError, +} from '@ai-sdk/provider'; import { Resolvable, postJsonToApi, @@ -100,6 +104,18 @@ export class FireworksImageModel implements ImageModelV1 { }: Parameters[0]): Promise< Awaited> > { + if (n > 1) { + throw new UnsupportedFunctionalityError({ + functionality: 'multiple images', + }); + } + + if (size) { + throw new UnsupportedFunctionalityError({ + functionality: 'image size', + }); + } + const url = `${this.config.baseURL}/workflows/${this.modelId}/text_to_image`; const body = { prompt, From c5632f35d484cb7c6fef8376baf78ad3e90be47f Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 18:21:07 -0800 Subject: [PATCH 06/25] clean up config, add options demo --- examples/ai-core/src/generate-image/fireworks.ts | 9 +++++++++ packages/fireworks/src/fireworks-image-model.test.ts | 6 +++--- packages/fireworks/src/fireworks-image-model.ts | 9 ++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts index 15e356750ce3..64315bb3a0bd 100644 --- a/examples/ai-core/src/generate-image/fireworks.ts +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -7,6 +7,15 @@ async function main() { const { image } = await generateImage({ model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), prompt: 'A burrito launched through a tunnel', + providerOptions: { + fireworks: { + // https://fireworks.ai/models/fireworks/flux-1-dev-fp8/playground + aspect_ratio: '1:1', + guidance_scale: 10, + num_inference_steps: 10, + seed: 123, + }, + }, }); const filename = `image-${Date.now()}.png`; diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts index d19be7f87888..9cb1d404092d 100644 --- a/packages/fireworks/src/fireworks-image-model.test.ts +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -11,7 +11,7 @@ const model = new FireworksImageModel( { provider: 'fireworks', baseURL: 'https://api.example.com', - headers: { 'api-key': 'test-key' }, + headers: () => ({ 'api-key': 'test-key' }), }, ); @@ -52,9 +52,9 @@ describe('FireworksImageModel', () => { { provider: 'fireworks', baseURL: 'https://api.example.com', - headers: { + headers: () => ({ 'Custom-Provider-Header': 'provider-header-value', - }, + }), }, ); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 382a06b652c4..4036222d8de2 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -4,12 +4,11 @@ import { UnsupportedFunctionalityError, } from '@ai-sdk/provider'; import { - Resolvable, postJsonToApi, combineHeaders, - resolve, extractResponseHeaders, ResponseHandler, + FetchFunction, } from '@ai-sdk/provider-utils'; import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; @@ -22,8 +21,8 @@ export type FireworksImageModelId = interface FireworksImageModelConfig { provider: string; baseURL: string; - headers?: Resolvable>; - fetch?: typeof fetch; + headers: () => Record; + fetch?: FetchFunction; } const createBinaryResponseHandler = @@ -124,7 +123,7 @@ export class FireworksImageModel implements ImageModelV1 { const { value: response } = await postJsonToApi({ url, - headers: combineHeaders(await resolve(this.config.headers), headers), + headers: combineHeaders(this.config.headers(), headers), body, failedResponseHandler: statusCodeErrorResponseHandler, successfulResponseHandler: createBinaryResponseHandler(), From a2a1e1f89bd04f33c8aa3a0ae529182188d74c65 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Fri, 3 Jan 2025 18:56:39 -0800 Subject: [PATCH 07/25] minor cleanup --- .../ai-core/src/generate-image/fireworks.ts | 2 +- packages/fireworks/src/fireworks-config.ts | 8 ------- packages/fireworks/src/fireworks-provider.ts | 24 +++++++++++-------- 3 files changed, 15 insertions(+), 19 deletions(-) delete mode 100644 packages/fireworks/src/fireworks-config.ts diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts index 64315bb3a0bd..6369d37e7390 100644 --- a/examples/ai-core/src/generate-image/fireworks.ts +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -13,7 +13,7 @@ async function main() { aspect_ratio: '1:1', guidance_scale: 10, num_inference_steps: 10, - seed: 123, + seed: 0, }, }, }); diff --git a/packages/fireworks/src/fireworks-config.ts b/packages/fireworks/src/fireworks-config.ts deleted file mode 100644 index 0d7f037a57c9..000000000000 --- a/packages/fireworks/src/fireworks-config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { FetchFunction } from '@ai-sdk/provider-utils'; - -export type FireworksConfig = { - provider: string; - url: ({ path }: { path: string }) => string; - headers: () => Record; - fetch?: FetchFunction; -}; diff --git a/packages/fireworks/src/fireworks-provider.ts b/packages/fireworks/src/fireworks-provider.ts index 52363ae7b326..92d1501a3405 100644 --- a/packages/fireworks/src/fireworks-provider.ts +++ b/packages/fireworks/src/fireworks-provider.ts @@ -31,7 +31,6 @@ import { FireworksImageModel, FireworksImageModelId, } from './fireworks-image-model'; -import { FireworksConfig } from './fireworks-config'; export type FireworksErrorData = z.infer; @@ -98,8 +97,8 @@ Creates a text embedding model for text generation. ): EmbeddingModelV1; /** - * Creates a model for image generation. - */ +Creates a model for image generation. +*/ image(modelId: FireworksImageModelId): ImageModelV1; } @@ -118,7 +117,14 @@ export function createFireworks( ...options.headers, }); - const createCommonModelConfig = (modelType: string): FireworksConfig => ({ + interface CommonModelConfig { + provider: string; + url: ({ path }: { path: string }) => string; + headers: () => Record; + fetch?: FetchFunction; + } + + const getCommonModelConfig = (modelType: string): CommonModelConfig => ({ provider: `fireworks.${modelType}`, url: ({ path }) => `${baseURL}${path}`, headers: getHeaders, @@ -130,7 +136,7 @@ export function createFireworks( settings: FireworksChatSettings = {}, ) => { return new OpenAICompatibleChatLanguageModel(modelId, settings, { - ...createCommonModelConfig('chat'), + ...getCommonModelConfig('chat'), errorStructure: fireworksErrorStructure, defaultObjectGenerationMode: 'json', }); @@ -141,7 +147,7 @@ export function createFireworks( settings: FireworksCompletionSettings = {}, ) => new OpenAICompatibleCompletionLanguageModel(modelId, settings, { - ...createCommonModelConfig('completion'), + ...getCommonModelConfig('completion'), errorStructure: fireworksErrorStructure, }); @@ -150,15 +156,13 @@ export function createFireworks( settings: FireworksEmbeddingSettings = {}, ) => new OpenAICompatibleEmbeddingModel(modelId, settings, { - ...createCommonModelConfig('embedding'), + ...getCommonModelConfig('embedding'), errorStructure: fireworksErrorStructure, }); const createImageModel = (modelId: FireworksImageModelId) => new FireworksImageModel(modelId, { - ...createCommonModelConfig('image'), - // Image model urls can vary in structure across model types, so we don't - // rely on the common config url function. + ...getCommonModelConfig('image'), baseURL: baseURL ?? defaultBaseURL, }); From 18d48e1c9c35f16ce5cabbbb5cc2d40ce89515c8 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 12:49:11 -0800 Subject: [PATCH 08/25] add image model options --- .../src/image-model/v1/image-model-v1.ts | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/provider/src/image-model/v1/image-model-v1.ts b/packages/provider/src/image-model/v1/image-model-v1.ts index 4cac218766e7..b95d96562af6 100644 --- a/packages/provider/src/image-model/v1/image-model-v1.ts +++ b/packages/provider/src/image-model/v1/image-model-v1.ts @@ -23,6 +23,12 @@ Provider-specific model ID for logging purposes. */ readonly modelId: string; + /** +Limit of how many images can be generated in a single API call. +If undefined, we will max generate one image per call. + */ + readonly maxImagesPerCall: number | undefined; + /** Generates an array of images. */ @@ -38,10 +44,26 @@ Number of images to generate. n: number; /** -Size of the images to generate. Must have the format `{width}x{height}`. +Size of the images to generate. +Must have the format `{width}x{height}`. +`undefined` will use the provider's default size. */ size: `${number}x${number}` | undefined; + /** +Aspect ratio of the images to generate. +Must have the format `{width}:{height}`. +`undefined` will use the provider's default aspect ratio. + */ + aspectRatio: `${number}:${number}` | undefined; + + /** +Seed for the image generation. +Set to `'random'` to use a random seed. +`undefined` will use the provider's default seed. + */ + seed: number | 'random' | undefined; + /** Additional provider-specific options that are passed through to the provider as body parameters. @@ -70,8 +92,12 @@ Abort signal for cancelling the operation. headers?: Record; }): PromiseLike<{ /** -Generated images as base64 encoded strings. +Generated images as base64 encoded strings or binary data. +The images should be returned without any unnecessary conversion. +If the API returns base64 encoded strings, the images should be returned +as base64 encoded strings. If the API returns binary data, the images should +be returned as binary data. */ - images: Array; + images: Array | Array; }>; }; From 84383365835d7d0c94bd52d8adc814f04bece0af Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 13:08:22 -0800 Subject: [PATCH 09/25] updating for image model arg changes --- .../ai-core/src/generate-image/fireworks.ts | 6 +- .../src/fireworks-image-model.test.ts | 77 +++++++++++++++---- .../fireworks/src/fireworks-image-model.ts | 15 ++-- 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts index 6369d37e7390..b5508c9c9515 100644 --- a/examples/ai-core/src/generate-image/fireworks.ts +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -7,13 +7,15 @@ async function main() { const { image } = await generateImage({ model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), prompt: 'A burrito launched through a tunnel', + aspectRatio: '1:1', + seed: 0, providerOptions: { fireworks: { // https://fireworks.ai/models/fireworks/flux-1-dev-fp8/playground - aspect_ratio: '1:1', + // aspect_ratio: '1:1', + // seed: 0, guidance_scale: 10, num_inference_steps: 10, - seed: 0, }, }, }); diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts index 9cb1d404092d..bb4d74f4ea84 100644 --- a/packages/fireworks/src/fireworks-image-model.test.ts +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -28,22 +28,44 @@ describe('FireworksImageModel', () => { server.responseBody = mockImageBuffer; } - it('should pass the correct parameters', async () => { + it('should pass the correct parameters including aspect ratio and seed', async () => { prepareBinaryResponse(); await model.doGenerate({ prompt, n: 1, size: undefined, + aspectRatio: '16:9', + seed: 42, providerOptions: { fireworks: { additional_param: 'value' } }, }); expect(await server.getRequestBodyJson()).toStrictEqual({ prompt, + aspect_ratio: '16:9', + seed: 42, additional_param: 'value', }); }); + it('should convert "random" seed to 0', async () => { + prepareBinaryResponse(); + + await model.doGenerate({ + prompt, + n: 1, + size: undefined, + aspectRatio: undefined, + seed: 'random', + providerOptions: {}, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + prompt, + seed: 0, + }); + }); + it('should pass headers', async () => { prepareBinaryResponse(); @@ -62,6 +84,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, headers: { 'Custom-Request-Header': 'request-header-value', @@ -77,7 +101,7 @@ describe('FireworksImageModel', () => { }); }); - it('should return base64 encoded image', async () => { + it('should return binary image data', async () => { const mockImageBuffer = Buffer.from('mock-image-data'); server.responseBody = mockImageBuffer; @@ -85,11 +109,14 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }); expect(result.images).toHaveLength(1); - expect(result.images[0]).toBe(mockImageBuffer.toString('base64')); + expect(result.images[0]).toBeInstanceOf(Uint8Array); + expect(Buffer.from(result.images[0])).toEqual(mockImageBuffer); }); it('should handle empty response body', async () => { @@ -100,6 +127,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toThrow(APICallError); @@ -109,6 +138,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toMatchObject({ @@ -130,6 +161,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toThrow(APICallError); @@ -139,6 +172,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toMatchObject({ @@ -152,19 +187,27 @@ describe('FireworksImageModel', () => { }); }); - it('should throw error when requesting multiple images', async () => { - await expect( - model.doGenerate({ - prompt, - n: 2, - size: undefined, - providerOptions: {}, - }), - ).rejects.toThrowError( - new UnsupportedFunctionalityError({ - functionality: 'multiple images', - }), - ); + it('should allow requesting multiple images', async () => { + const mockImageBuffer = Buffer.from('mock-image-data'); + server.responseBody = mockImageBuffer; + server.responseStatus = 200; + + const result = await model.doGenerate({ + prompt, + n: 2, + size: undefined, + aspectRatio: undefined, + seed: undefined, + providerOptions: {}, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + prompt, + }); + + expect(result.images).toHaveLength(1); + expect(result.images[0]).toBeInstanceOf(Uint8Array); + expect(Buffer.from(result.images[0])).toEqual(mockImageBuffer); }); it('should throw error when specifying image size', async () => { @@ -173,6 +216,8 @@ describe('FireworksImageModel', () => { prompt, n: 1, size: '512x512', + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toThrowError( diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 4036222d8de2..4f9737df3a56 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -10,7 +10,6 @@ import { ResponseHandler, FetchFunction, } from '@ai-sdk/provider-utils'; -import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; // https://fireworks.ai/models?type=image export type FireworksImageModelId = @@ -88,6 +87,8 @@ export class FireworksImageModel implements ImageModelV1 { return this.config.provider; } + readonly maxImagesPerCall = 1; + constructor( readonly modelId: FireworksImageModelId, private config: FireworksImageModelConfig, @@ -97,18 +98,14 @@ export class FireworksImageModel implements ImageModelV1 { prompt, n, size, + aspectRatio, + seed, providerOptions, headers, abortSignal, }: Parameters[0]): Promise< Awaited> > { - if (n > 1) { - throw new UnsupportedFunctionalityError({ - functionality: 'multiple images', - }); - } - if (size) { throw new UnsupportedFunctionalityError({ functionality: 'image size', @@ -118,6 +115,8 @@ export class FireworksImageModel implements ImageModelV1 { const url = `${this.config.baseURL}/workflows/${this.modelId}/text_to_image`; const body = { prompt, + aspect_ratio: aspectRatio, + seed: seed === 'random' ? 0 : seed, ...(providerOptions.fireworks ?? {}), }; @@ -132,7 +131,7 @@ export class FireworksImageModel implements ImageModelV1 { }); return { - images: [convertUint8ArrayToBase64(new Uint8Array(response))], + images: [new Uint8Array(response)], }; } } From d2070aa9f92247b2a8a00bb1791ef4c2bd609212 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 14:07:42 -0800 Subject: [PATCH 10/25] generateImage new params, handle binary image results --- .../ai-core/src/generate-image/fireworks.ts | 6 +-- .../src/generate-image/google-vertex.ts | 5 ++- .../generate-image/generate-image.test.ts | 41 ++++++++++++++++++- .../ai/core/generate-image/generate-image.ts | 38 ++++++++++++----- packages/ai/core/test/mock-image-model-v1.ts | 2 +- .../src/google-vertex-image-model.ts | 9 +++- packages/openai/src/openai-image-model.ts | 14 +++++++ 7 files changed, 97 insertions(+), 18 deletions(-) diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts index b5508c9c9515..e5472b668c31 100644 --- a/examples/ai-core/src/generate-image/fireworks.ts +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -7,13 +7,11 @@ async function main() { const { image } = await generateImage({ model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), prompt: 'A burrito launched through a tunnel', - aspectRatio: '1:1', - seed: 0, + aspectRatio: '4:3', + seed: 'random', providerOptions: { fireworks: { // https://fireworks.ai/models/fireworks/flux-1-dev-fp8/playground - // aspect_ratio: '1:1', - // seed: 0, guidance_scale: 10, num_inference_steps: 10, }, diff --git a/examples/ai-core/src/generate-image/google-vertex.ts b/examples/ai-core/src/generate-image/google-vertex.ts index 699ead66c91b..c10a1e34613e 100644 --- a/examples/ai-core/src/generate-image/google-vertex.ts +++ b/examples/ai-core/src/generate-image/google-vertex.ts @@ -7,9 +7,12 @@ async function main() { const { image } = await generateImage({ model: vertex.image('imagen-3.0-generate-001'), prompt: 'A burrito launched through a tunnel', + aspectRatio: '1:1', + seed: 'random', providerOptions: { vertex: { - aspectRatio: '16:9', + // More here as desired + // https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list }, }, }); diff --git a/packages/ai/core/generate-image/generate-image.test.ts b/packages/ai/core/generate-image/generate-image.test.ts index 51902348a410..7b815970bea9 100644 --- a/packages/ai/core/generate-image/generate-image.test.ts +++ b/packages/ai/core/generate-image/generate-image.test.ts @@ -1,7 +1,10 @@ import { ImageModelV1 } from '@ai-sdk/provider'; import { MockImageModelV1 } from '../test/mock-image-model-v1'; import { generateImage } from './generate-image'; -import { convertBase64ToUint8Array } from '@ai-sdk/provider-utils'; +import { + convertBase64ToUint8Array, + convertUint8ArrayToBase64, +} from '@ai-sdk/provider-utils'; const prompt = 'sunny day at the beach'; @@ -21,6 +24,8 @@ describe('generateImage', () => { }), prompt, size: '1024x1024', + aspectRatio: '16:9', + seed: 12345, providerOptions: { openai: { style: 'vivid' } }, headers: { 'custom-request-header': 'request-header-value' }, abortSignal, @@ -30,12 +35,46 @@ describe('generateImage', () => { n: 1, prompt, size: '1024x1024', + aspectRatio: '16:9', + seed: 12345, providerOptions: { openai: { style: 'vivid' } }, headers: { 'custom-request-header': 'request-header-value' }, abortSignal, }); }); + it('should handle base64 strings', async () => { + const base64String = 'SGVsbG8gV29ybGQ='; + const result = await generateImage({ + model: new MockImageModelV1({ + doGenerate: async () => ({ images: [base64String] }), + }), + prompt, + }); + expect(result.images).toStrictEqual([ + { + base64: base64String, + uint8Array: convertBase64ToUint8Array(base64String), + }, + ]); + }); + + it('should handle Uint8Arrays', async () => { + const uint8Array = new Uint8Array([72, 101, 108, 108, 111]); + const result = await generateImage({ + model: new MockImageModelV1({ + doGenerate: async () => ({ images: [uint8Array] }), + }), + prompt, + }); + expect(result.images).toStrictEqual([ + { + base64: convertUint8ArrayToBase64(uint8Array), + uint8Array: uint8Array, + }, + ]); + }); + it('should return generated images', async () => { const base64Images = [ 'SGVsbG8gV29ybGQ=', // "Hello World" in base64 diff --git a/packages/ai/core/generate-image/generate-image.ts b/packages/ai/core/generate-image/generate-image.ts index c1e11c5ff6da..4e9c3a21d9cb 100644 --- a/packages/ai/core/generate-image/generate-image.ts +++ b/packages/ai/core/generate-image/generate-image.ts @@ -1,7 +1,8 @@ import { ImageModelV1, JSONValue } from '@ai-sdk/provider'; -import { convertBase64ToUint8Array } from '@ai-sdk/provider-utils'; import { prepareRetries } from '../prompt/prepare-retries'; import { GeneratedImage, GenerateImageResult } from './generate-image-result'; +import { convertBase64ToUint8Array } from '@ai-sdk/provider-utils'; +import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; /** Generates images using an image model. @@ -10,6 +11,8 @@ Generates images using an image model. @param prompt - The prompt that should be used to generate the image. @param n - Number of images to generate. Default: 1. @param size - Size of the images to generate. Must have the format `{width}x{height}`. +@param aspectRatio - Aspect ratio of the images to generate. Must have the format `{width}:{height}`. +@param seed - Seed for the image generation. Set to `'random'` to use a random seed. @param providerOptions - Additional provider-specific options that are passed through to the provider as body parameters. @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2. @@ -23,6 +26,8 @@ export async function generateImage({ prompt, n, size, + aspectRatio, + seed, providerOptions, maxRetries: maxRetriesArg, abortSignal, @@ -44,10 +49,20 @@ Number of images to generate. n?: number; /** -Size of the images to generate. Must have the format `{width}x{height}`. +Size of the images to generate. Must have the format `{width}x{height}`. If not provided, the default size will be used. */ size?: `${number}x${number}`; + /** +Aspect ratio of the images to generate. Must have the format `{width}:{height}`. If not provided, the default aspect ratio will be used. + */ + aspectRatio?: `${number}:${number}`; + + /** +Seed for the image generation. Set to `'random'` to use a random seed. If not provided, the default seed will be used. + */ + seed?: number | 'random'; + /** Additional provider-specific options that are passed through to the provider as body parameters. @@ -91,23 +106,26 @@ Only applicable for HTTP-based providers. abortSignal, headers, size, + aspectRatio, + seed, providerOptions: providerOptions ?? {}, }), ); - return new DefaultGenerateImageResult({ base64Images: images }); + return new DefaultGenerateImageResult({ images }); } class DefaultGenerateImageResult implements GenerateImageResult { readonly images: Array; - constructor(options: { base64Images: Array }) { - this.images = options.base64Images.map(base64 => ({ - base64, - get uint8Array() { - return convertBase64ToUint8Array(this.base64); - }, - })); + constructor(options: { images: Array | Array }) { + this.images = options.images.map(image => { + const isUint8Array = image instanceof Uint8Array; + return { + base64: isUint8Array ? convertUint8ArrayToBase64(image) : image, + uint8Array: isUint8Array ? image : convertBase64ToUint8Array(image), + }; + }); } get image() { diff --git a/packages/ai/core/test/mock-image-model-v1.ts b/packages/ai/core/test/mock-image-model-v1.ts index 74997ad1f488..706b835925a7 100644 --- a/packages/ai/core/test/mock-image-model-v1.ts +++ b/packages/ai/core/test/mock-image-model-v1.ts @@ -3,9 +3,9 @@ import { notImplemented } from './not-implemented'; export class MockImageModelV1 implements ImageModelV1 { readonly specificationVersion = 'v1'; - readonly provider: ImageModelV1['provider']; readonly modelId: ImageModelV1['modelId']; + readonly maxImagesPerCall = 1; doGenerate: ImageModelV1['doGenerate']; diff --git a/packages/google-vertex/src/google-vertex-image-model.ts b/packages/google-vertex/src/google-vertex-image-model.ts index f93bdf6f1065..b7c00b79079e 100644 --- a/packages/google-vertex/src/google-vertex-image-model.ts +++ b/packages/google-vertex/src/google-vertex-image-model.ts @@ -25,6 +25,9 @@ interface GoogleVertexImageModelConfig { export class GoogleVertexImageModel implements ImageModelV1 { readonly specificationVersion = 'v1'; + // https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list + readonly maxImagesPerCall = 4; + get provider(): string { return this.config.provider; } @@ -38,6 +41,8 @@ export class GoogleVertexImageModel implements ImageModelV1 { prompt, n, size, + aspectRatio, + seed, providerOptions, headers, abortSignal, @@ -47,7 +52,7 @@ export class GoogleVertexImageModel implements ImageModelV1 { if (size) { throw new Error( 'Google Vertex does not support the `size` option. Use ' + - '`providerOptions.vertex.aspectRatio` instead. See ' + + '`aspectRatio` instead. See ' + 'https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio', ); } @@ -56,6 +61,8 @@ export class GoogleVertexImageModel implements ImageModelV1 { instances: [{ prompt }], parameters: { sampleCount: n, + ...(aspectRatio !== undefined ? { aspectRatio } : {}), + ...(seed !== undefined ? { seed } : {}), ...(providerOptions.vertex ?? {}), }, }; diff --git a/packages/openai/src/openai-image-model.ts b/packages/openai/src/openai-image-model.ts index b9e9b7e9d79b..9a04ce1d985e 100644 --- a/packages/openai/src/openai-image-model.ts +++ b/packages/openai/src/openai-image-model.ts @@ -16,6 +16,11 @@ export class OpenAIImageModel implements ImageModelV1 { private readonly config: OpenAIConfig; + // TODO: This must vary by model: + // https://platform.openai.com/docs/guides/images + // You can request 1 image at a time with DALL·E 3 (request more by making parallel requests) or up to 10 images at a time using DALL·E 2 with the n parameter. + readonly maxImagesPerCall = 10; + get provider(): string { return this.config.provider; } @@ -29,12 +34,21 @@ export class OpenAIImageModel implements ImageModelV1 { prompt, n, size, + aspectRatio, + seed, providerOptions, headers, abortSignal, }: Parameters[0]): Promise< Awaited> > { + if (aspectRatio !== undefined || seed !== undefined) { + throw new Error( + 'OpenAI does not support the `aspectRatio` or `seed` options. ' + + 'See https://platform.openai.com/docs/guides/images', + ); + } + const { value: response } = await postJsonToApi({ url: this.config.url({ path: '/images/generations', From 5213ac52ad3bdf019ee31d7338aeaf3e36c310ec Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 14:28:25 -0800 Subject: [PATCH 11/25] changeset for add'l package edits --- .changeset/cuddly-kiwis-guess.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/cuddly-kiwis-guess.md diff --git a/.changeset/cuddly-kiwis-guess.md b/.changeset/cuddly-kiwis-guess.md new file mode 100644 index 000000000000..d342f01d8f43 --- /dev/null +++ b/.changeset/cuddly-kiwis-guess.md @@ -0,0 +1,7 @@ +--- +'@ai-sdk/google-vertex': patch +'@ai-sdk/openai': patch +'ai': patch +--- + +feat (ai/core): expand generateImage parameters From 337208a0016eca3409be7a2ad9251b96489f42a6 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 14:32:12 -0800 Subject: [PATCH 12/25] fix test --- packages/openai/src/openai-image-model.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/openai/src/openai-image-model.test.ts b/packages/openai/src/openai-image-model.test.ts index 2f3cc6210028..74692fc83f7e 100644 --- a/packages/openai/src/openai-image-model.test.ts +++ b/packages/openai/src/openai-image-model.test.ts @@ -36,6 +36,8 @@ describe('doGenerate', () => { prompt, n: 2, size: '1024x1024', + aspectRatio: undefined, + seed: undefined, providerOptions: { openai: { style: 'vivid' } }, }); @@ -65,6 +67,8 @@ describe('doGenerate', () => { prompt, n: 2, size: '1024x1024', + aspectRatio: undefined, + seed: undefined, providerOptions: { openai: { style: 'vivid' } }, headers: { 'Custom-Request-Header': 'request-header-value', @@ -90,6 +94,8 @@ describe('doGenerate', () => { prompt, n: 2, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }); From 65fb2df7be8ec5cc093423cdea1cead220635562 Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 17:26:08 -0800 Subject: [PATCH 13/25] parameterize openai maxImagesPerCall --- packages/openai/src/openai-image-model.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/openai/src/openai-image-model.ts b/packages/openai/src/openai-image-model.ts index 9a04ce1d985e..2bc493a10b0e 100644 --- a/packages/openai/src/openai-image-model.ts +++ b/packages/openai/src/openai-image-model.ts @@ -10,16 +10,21 @@ import { openaiFailedResponseHandler } from './openai-error'; export type OpenAIImageModelId = 'dall-e-3' | 'dall-e-2' | (string & {}); +// https://platform.openai.com/docs/guides/images +const modelMaxImagesPerCall: Record = { + 'dall-e-3': 1, + 'dall-e-2': 10, +}; + export class OpenAIImageModel implements ImageModelV1 { readonly specificationVersion = 'v1'; readonly modelId: OpenAIImageModelId; private readonly config: OpenAIConfig; - // TODO: This must vary by model: - // https://platform.openai.com/docs/guides/images - // You can request 1 image at a time with DALL·E 3 (request more by making parallel requests) or up to 10 images at a time using DALL·E 2 with the n parameter. - readonly maxImagesPerCall = 10; + get maxImagesPerCall(): number { + return modelMaxImagesPerCall[this.modelId] ?? 1; + } get provider(): string { return this.config.provider; From a29e0cbf4624152bc865309de1fbc2a0da5c70ee Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 17:32:34 -0800 Subject: [PATCH 14/25] fix vertex image tests --- .../src/google-vertex-image-model.test.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/packages/google-vertex/src/google-vertex-image-model.test.ts b/packages/google-vertex/src/google-vertex-image-model.test.ts index e45ded4f6fd8..7f641cbc3d06 100644 --- a/packages/google-vertex/src/google-vertex-image-model.test.ts +++ b/packages/google-vertex/src/google-vertex-image-model.test.ts @@ -34,6 +34,8 @@ describe('GoogleVertexImageModel', () => { prompt, n: 2, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: { vertex: { aspectRatio: '1:1' } }, }); @@ -64,6 +66,8 @@ describe('GoogleVertexImageModel', () => { prompt, n: 2, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, headers: { 'Custom-Request-Header': 'request-header-value', @@ -86,6 +90,8 @@ describe('GoogleVertexImageModel', () => { prompt, n: 2, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }); @@ -103,6 +109,8 @@ describe('GoogleVertexImageModel', () => { prompt: 'test prompt', n: 1, size: '1024x1024', + aspectRatio: undefined, + seed: undefined, providerOptions: {}, }), ).rejects.toThrow(/Google Vertex does not support the `size` option./); @@ -115,6 +123,8 @@ describe('GoogleVertexImageModel', () => { prompt: 'test prompt', n: 1, size: undefined, + aspectRatio: undefined, + seed: undefined, providerOptions: { vertex: { aspectRatio: '16:9', @@ -130,5 +140,74 @@ describe('GoogleVertexImageModel', () => { }, }); }); + + it('should pass aspect ratio directly when specified', async () => { + prepareJsonResponse(); + + await model.doGenerate({ + prompt: 'test prompt', + n: 1, + size: undefined, + aspectRatio: '16:9', + seed: undefined, + providerOptions: {}, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + instances: [{ prompt: 'test prompt' }], + parameters: { + sampleCount: 1, + aspectRatio: '16:9', + }, + }); + }); + + it('should pass seed directly when specified', async () => { + prepareJsonResponse(); + + await model.doGenerate({ + prompt: 'test prompt', + n: 1, + size: undefined, + aspectRatio: undefined, + seed: 42, + providerOptions: {}, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + instances: [{ prompt: 'test prompt' }], + parameters: { + sampleCount: 1, + seed: 42, + }, + }); + }); + + it('should combine aspectRatio, seed and provider options', async () => { + prepareJsonResponse(); + + await model.doGenerate({ + prompt: 'test prompt', + n: 1, + size: undefined, + aspectRatio: '1:1', + seed: 42, + providerOptions: { + vertex: { + temperature: 0.8, + }, + }, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + instances: [{ prompt: 'test prompt' }], + parameters: { + sampleCount: 1, + aspectRatio: '1:1', + seed: 42, + temperature: 0.8, + }, + }); + }); }); }); From 82f4e2348ab841463bbd35c874defdf7e3d9154f Mon Sep 17 00:00:00 2001 From: Walter Korman Date: Mon, 6 Jan 2025 18:05:25 -0800 Subject: [PATCH 15/25] fix vertex random seed impl --- .../src/generate-image/google-vertex.ts | 2 +- .../src/google-vertex-image-model.test.ts | 20 +++++++++++++++++++ .../src/google-vertex-image-model.ts | 4 +++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/examples/ai-core/src/generate-image/google-vertex.ts b/examples/ai-core/src/generate-image/google-vertex.ts index c10a1e34613e..176613654486 100644 --- a/examples/ai-core/src/generate-image/google-vertex.ts +++ b/examples/ai-core/src/generate-image/google-vertex.ts @@ -11,8 +11,8 @@ async function main() { seed: 'random', providerOptions: { vertex: { - // More here as desired // https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list + addWatermark: false, }, }, }); diff --git a/packages/google-vertex/src/google-vertex-image-model.test.ts b/packages/google-vertex/src/google-vertex-image-model.test.ts index 7f641cbc3d06..863a15ab6c3d 100644 --- a/packages/google-vertex/src/google-vertex-image-model.test.ts +++ b/packages/google-vertex/src/google-vertex-image-model.test.ts @@ -183,6 +183,26 @@ describe('GoogleVertexImageModel', () => { }); }); + it('should omit seed when set to random', async () => { + prepareJsonResponse(); + + await model.doGenerate({ + prompt: 'test prompt', + n: 1, + size: undefined, + aspectRatio: undefined, + seed: 'random', + providerOptions: {}, + }); + + expect(await server.getRequestBodyJson()).toStrictEqual({ + instances: [{ prompt: 'test prompt' }], + parameters: { + sampleCount: 1, + }, + }); + }); + it('should combine aspectRatio, seed and provider options', async () => { prepareJsonResponse(); diff --git a/packages/google-vertex/src/google-vertex-image-model.ts b/packages/google-vertex/src/google-vertex-image-model.ts index b7c00b79079e..275fb22f62be 100644 --- a/packages/google-vertex/src/google-vertex-image-model.ts +++ b/packages/google-vertex/src/google-vertex-image-model.ts @@ -62,7 +62,9 @@ export class GoogleVertexImageModel implements ImageModelV1 { parameters: { sampleCount: n, ...(aspectRatio !== undefined ? { aspectRatio } : {}), - ...(seed !== undefined ? { seed } : {}), + // For Vertex obtaining a random seed is not explicitly documented but + // experimentation shows omitting is sufficient. + ...(seed !== undefined && seed !== 'random' ? { seed } : {}), ...(providerOptions.vertex ?? {}), }, }; From 1c12fb7364b0c17764dd858f17883182fed0987d Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 09:52:30 +0100 Subject: [PATCH 16/25] details --- .changeset/cuddly-kiwis-guess.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cuddly-kiwis-guess.md b/.changeset/cuddly-kiwis-guess.md index d342f01d8f43..b7e7f6c68828 100644 --- a/.changeset/cuddly-kiwis-guess.md +++ b/.changeset/cuddly-kiwis-guess.md @@ -4,4 +4,4 @@ 'ai': patch --- -feat (ai/core): expand generateImage parameters +feat (ai/core): add aspectRatio and seed to generateImage From 48ef98ac2430b6046a053ec4131971f9a3ef7d56 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 09:52:54 +0100 Subject: [PATCH 17/25] fix typpo --- .changeset/cuddly-kiwis-guess.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cuddly-kiwis-guess.md b/.changeset/cuddly-kiwis-guess.md index b7e7f6c68828..ecf8bb170af1 100644 --- a/.changeset/cuddly-kiwis-guess.md +++ b/.changeset/cuddly-kiwis-guess.md @@ -4,4 +4,4 @@ 'ai': patch --- -feat (ai/core): add aspectRatio and seed to generateImage +feat (ai/core): add aspectRatio and seed options to generateImage From 76e0275c3860a781cbba0a5891d3e2a31c4e77c7 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:06:53 +0100 Subject: [PATCH 18/25] remove random --- .changeset/perfect-lobsters-guess.md | 5 ++ .../ai-core/src/generate-image/fireworks.ts | 2 +- .../ai/core/generate-image/generate-image.ts | 6 +- .../src/fireworks-image-model.test.ts | 57 ++++++------------- .../fireworks/src/fireworks-image-model.ts | 23 +++++--- .../src/google-vertex-image-model.test.ts | 22 +------ .../src/google-vertex-image-model.ts | 31 +++++----- .../errors/unsupported-functionality-error.ts | 14 +++-- .../src/image-model/v1/image-model-v1.ts | 3 +- 9 files changed, 70 insertions(+), 93 deletions(-) create mode 100644 .changeset/perfect-lobsters-guess.md diff --git a/.changeset/perfect-lobsters-guess.md b/.changeset/perfect-lobsters-guess.md new file mode 100644 index 000000000000..2009b380e69c --- /dev/null +++ b/.changeset/perfect-lobsters-guess.md @@ -0,0 +1,5 @@ +--- +'@ai-sdk/provider': patch +--- + +feat (provider): add message option to UnsupportedFunctionalityError diff --git a/examples/ai-core/src/generate-image/fireworks.ts b/examples/ai-core/src/generate-image/fireworks.ts index e5472b668c31..109bd15426b0 100644 --- a/examples/ai-core/src/generate-image/fireworks.ts +++ b/examples/ai-core/src/generate-image/fireworks.ts @@ -8,7 +8,7 @@ async function main() { model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), prompt: 'A burrito launched through a tunnel', aspectRatio: '4:3', - seed: 'random', + seed: 0, // 0 is random seed for this model providerOptions: { fireworks: { // https://fireworks.ai/models/fireworks/flux-1-dev-fp8/playground diff --git a/packages/ai/core/generate-image/generate-image.ts b/packages/ai/core/generate-image/generate-image.ts index 4e9c3a21d9cb..fea4a1b616a1 100644 --- a/packages/ai/core/generate-image/generate-image.ts +++ b/packages/ai/core/generate-image/generate-image.ts @@ -12,7 +12,7 @@ Generates images using an image model. @param n - Number of images to generate. Default: 1. @param size - Size of the images to generate. Must have the format `{width}x{height}`. @param aspectRatio - Aspect ratio of the images to generate. Must have the format `{width}:{height}`. -@param seed - Seed for the image generation. Set to `'random'` to use a random seed. +@param seed - Seed for the image generation. @param providerOptions - Additional provider-specific options that are passed through to the provider as body parameters. @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2. @@ -59,9 +59,9 @@ Aspect ratio of the images to generate. Must have the format `{width}:{height}`. aspectRatio?: `${number}:${number}`; /** -Seed for the image generation. Set to `'random'` to use a random seed. If not provided, the default seed will be used. +Seed for the image generation. If not provided, the default seed will be used. */ - seed?: number | 'random'; + seed?: number; /** Additional provider-specific options that are passed through to the provider diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts index bb4d74f4ea84..ace5eb8f772c 100644 --- a/packages/fireworks/src/fireworks-image-model.test.ts +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -48,24 +48,6 @@ describe('FireworksImageModel', () => { }); }); - it('should convert "random" seed to 0', async () => { - prepareBinaryResponse(); - - await model.doGenerate({ - prompt, - n: 1, - size: undefined, - aspectRatio: undefined, - seed: 'random', - providerOptions: {}, - }); - - expect(await server.getRequestBodyJson()).toStrictEqual({ - prompt, - seed: 0, - }); - }); - it('should pass headers', async () => { prepareBinaryResponse(); @@ -187,27 +169,22 @@ describe('FireworksImageModel', () => { }); }); - it('should allow requesting multiple images', async () => { - const mockImageBuffer = Buffer.from('mock-image-data'); - server.responseBody = mockImageBuffer; - server.responseStatus = 200; - - const result = await model.doGenerate({ - prompt, - n: 2, - size: undefined, - aspectRatio: undefined, - seed: undefined, - providerOptions: {}, - }); - - expect(await server.getRequestBodyJson()).toStrictEqual({ - prompt, - }); - - expect(result.images).toHaveLength(1); - expect(result.images[0]).toBeInstanceOf(Uint8Array); - expect(Buffer.from(result.images[0])).toEqual(mockImageBuffer); + it('should throw error when requesting more than one image', async () => { + await expect( + model.doGenerate({ + prompt, + n: 2, + size: undefined, + aspectRatio: undefined, + seed: undefined, + providerOptions: {}, + }), + ).rejects.toThrowError( + new UnsupportedFunctionalityError({ + functionality: 'generate multiple images', + message: `Fireworks does not support generating more than 1 images at a time.`, + }), + ); }); it('should throw error when specifying image size', async () => { @@ -223,6 +200,8 @@ describe('FireworksImageModel', () => { ).rejects.toThrowError( new UnsupportedFunctionalityError({ functionality: 'image size', + message: + 'Fireworks does not support the `size` option. Use `aspectRatio` instead.', }), ); }); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 4f9737df3a56..7010b126834a 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -4,11 +4,11 @@ import { UnsupportedFunctionalityError, } from '@ai-sdk/provider'; import { - postJsonToApi, combineHeaders, extractResponseHeaders, - ResponseHandler, FetchFunction, + postJsonToApi, + ResponseHandler, } from '@ai-sdk/provider-utils'; // https://fireworks.ai/models?type=image @@ -106,9 +106,18 @@ export class FireworksImageModel implements ImageModelV1 { }: Parameters[0]): Promise< Awaited> > { - if (size) { + if (size != null) { throw new UnsupportedFunctionalityError({ functionality: 'image size', + message: + 'Fireworks does not support the `size` option. Use `aspectRatio` instead.', + }); + } + + if (n > 1) { + throw new UnsupportedFunctionalityError({ + functionality: 'generate multiple images', + message: `Fireworks does not support generating more than ${this.maxImagesPerCall} images at a time.`, }); } @@ -116,7 +125,7 @@ export class FireworksImageModel implements ImageModelV1 { const body = { prompt, aspect_ratio: aspectRatio, - seed: seed === 'random' ? 0 : seed, + seed, ...(providerOptions.fireworks ?? {}), }; @@ -126,12 +135,10 @@ export class FireworksImageModel implements ImageModelV1 { body, failedResponseHandler: statusCodeErrorResponseHandler, successfulResponseHandler: createBinaryResponseHandler(), - abortSignal: abortSignal, + abortSignal, fetch: this.config.fetch, }); - return { - images: [new Uint8Array(response)], - }; + return { images: [new Uint8Array(response)] }; } } diff --git a/packages/google-vertex/src/google-vertex-image-model.test.ts b/packages/google-vertex/src/google-vertex-image-model.test.ts index 863a15ab6c3d..31604e10ec42 100644 --- a/packages/google-vertex/src/google-vertex-image-model.test.ts +++ b/packages/google-vertex/src/google-vertex-image-model.test.ts @@ -1,6 +1,6 @@ import { JsonTestServer } from '@ai-sdk/provider-utils/test'; +import { describe, expect, it } from 'vitest'; import { GoogleVertexImageModel } from './google-vertex-image-model'; -import { describe, it, expect, vi } from 'vitest'; const prompt = 'A cute baby sea otter'; @@ -183,26 +183,6 @@ describe('GoogleVertexImageModel', () => { }); }); - it('should omit seed when set to random', async () => { - prepareJsonResponse(); - - await model.doGenerate({ - prompt: 'test prompt', - n: 1, - size: undefined, - aspectRatio: undefined, - seed: 'random', - providerOptions: {}, - }); - - expect(await server.getRequestBodyJson()).toStrictEqual({ - instances: [{ prompt: 'test prompt' }], - parameters: { - sampleCount: 1, - }, - }); - }); - it('should combine aspectRatio, seed and provider options', async () => { prepareJsonResponse(); diff --git a/packages/google-vertex/src/google-vertex-image-model.ts b/packages/google-vertex/src/google-vertex-image-model.ts index 275fb22f62be..1a0c63340570 100644 --- a/packages/google-vertex/src/google-vertex-image-model.ts +++ b/packages/google-vertex/src/google-vertex-image-model.ts @@ -1,9 +1,9 @@ -import { ImageModelV1, JSONValue } from '@ai-sdk/provider'; +import { ImageModelV1, UnsupportedFunctionalityError } from '@ai-sdk/provider'; import { Resolvable, - postJsonToApi, combineHeaders, createJsonResponseHandler, + postJsonToApi, resolve, } from '@ai-sdk/provider-utils'; import { z } from 'zod'; @@ -49,22 +49,27 @@ export class GoogleVertexImageModel implements ImageModelV1 { }: Parameters[0]): Promise< Awaited> > { - if (size) { - throw new Error( - 'Google Vertex does not support the `size` option. Use ' + - '`aspectRatio` instead. See ' + - 'https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio', - ); + if (size != null) { + throw new UnsupportedFunctionalityError({ + functionality: 'image size', + message: + 'Google Vertex does not support the `size` option. Use `aspectRatio` instead.', + }); + } + + if (n > this.maxImagesPerCall) { + throw new UnsupportedFunctionalityError({ + functionality: 'generate multiple images', + message: `Google Vertex does not support generating more than ${this.maxImagesPerCall} images at a time.`, + }); } const body = { instances: [{ prompt }], parameters: { sampleCount: n, - ...(aspectRatio !== undefined ? { aspectRatio } : {}), - // For Vertex obtaining a random seed is not explicitly documented but - // experimentation shows omitting is sufficient. - ...(seed !== undefined && seed !== 'random' ? { seed } : {}), + ...(aspectRatio != null ? { aspectRatio } : {}), + ...(seed != null ? { seed } : {}), ...(providerOptions.vertex ?? {}), }, }; @@ -77,7 +82,7 @@ export class GoogleVertexImageModel implements ImageModelV1 { successfulResponseHandler: createJsonResponseHandler( vertexImageResponseSchema, ), - abortSignal: abortSignal, + abortSignal, fetch: this.config.fetch, }); diff --git a/packages/provider/src/errors/unsupported-functionality-error.ts b/packages/provider/src/errors/unsupported-functionality-error.ts index 3efb22e9354c..679240d593f6 100644 --- a/packages/provider/src/errors/unsupported-functionality-error.ts +++ b/packages/provider/src/errors/unsupported-functionality-error.ts @@ -9,12 +9,14 @@ export class UnsupportedFunctionalityError extends AISDKError { readonly functionality: string; - constructor({ functionality }: { functionality: string }) { - super({ - name, - message: `'${functionality}' functionality not supported.`, - }); - + constructor({ + functionality, + message = `'${functionality}' functionality not supported.`, + }: { + functionality: string; + message?: string; + }) { + super({ name, message }); this.functionality = functionality; } diff --git a/packages/provider/src/image-model/v1/image-model-v1.ts b/packages/provider/src/image-model/v1/image-model-v1.ts index b95d96562af6..138f916f2efd 100644 --- a/packages/provider/src/image-model/v1/image-model-v1.ts +++ b/packages/provider/src/image-model/v1/image-model-v1.ts @@ -59,10 +59,9 @@ Must have the format `{width}:{height}`. /** Seed for the image generation. -Set to `'random'` to use a random seed. `undefined` will use the provider's default seed. */ - seed: number | 'random' | undefined; + seed: number | undefined; /** Additional provider-specific options that are passed through to the provider From f6638c2af09d2c1a0f1593a5f1989199eeaedfdf Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:07:38 +0100 Subject: [PATCH 19/25] rm random from example --- examples/ai-core/src/generate-image/google-vertex.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/ai-core/src/generate-image/google-vertex.ts b/examples/ai-core/src/generate-image/google-vertex.ts index 176613654486..faa8eb458a28 100644 --- a/examples/ai-core/src/generate-image/google-vertex.ts +++ b/examples/ai-core/src/generate-image/google-vertex.ts @@ -8,7 +8,6 @@ async function main() { model: vertex.image('imagen-3.0-generate-001'), prompt: 'A burrito launched through a tunnel', aspectRatio: '1:1', - seed: 'random', providerOptions: { vertex: { // https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#parameter_list From bfbb99da0ec99273ef0dd96d7f2530259e6afac9 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:16:25 +0100 Subject: [PATCH 20/25] errors --- .../src/fireworks-image-model.test.ts | 4 +-- .../fireworks/src/fireworks-image-model.ts | 8 +++--- .../src/google-vertex-image-model.test.ts | 9 ++++++- .../src/google-vertex-image-model.ts | 6 ++--- .../openai/src/openai-image-model.test.ts | 8 +++--- packages/openai/src/openai-image-model.ts | 25 ++++++++++++++----- 6 files changed, 40 insertions(+), 20 deletions(-) diff --git a/packages/fireworks/src/fireworks-image-model.test.ts b/packages/fireworks/src/fireworks-image-model.test.ts index ace5eb8f772c..119bff09b6c2 100644 --- a/packages/fireworks/src/fireworks-image-model.test.ts +++ b/packages/fireworks/src/fireworks-image-model.test.ts @@ -182,7 +182,7 @@ describe('FireworksImageModel', () => { ).rejects.toThrowError( new UnsupportedFunctionalityError({ functionality: 'generate multiple images', - message: `Fireworks does not support generating more than 1 images at a time.`, + message: `This model does not support generating more than 1 images at a time.`, }), ); }); @@ -201,7 +201,7 @@ describe('FireworksImageModel', () => { new UnsupportedFunctionalityError({ functionality: 'image size', message: - 'Fireworks does not support the `size` option. Use `aspectRatio` instead.', + 'This model does not support the `size` option. Use `aspectRatio` instead.', }), ); }); diff --git a/packages/fireworks/src/fireworks-image-model.ts b/packages/fireworks/src/fireworks-image-model.ts index 7010b126834a..8a880cb3dd39 100644 --- a/packages/fireworks/src/fireworks-image-model.ts +++ b/packages/fireworks/src/fireworks-image-model.ts @@ -110,14 +110,14 @@ export class FireworksImageModel implements ImageModelV1 { throw new UnsupportedFunctionalityError({ functionality: 'image size', message: - 'Fireworks does not support the `size` option. Use `aspectRatio` instead.', + 'This model does not support the `size` option. Use `aspectRatio` instead.', }); } - if (n > 1) { + if (n > this.maxImagesPerCall) { throw new UnsupportedFunctionalityError({ - functionality: 'generate multiple images', - message: `Fireworks does not support generating more than ${this.maxImagesPerCall} images at a time.`, + functionality: `generate more than ${this.maxImagesPerCall} images`, + message: `This model does not support generating more than ${this.maxImagesPerCall} images at a time.`, }); } diff --git a/packages/google-vertex/src/google-vertex-image-model.test.ts b/packages/google-vertex/src/google-vertex-image-model.test.ts index 31604e10ec42..f28295cfd7ac 100644 --- a/packages/google-vertex/src/google-vertex-image-model.test.ts +++ b/packages/google-vertex/src/google-vertex-image-model.test.ts @@ -1,6 +1,7 @@ import { JsonTestServer } from '@ai-sdk/provider-utils/test'; import { describe, expect, it } from 'vitest'; import { GoogleVertexImageModel } from './google-vertex-image-model'; +import { UnsupportedFunctionalityError } from '@ai-sdk/provider'; const prompt = 'A cute baby sea otter'; @@ -113,7 +114,13 @@ describe('GoogleVertexImageModel', () => { seed: undefined, providerOptions: {}, }), - ).rejects.toThrow(/Google Vertex does not support the `size` option./); + ).rejects.toThrow( + new UnsupportedFunctionalityError({ + functionality: 'image size', + message: + 'This model does not support the `size` option. Use `aspectRatio` instead.', + }), + ); }); it('sends aspect ratio in the request', async () => { diff --git a/packages/google-vertex/src/google-vertex-image-model.ts b/packages/google-vertex/src/google-vertex-image-model.ts index 1a0c63340570..507d799b82f4 100644 --- a/packages/google-vertex/src/google-vertex-image-model.ts +++ b/packages/google-vertex/src/google-vertex-image-model.ts @@ -53,14 +53,14 @@ export class GoogleVertexImageModel implements ImageModelV1 { throw new UnsupportedFunctionalityError({ functionality: 'image size', message: - 'Google Vertex does not support the `size` option. Use `aspectRatio` instead.', + 'This model does not support the `size` option. Use `aspectRatio` instead.', }); } if (n > this.maxImagesPerCall) { throw new UnsupportedFunctionalityError({ - functionality: 'generate multiple images', - message: `Google Vertex does not support generating more than ${this.maxImagesPerCall} images at a time.`, + functionality: `generate more than ${this.maxImagesPerCall} images`, + message: `This model does not support generating more than ${this.maxImagesPerCall} images at a time.`, }); } diff --git a/packages/openai/src/openai-image-model.test.ts b/packages/openai/src/openai-image-model.test.ts index 74692fc83f7e..3e70e4f67a72 100644 --- a/packages/openai/src/openai-image-model.test.ts +++ b/packages/openai/src/openai-image-model.test.ts @@ -34,7 +34,7 @@ describe('doGenerate', () => { await model.doGenerate({ prompt, - n: 2, + n: 1, size: '1024x1024', aspectRatio: undefined, seed: undefined, @@ -44,7 +44,7 @@ describe('doGenerate', () => { expect(await server.getRequestBodyJson()).toStrictEqual({ model: 'dall-e-3', prompt, - n: 2, + n: 1, size: '1024x1024', style: 'vivid', response_format: 'b64_json', @@ -65,7 +65,7 @@ describe('doGenerate', () => { await provider.image('dall-e-3').doGenerate({ prompt, - n: 2, + n: 1, size: '1024x1024', aspectRatio: undefined, seed: undefined, @@ -92,7 +92,7 @@ describe('doGenerate', () => { const result = await model.doGenerate({ prompt, - n: 2, + n: 1, size: undefined, aspectRatio: undefined, seed: undefined, diff --git a/packages/openai/src/openai-image-model.ts b/packages/openai/src/openai-image-model.ts index 2bc493a10b0e..409ce120d8ca 100644 --- a/packages/openai/src/openai-image-model.ts +++ b/packages/openai/src/openai-image-model.ts @@ -1,4 +1,4 @@ -import { ImageModelV1 } from '@ai-sdk/provider'; +import { ImageModelV1, UnsupportedFunctionalityError } from '@ai-sdk/provider'; import { combineHeaders, createJsonResponseHandler, @@ -47,11 +47,24 @@ export class OpenAIImageModel implements ImageModelV1 { }: Parameters[0]): Promise< Awaited> > { - if (aspectRatio !== undefined || seed !== undefined) { - throw new Error( - 'OpenAI does not support the `aspectRatio` or `seed` options. ' + - 'See https://platform.openai.com/docs/guides/images', - ); + if (aspectRatio != null) { + throw new UnsupportedFunctionalityError({ + functionality: 'image aspect ratio', + message: + 'This model does not support aspect ratio. Use `size` instead.', + }); + } + + if (seed != null) { + throw new UnsupportedFunctionalityError({ + functionality: 'image seed', + }); + } + + if (n > this.maxImagesPerCall) { + throw new UnsupportedFunctionalityError({ + functionality: `generate more than ${this.maxImagesPerCall} images`, + }); } const { value: response } = await postJsonToApi({ From 86ea9a7679bc0b362e468b110e0e88bc6b54f1c3 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:23:02 +0100 Subject: [PATCH 21/25] lazy conversion --- .../ai/core/generate-image/generate-image.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/ai/core/generate-image/generate-image.ts b/packages/ai/core/generate-image/generate-image.ts index fea4a1b616a1..ac8b1f086cd4 100644 --- a/packages/ai/core/generate-image/generate-image.ts +++ b/packages/ai/core/generate-image/generate-image.ts @@ -1,8 +1,10 @@ import { ImageModelV1, JSONValue } from '@ai-sdk/provider'; +import { + convertBase64ToUint8Array, + convertUint8ArrayToBase64, +} from '@ai-sdk/provider-utils'; import { prepareRetries } from '../prompt/prepare-retries'; import { GeneratedImage, GenerateImageResult } from './generate-image-result'; -import { convertBase64ToUint8Array } from '@ai-sdk/provider-utils'; -import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; /** Generates images using an image model. @@ -122,8 +124,15 @@ class DefaultGenerateImageResult implements GenerateImageResult { this.images = options.images.map(image => { const isUint8Array = image instanceof Uint8Array; return { - base64: isUint8Array ? convertUint8ArrayToBase64(image) : image, - uint8Array: isUint8Array ? image : convertBase64ToUint8Array(image), + // lazy conversion to base64 inside get to avoid unnecessary conversion overhead: + get base64() { + return isUint8Array ? convertUint8ArrayToBase64(image) : image; + }, + + // lazy conversion to uint8array inside get to avoid unnecessary conversion overhead: + get uint8Array() { + return isUint8Array ? image : convertBase64ToUint8Array(image); + }, }; }); } From ebf023f3f4b972d56805ff91eb75138c3897d1c0 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:42:07 +0100 Subject: [PATCH 22/25] doc --- .../03-ai-sdk-core/35-image-generation.mdx | 75 ++++++++++++++++--- .../01-ai-sdk-core/10-generate-image.mdx | 13 ++++ 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/content/docs/03-ai-sdk-core/35-image-generation.mdx b/content/docs/03-ai-sdk-core/35-image-generation.mdx index 771857c2c2d0..469123631de7 100644 --- a/content/docs/03-ai-sdk-core/35-image-generation.mdx +++ b/content/docs/03-ai-sdk-core/35-image-generation.mdx @@ -17,7 +17,6 @@ import { openai } from '@ai-sdk/openai'; const { image } = await generateImage({ model: openai.image('dall-e-3'), prompt: 'Santa Claus driving a Cadillac', - size: '1024x1024', }); ``` @@ -28,11 +27,50 @@ const base64 = image.base64; // base64 image data const uint8Array = image.uint8Array; // Uint8Array image data ``` +### Size and Aspect Ratio + +Depending on the model, you can either specify the size or the aspect ratio. + +##### Size + +The size is specified as a string in the format `{width}x{height}`. +Models only support a few sizes, and the supported sizes are different for each model and provider. + +```tsx highlight={"7"} +import { experimental_generateImage as generateImage } from 'ai'; +import { openai } from '@ai-sdk/openai'; + +const { image } = await generateImage({ + model: openai.image('dall-e-3'), + prompt: 'Santa Claus driving a Cadillac', + size: '1024x1024', +}); +``` + +##### Aspect Ratio + +The aspect ratio is specified as a string in the format `{width}:{height}`. +Models only support a few aspect ratios, and the supported aspect ratios are different for each model and provider. + +```tsx highlight={"7"} +import { experimental_generateImage as generateImage } from 'ai'; +import { vertex } from '@ai-sdk/google-vertex'; + +const { image } = await generateImage({ + model: vertex.image('imagen-3.0-generate-001'), + prompt: 'Santa Claus driving a Cadillac', + aspectRatio: '16:9', +}); +``` + ### Generating Multiple Images `generateImage` also supports generating multiple images at once for models that support it: -```tsx highlight={"4"} +```tsx highlight={"7"} +import { experimental_generateImage as generateImage } from 'ai'; +import { openai } from '@ai-sdk/openai'; + const { images } = await generateImage({ model: openai.image('dall-e-2'), prompt: 'Santa Claus driving a Cadillac', @@ -40,6 +78,22 @@ const { images } = await generateImage({ }); ``` +### Providing a Seed + +You can provide a seed to the `generateImage` function to control the output of the image generation process. +If supported by the model, the same seed will always produce the same image. + +```tsx highlight={"7"} +import { experimental_generateImage as generateImage } from 'ai'; +import { openai } from '@ai-sdk/openai'; + +const { image } = await generateImage({ + model: openai.image('dall-e-3'), + prompt: 'Santa Claus driving a Cadillac', + seed: 1234567890, +}); +``` + ### Provider-specific Settings Image models often have provider- or even model-specific settings. @@ -47,7 +101,10 @@ You can pass such settings to the `generateImage` function using the `providerOptions` parameter. The options for the provider (`openai` in the example below) become request body properties. -```tsx highlight={"5-7"} +```tsx highlight={"9"} +import { experimental_generateImage as generateImage } from 'ai'; +import { openai } from '@ai-sdk/openai'; + const { image } = await generateImage({ model: openai.image('dall-e-3'), prompt: 'Santa Claus driving a Cadillac', @@ -93,9 +150,9 @@ const { image } = await generateImage({ ## Image Models -| Provider | Model | Supported Sizes | -| ----------------------------------------------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | See [aspect ratios](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | See [aspect ratios](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | +| Provider | Model | Sizes | Aspect Ratios | +| ----------------------------------------------------------------------- | ------------------------------ | ------------------------------- | ------------------------- | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | use size | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | use size | diff --git a/content/docs/07-reference/01-ai-sdk-core/10-generate-image.mdx b/content/docs/07-reference/01-ai-sdk-core/10-generate-image.mdx index 52fe2bc03a90..7f2d25ae7926 100644 --- a/content/docs/07-reference/01-ai-sdk-core/10-generate-image.mdx +++ b/content/docs/07-reference/01-ai-sdk-core/10-generate-image.mdx @@ -61,6 +61,19 @@ console.log(images); description: 'Size of the images to generate. Format: `{width}x{height}`.', }, + { + name: 'aspectRatio', + type: 'string', + isOptional: true, + description: + 'Aspect ratio of the images to generate. Format: `{width}:{height}`.', + }, + { + name: 'seed', + type: 'number', + isOptional: true, + description: 'Seed for the image generation.', + }, { name: 'providerOptions', type: 'Record>', From 3160e9327ac1970bbc1033bea23ba33948bb96c0 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 10:55:46 +0100 Subject: [PATCH 23/25] provider docs --- .../01-ai-sdk-providers/01-openai.mdx | 7 ++++++- .../01-ai-sdk-providers/11-google-vertex.mdx | 19 ++++++++++--------- examples/ai-core/src/generate-image/openai.ts | 4 ---- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/content/providers/01-ai-sdk-providers/01-openai.mdx b/content/providers/01-ai-sdk-providers/01-openai.mdx index 4b1637521396..ca2d5a8459d2 100644 --- a/content/providers/01-ai-sdk-providers/01-openai.mdx +++ b/content/providers/01-ai-sdk-providers/01-openai.mdx @@ -618,9 +618,14 @@ using the `.image()` factory method. const model = openai.image('dall-e-3'); ``` + + Dall-E models do not support the `aspectRatio` parameter. Use the `size` + parameter instead. + + ### Model Capabilities -| Model | Supported Sizes | +| Model | Sizes | | ---------- | ------------------------------- | | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | | `dall-e-2` | 256x256, 512x512, 1024x1024 | diff --git a/content/providers/01-ai-sdk-providers/11-google-vertex.mdx b/content/providers/01-ai-sdk-providers/11-google-vertex.mdx index 4967d67872df..a9f62f57816c 100644 --- a/content/providers/01-ai-sdk-providers/11-google-vertex.mdx +++ b/content/providers/01-ai-sdk-providers/11-google-vertex.mdx @@ -564,8 +564,6 @@ The following optional settings are available for Google Vertex AI embedding mod You can create [Imagen](https://cloud.google.com/vertex-ai/generative-ai/docs/image/overview) models that call the [Imagen on Vertex AI API](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images) using the `.image()` factory method. For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image). -Note that Imagen does not support an explicit size parameter. Instead, size is driven by the [aspect ratio](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) of the input image. - ```ts import { vertex } from '@ai-sdk/google-vertex'; import { experimental_generateImage as generateImage } from 'ai'; @@ -573,18 +571,21 @@ import { experimental_generateImage as generateImage } from 'ai'; const { image } = await generateImage({ model: vertex.image('imagen-3.0-generate-001'), prompt: 'A futuristic cityscape at sunset', - providerOptions: { - vertex: { aspectRatio: '16:9' }, - }, + aspectRatio: '16:9', }); ``` + + Imagen models do not support the `size` parameter. Use the `aspectRatio` + parameter instead. + + #### Model Capabilities -| Model | Supported Sizes | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| `imagen-3.0-generate-001` | See [aspect ratios](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) | -| `imagen-3.0-fast-generate-001` | See [aspect ratios](https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-images#aspect-ratio) | +| Model | Aspect Ratios | +| ------------------------------ | ------------------------- | +| `imagen-3.0-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | +| `imagen-3.0-fast-generate-001` | 1:1, 3:4, 4:3, 9:16, 16:9 | ## Google Vertex Anthropic Provider Usage diff --git a/examples/ai-core/src/generate-image/openai.ts b/examples/ai-core/src/generate-image/openai.ts index 98a420e0c918..d035a33586e2 100644 --- a/examples/ai-core/src/generate-image/openai.ts +++ b/examples/ai-core/src/generate-image/openai.ts @@ -7,10 +7,6 @@ async function main() { const { image } = await generateImage({ model: openai.image('dall-e-3'), prompt: 'Santa Claus driving a Cadillac', - size: '1024x1024', - providerOptions: { - openai: { style: 'vivid', quality: 'hd' }, - }, }); const filename = `image-${Date.now()}.png`; From 5efd124017583b35304635cea40dc238f29e0dd6 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 11:05:38 +0100 Subject: [PATCH 24/25] fireworks docs --- .../01-ai-sdk-providers/26-fireworks.mdx | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/content/providers/01-ai-sdk-providers/26-fireworks.mdx b/content/providers/01-ai-sdk-providers/26-fireworks.mdx index 3a8aa783d66f..16cc989f73b1 100644 --- a/content/providers/01-ai-sdk-providers/26-fireworks.mdx +++ b/content/providers/01-ai-sdk-providers/26-fireworks.mdx @@ -87,7 +87,7 @@ const { text } = await generateText({ Fireworks language models can also be used in the `streamText` and `streamUI` functions (see [AI SDK Core](/docs/ai-sdk-core) and [AI SDK RSC](/docs/ai-sdk-rsc)). -## Completion Models +### Completion Models You can create models that call the Fireworks completions API using the `.completion()` factory method: @@ -95,17 +95,7 @@ You can create models that call the Fireworks completions API using the `.comple const model = fireworks.completion('accounts/fireworks/models/firefunction-v1'); ``` -## Embedding Models - -You can create models that call the Fireworks embeddings API using the `.textEmbeddingModel()` factory method: - -```ts -const model = fireworks.textEmbeddingModel( - 'accounts/fireworks/models/nomic-embed-text-v1', -); -``` - -## Model Capabilities +### Model Capabilities | Model | Image Input | Object Generation | Tool Usage | Tool Streaming | | ---------------------------------------------------------- | ------------------- | ------------------- | ------------------- | ------------------- | @@ -124,3 +114,41 @@ const model = fireworks.textEmbeddingModel( The table above lists popular models. Please see the [Fireworks models page](https://fireworks.ai/models) for a full list of available models. + +## Embedding Models + +You can create models that call the Fireworks embeddings API using the `.textEmbeddingModel()` factory method: + +```ts +const model = fireworks.textEmbeddingModel( + 'accounts/fireworks/models/nomic-embed-text-v1', +); +``` + +## Image Models + +You can create Fireworks image models using the `.image()` factory method. +For more on image generation with the AI SDK see [generateImage()](/docs/reference/ai-sdk-core/generate-image). + +```ts +import { fireworks } from '@ai-sdk/fireworks'; +import { experimental_generateImage as generateImage } from 'ai'; + +const { image } = await generateImage({ + model: fireworks.image('accounts/fireworks/models/flux-1-dev-fp8'), + prompt: 'A futuristic cityscape at sunset', + aspectRatio: '16:9', +}); +``` + + + Fireworks models do not support the `size` parameter. Use the `aspectRatio` + parameter instead. + + +### Model Capabilities + +| Model | Aspect Ratios | +| ---------------------------------------------- | ----------------------------------------------- | +| `accounts/fireworks/models/flux-1-dev-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | +| `accounts/fireworks/models/flux-1-schnell-fp8` | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | From b1df2aa93d2b168d0f23aaa41c6bdbea5623aba8 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Tue, 7 Jan 2025 11:06:42 +0100 Subject: [PATCH 25/25] more docs --- .../docs/03-ai-sdk-core/35-image-generation.mdx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/content/docs/03-ai-sdk-core/35-image-generation.mdx b/content/docs/03-ai-sdk-core/35-image-generation.mdx index 469123631de7..f6a157f3f97e 100644 --- a/content/docs/03-ai-sdk-core/35-image-generation.mdx +++ b/content/docs/03-ai-sdk-core/35-image-generation.mdx @@ -150,9 +150,11 @@ const { image } = await generateImage({ ## Image Models -| Provider | Model | Sizes | Aspect Ratios | -| ----------------------------------------------------------------------- | ------------------------------ | ------------------------------- | ------------------------- | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | -| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | use size | -| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | use size | +| Provider | Model | Sizes | Aspect Ratios | +| ----------------------------------------------------------------------- | ---------------------------------------------- | ------------------------------- | ----------------------------------------------- | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [Google Vertex](/providers/ai-sdk-providers/google-vertex#image-models) | `imagen-3.0-fast-generate-001` | Use aspect ratio | 1:1, 3:4, 4:3, 9:16, 16:9 | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-3` | 1024x1024, 1792x1024, 1024x1792 | use size | +| [OpenAI](/providers/ai-sdk-providers/openai#image-models) | `dall-e-2` | 256x256, 512x512, 1024x1024 | use size | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-dev-fp8` | Use aspect ratio | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 | +| [Fireworks](/providers/ai-sdk-providers/fireworks#image-models) | `accounts/fireworks/models/flux-1-schnell-fp8` | Use aspect ratio | 1:1, 2:3, 3:2, 4:5, 5:4, 16:9, 9:16, 9:21, 21:9 |