Jammy/TrueGrowth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial TrueGrowth source import

Browse Source
This commit is contained in:
Jammy
2026-07-07 09:36:36 +08:00
commit 3b6781d695
2283 changed files with 691996 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
packages/drawnix/src/utils/gemini-api/apiCalls.test.ts Normal file
View File

@@ -0,0 +1,213 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { sendMock, analyticsMock } = vi.hoisted(() => ({
sendMock: vi.fn(),
analyticsMock: {
trackAPICallStart: vi.fn(),
trackAPICallSuccess: vi.fn(),
trackAPICallFailure: vi.fn(),
},
}));
vi.mock('../../services/provider-routing', () => ({
providerTransport: {
send: (...args: unknown[]) => sendMock(...args),
},
}));
vi.mock('../posthog-analytics', () => ({
analytics: analyticsMock,
getProviderEndpointAnalytics: (baseUrl?: string | null) => {
if (!baseUrl) return null;
const url = new URL(baseUrl);
return {
origin: url.origin,
host: url.host,
protocol: url.protocol.replace(':', ''),
};
},
}));
import { callGoogleGenerateContentRaw } from './apiCalls';
describe('callGoogleGenerateContentRaw', () => {
beforeEach(() => {
sendMock.mockReset();
analyticsMock.trackAPICallStart.mockReset();
analyticsMock.trackAPICallSuccess.mockReset();
analyticsMock.trackAPICallFailure.mockReset();
sendMock.mockResolvedValue(
new Response(
JSON.stringify({
candidates: [
{
content: {
parts: [{ text: 'ok' }],
},
},
],
}),
{
status: 200,
headers: {
'Content-Type': 'application/json',
},
}
)
);
});
it('tracks http failures once', async () => {
sendMock.mockResolvedValue(
new Response(JSON.stringify({ error: { message: 'bad request' } }), {
status: 400,
headers: {
'Content-Type': 'application/json',
},
})
);
await expect(
callGoogleGenerateContentRaw(
{
apiKey: 'secret',
baseUrl: 'https://api.example.com',
modelName: 'gemini-3.1-flash-image-preview-4k',
protocol: 'google.generateContent',
authType: 'query',
},
[
{
role: 'user',
content: [{ type: 'text', text: 'draw a cat' }],
},
],
{ stream: false }
)
).rejects.toThrow('HTTP 400: bad request');
expect(analyticsMock.trackAPICallFailure).toHaveBeenCalledTimes(1);
});
it('uses provider baseUrl for analytics host', async () => {
await callGoogleGenerateContentRaw(
{
apiKey: 'secret',
baseUrl: '',
modelName: 'gemini-3.1-flash-image-preview-4k',
protocol: 'google.generateContent',
authType: 'query',
provider: {
profileId: 'provider-a',
profileName: 'Provider A',
providerType: 'gemini-compatible',
baseUrl: 'https://provider.example.com/v1beta',
apiKey: 'secret',
authType: 'query',
},
},
[
{
role: 'user',
content: [{ type: 'text', text: 'draw a cat' }],
},
],
{ stream: false }
);
expect(analyticsMock.trackAPICallStart).toHaveBeenCalledWith(
expect.objectContaining({
providerHost: 'provider.example.com',
providerOrigin: 'https://provider.example.com',
})
);
});
it('serializes inline data with google contents parts and mime_type', async () => {
await callGoogleGenerateContentRaw(
{
apiKey: 'secret',
baseUrl: 'https://api.example.com/v1',
modelName: 'gemini-3.1-pro-preview-thinking',
protocol: 'google.generateContent',
authType: 'bearer',
},
[
{
role: 'user',
content: [
{ type: 'text', text: '分析视频中的具体应用场景.' },
{ type: 'inline_data', mimeType: 'video/mp4', data: 'VIDEO_B64' },
],
},
],
{ stream: false }
);
const [, request] = sendMock.mock.calls[0];
const body = JSON.parse(String((request as { body: string }).body));
expect(body).toMatchObject({
contents: [
{
parts: [
{ text: '分析视频中的具体应用场景.' },
{
inline_data: {
mime_type: 'video/mp4',
data: 'VIDEO_B64',
},
},
],
},
],
});
expect(JSON.stringify(body)).not.toContain('mimeType');
expect(body).not.toHaveProperty('messages');
});
it('normalizes legacy generateContent paths missing the models segment', async () => {
await callGoogleGenerateContentRaw(
{
apiKey: 'secret',
baseUrl: 'https://api.example.com',
modelName: 'gemini-3.1-flash-image-preview-4k',
protocol: 'google.generateContent',
authType: 'query',
binding: {
id: 'binding',
profileId: 'provider-a',
modelId: 'gemini-3.1-flash-image-preview-4k',
operation: 'image',
protocol: 'google.generateContent',
requestSchema: 'google.generate-content.image-inline',
responseSchema: 'google.generate-content.parts',
submitPath: '/v1beta/{model}:generateContent',
baseUrlStrategy: 'trim-v1',
priority: 100,
confidence: 'high',
source: 'manual',
},
},
[
{
role: 'user',
content: [{ type: 'text', text: 'draw a cat' }],
},
],
{ stream: false }
);
expect(sendMock).toHaveBeenCalledWith(
expect.objectContaining({
baseUrl: 'https://api.example.com',
}),
expect.objectContaining({
path: '/v1beta/models/gemini-3.1-flash-image-preview-4k:generateContent',
baseUrlStrategy: 'trim-v1',
method: 'POST',
})
);
});
});

1091
packages/drawnix/src/utils/gemini-api/apiCalls.ts Normal file
View File

File diff suppressed because it is too large Load Diff

306
packages/drawnix/src/utils/gemini-api/auth.ts Normal file
View File

@@ -0,0 +1,306 @@
/**
* Gemini API 认证和配置管理
*/
import { GeminiConfig } from './types';
import {
geminiSettings,
TRUEMODEL_PROVIDER_API_KEY_URL,
} from '../settings-manager';
/**
* DOM弹窗获取API Key
*/
export function promptForApiKey(): Promise<string | null> {
if (typeof window === 'undefined') return Promise.resolve(null);
return new Promise((resolve) => {
// 创建弹窗遮罩
const overlay = document.createElement('div');
overlay.style.cssText = `
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.5);
z-index: 10000;
display: flex;
align-items: center;
justify-content: center;
`;
// 创建弹窗内容
const dialog = document.createElement('div');
dialog.style.cssText = `
background: white;
padding: 24px;
border-radius: 8px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15);
width: 400px;
max-width: 90vw;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
`;
dialog.innerHTML = `
<h3 style="margin: 0 0 16px 0; color: #333; font-size: 18px;">配置 API Key</h3>
<p style="margin: 0 0 16px 0; color: #666; line-height: 1.5;">
请输入您的 API Key，输入后将自动保存到本地存储中。
</p>
<p style="margin: 0 0 8px 0; color: #666; line-height: 1.5;">
您可以从以下地址注册并获取 API Key:
<a href="${TRUEMODEL_PROVIDER_API_KEY_URL}" target="_blank" rel="noopener noreferrer"
style="color: #ff5b00; text-decoration: none;">
${TRUEMODEL_PROVIDER_API_KEY_URL}
</a>
</p>
<input type="text" id="apiKeyInput" placeholder="请输入 API Key"
style="width: 100%; padding: 8px 12px; border: 1px solid #d9d9d9; border-radius: 4px; font-size: 14px; box-sizing: border-box; margin-bottom: 16px;" />
<div style="display: flex; gap: 8px; justify-content: flex-end;">
<button id="cancelBtn"
style="padding: 8px 16px; border: 1px solid #d9d9d9; border-radius: 4px; background: white; color: #333; cursor: pointer; font-size: 14px;">
取消
</button>
<button id="confirmBtn"
style="padding: 8px 16px; border: 1px solid #ff5b00; border-radius: 4px; background: #ff5b00; color: white; cursor: pointer; font-size: 14px;">
确认
</button>
</div>
`;
overlay.appendChild(dialog);
document.body.appendChild(overlay);
// 获取元素
const input = dialog.querySelector('#apiKeyInput') as HTMLInputElement;
const cancelBtn = dialog.querySelector('#cancelBtn') as HTMLButtonElement;
const confirmBtn = dialog.querySelector('#confirmBtn') as HTMLButtonElement;
// 阻止所有键盘事件冒泡到页面其他元素（防止输入被捕获到表格等）
const stopKeyboardPropagation = (e: KeyboardEvent) => {
e.stopPropagation();
};
overlay.addEventListener('keydown', stopKeyboardPropagation, true);
overlay.addEventListener('keyup', stopKeyboardPropagation, true);
overlay.addEventListener('keypress', stopKeyboardPropagation, true);
// 自动聚焦到输入框
setTimeout(() => input.focus(), 100);
// 清理函数
const cleanup = () => {
overlay.removeEventListener('keydown', stopKeyboardPropagation, true);
overlay.removeEventListener('keyup', stopKeyboardPropagation, true);
overlay.removeEventListener('keypress', stopKeyboardPropagation, true);
document.body.removeChild(overlay);
};
// 确认按钮点击
confirmBtn.addEventListener('click', async () => {
const apiKey = input.value.trim();
if (apiKey) {
// 更新本地设置（内部会 await syncToIndexedDB，确保 SW 能拿到最新配置）
await geminiSettings.update({ apiKey });
cleanup();
resolve(apiKey);
} else {
input.style.borderColor = '#ff4d4f';
input.focus();
}
});
// 取消按钮点击
cancelBtn.addEventListener('click', () => {
cleanup();
resolve(null);
});
// 回车键确认
input.addEventListener('keypress', (e) => {
if (e.key === 'Enter') {
confirmBtn.click();
}
});
// 点击遮罩关闭
overlay.addEventListener('click', (e) => {
if (e.target === overlay) {
cleanup();
resolve(null);
}
});
});
}
/**
* 验证并确保配置有效，如果缺少 API Key 则弹窗获取
*/
export async function validateAndEnsureConfig(
config: GeminiConfig
): Promise<GeminiConfig> {
// 检查 baseUrl
if (!config.baseUrl) {
throw new Error('Base URL 是必需的');
}
// 检查 apiKey，优先从全局设置获取
if (!config.apiKey) {
// 首先尝试从全局设置获取
const globalSettings = geminiSettings.get();
if (globalSettings.apiKey) {
// 更新原始config对象
config.apiKey = globalSettings.apiKey;
return config;
}
// 如果全局设置中也没有，则弹窗获取
const newApiKey = await promptForApiKey();
if (!newApiKey) {
throw new Error('API Key 是必需的，操作已取消');
}
// 更新原始config对象
config.apiKey = newApiKey;
return config;
}
return config;
}
/**
* 检查字符串是否是占位符格式
* 如 {key}、${key}、{{key}}、{apiKey} 等
*/
function isPlaceholder(value: string | null | undefined): boolean {
if (!value || typeof value !== 'string') return false;
// 匹配 {xxx}、${xxx}、{{xxx}} 等占位符格式
return (
/^[{$]*\{?\w+\}?\}*$/.test(value) ||
value.includes('{key}') ||
value.includes('${')
);
}
/**
* 从URL参数中获取apiKey
*/
function getApiKeyFromUrl(): string | null {
if (typeof window === 'undefined') return null;
const urlParams = new URLSearchParams(window.location.search);
const apiKey = urlParams.get('apiKey');
// 验证 apiKey 不是占位符格式
if (isPlaceholder(apiKey)) {
console.warn(
'[Auth] Detected placeholder in URL apiKey, ignoring:',
apiKey
);
return null;
}
return apiKey;
}
/**
* 从URL参数中获取settings配置
*/
function getSettingsFromUrl(): { apiKey?: string; baseUrl?: string } | null {
if (typeof window === 'undefined') return null;
const urlParams = new URLSearchParams(window.location.search);
const settingsParam = urlParams.get('settings');
if (!settingsParam) return null;
try {
const decoded = decodeURIComponent(settingsParam);
const settings = JSON.parse(decoded);
// 验证 apiKey 不是占位符格式
const apiKey = isPlaceholder(settings.key) ? undefined : settings.key;
if (settings.key && isPlaceholder(settings.key)) {
console.warn(
'[Auth] Detected placeholder in settings.key, ignoring:',
settings.key
);
}
return {
apiKey,
baseUrl: settings.url,
};
} catch (error) {
console.warn('Failed to parse settings parameter:', error);
return null;
}
}
/**
* 从URL中移除apiKey参数
*/
function removeApiKeyFromUrl(): void {
if (typeof window === 'undefined') return;
const url = new URL(window.location.href);
let hasChanges = false;
if (url.searchParams.has('apiKey')) {
url.searchParams.delete('apiKey');
hasChanges = true;
}
if (url.searchParams.has('settings')) {
url.searchParams.delete('settings');
hasChanges = true;
}
if (hasChanges) {
window.history.replaceState({}, document.title, url.toString());
}
}
/**
* 初始化设置：从URL获取settings参数并处理
*/
export function initializeSettings(): void {
// 处理settings参数
const settings = getSettingsFromUrl();
// 处理单独的apiKey参数
const apiKey = getApiKeyFromUrl();
if (settings?.apiKey || settings?.baseUrl || apiKey) {
geminiSettings.update({
...(settings?.apiKey && { apiKey: settings.apiKey }),
...(settings?.baseUrl && { baseUrl: settings.baseUrl }),
...(apiKey && { apiKey: apiKey }),
});
// Remove parameters from URL after processing
const url = new URL(window.location.href);
if (settings?.apiKey || settings?.baseUrl) {
url.searchParams.delete('settings');
}
if (apiKey) {
url.searchParams.delete('apiKey');
}
window.history.replaceState({}, '', url.toString());
}
}
// Initialize settings from URL if present
if (typeof window !== 'undefined') {
// 处理settings参数
const settings = getSettingsFromUrl();
// 处理单独的apiKey参数
const apiKey = getApiKeyFromUrl();
if (settings?.apiKey || settings?.baseUrl || apiKey) {
geminiSettings.update({
...(settings?.apiKey && { apiKey: settings.apiKey }),
...(settings?.baseUrl && { baseUrl: settings.baseUrl }),
...(apiKey && { apiKey: apiKey }),
});
removeApiKeyFromUrl();
}
}

126
packages/drawnix/src/utils/gemini-api/client.ts Normal file
View File

@@ -0,0 +1,126 @@
/**
* Gemini API 客户端类
*/
import {
GeminiConfig,
ImageInput,
VideoGenerationOptions,
GeminiMessage,
} from './types';
import { DEFAULT_CONFIG, VIDEO_DEFAULT_CONFIG } from './config';
import {
generateImageWithGemini,
generateVideoWithGemini,
chatWithGemini,
sendChatWithGemini,
} from './services';
import { geminiSettings, type ModelRef } from '../settings-manager';
/**
* Gemini API 客户端
*/
export class GeminiClient {
private isVideoClient: boolean;
constructor(isVideoClient = false) {
this.isVideoClient = isVideoClient;
}
/**
* 获取当前有效配置（直接从 localStorage 实时读取）
*/
private getEffectiveConfig(): GeminiConfig {
const globalSettings = geminiSettings.get();
if (this.isVideoClient) {
return {
...VIDEO_DEFAULT_CONFIG,
...globalSettings,
modelName:
globalSettings.videoModelName || VIDEO_DEFAULT_CONFIG.modelName,
};
} else {
return {
...DEFAULT_CONFIG,
...globalSettings,
modelName: globalSettings.imageModelName || DEFAULT_CONFIG.modelName,
};
}
}
/**
* 生成图片
*/
async generateImage(
prompt: string,
options: {
size?: string;
image?: string | string[];
response_format?: 'url' | 'b64_json';
omitDefaultResponseFormat?: boolean;
quality?: '1k' | '2k' | '4k';
count?: number;
params?: Record<string, unknown>;
model?: string; // 支持指定模型
modelRef?: ModelRef | null;
} = {}
) {
return generateImageWithGemini(prompt, options);
}
/**
* 生成视频
*/
async generateVideo(
prompt: string,
image: ImageInput | null,
options: VideoGenerationOptions = {}
) {
return generateVideoWithGemini(prompt, image, options);
}
/**
* 聊天对话（支持图片输入）
*/
async chat(
prompt: string,
images: ImageInput[] = [],
onChunk?: (content: string) => void
) {
return chatWithGemini(prompt, images, onChunk);
}
/**
* 发送多轮对话消息
* @param messages 消息列表
* @param onChunk 流式回调
* @param signal 取消信号
* @param temporaryModel 临时模型（仅在当前会话中使用，不影响全局设置）
*/
async sendChat(
messages: GeminiMessage[],
onChunk?: (content: string) => void,
signal?: AbortSignal,
temporaryModel?: string | ModelRef | null
) {
return sendChatWithGemini(messages, onChunk, signal, temporaryModel);
}
/**
* 获取当前配置
*/
getConfig(): GeminiConfig {
return this.getEffectiveConfig();
}
}
/**
* 创建默认的 Gemini 客户端实例（用于图片生成和聊天）
*/
export const defaultGeminiClient = new GeminiClient(false);
/**
* 创建视频生成专用的 Gemini 客户端实例
*/
export const videoGeminiClient = new GeminiClient(true);

41
packages/drawnix/src/utils/gemini-api/config.ts Normal file
View File

@@ -0,0 +1,41 @@
/**
* Gemini API 配置常量
*/
import { GeminiConfig } from './types';
// 默认配置
export const DEFAULT_CONFIG: Partial<GeminiConfig> = {
modelName: 'gemini-3-pro-image-preview-vip', // 图片生成和聊天的默认模型
};
// 视频生成专用配置
export const VIDEO_DEFAULT_CONFIG: Partial<GeminiConfig> = {
modelName: 'veo3', // 视频生成模型
};
/**
* 需要使用非流式调用的模型列表
* 这些模型在流式模式下可能返回不完整的响应
* 可动态扩展：添加模型名称即可
*/
export const NON_STREAM_MODELS: string[] = [
'seedream-4-0-250828',
'seedream-v4',
'doubao-seedream-4-5-251128',
'doubao-seedream-5-0-260128',
'gemini-3-pro-image-preview-async',
'gemini-3-pro-image-preview-2k-async',
'gemini-3-pro-image-preview-4k-async',
];
/**
* 检查模型是否需要使用非流式调用
*/
export function shouldUseNonStreamMode(modelName: string): boolean {
if (!modelName) return false;
const lowerModelName = modelName.toLowerCase();
return NON_STREAM_MODELS.some((m) =>
lowerModelName.includes(m.toLowerCase())
);
}

27
packages/drawnix/src/utils/gemini-api/index.ts Normal file
View File

@@ -0,0 +1,27 @@
/**
* Gemini API 模块统一导出
*/
// 导出类型
export * from './types';
// 导出配置
export * from './config';
// 导出工具函数
export * from './utils';
// 导出API调用函数
export * from './apiCalls';
// 导出服务函数
export * from './services';
// 导出带日志的调用包装
export * from './logged-calls';
// 导出认证相关
export * from './auth';
// 导出客户端
export { GeminiClient, defaultGeminiClient, videoGeminiClient } from './client';

60
packages/drawnix/src/utils/gemini-api/logged-calls.ts Normal file
View File

@@ -0,0 +1,60 @@
/**
* 带 LLM API 日志的 callGoogleGenerateContentRaw 包装
*
* 供直接调用 callGoogleGenerateContentRaw 的分析服务使用。
* 不修改 callGoogleGenerateContentRaw 本身，避免与已有手动日志产生双重记录。
*/
import type { GeminiConfig, GeminiMessage, GeminiResponse } from './types';
import { callGoogleGenerateContentRaw } from './apiCalls';
import {
startLLMApiLog,
completeLLMApiLog,
failLLMApiLog,
type LLMApiLog,
} from '../../services/media-executor/llm-api-logger';
import { truncate } from '@aitu/utils';
export async function callGoogleGenerateContentWithLog(
config: GeminiConfig,
messages: GeminiMessage[],
options: {
stream: boolean;
onChunk?: (content: string) => void;
signal?: AbortSignal;
generationConfig?: Record<string, unknown>;
},
logMeta: {
taskType: LLMApiLog['taskType'];
prompt?: string;
taskId?: string;
}
): Promise<GeminiResponse> {
const model = config.modelName || 'unknown';
const logId = startLLMApiLog({
endpoint: `generateContent${options.stream ? '?alt=sse' : ''}`,
model,
taskType: logMeta.taskType,
prompt: logMeta.prompt,
taskId: logMeta.taskId,
});
const startTime = Date.now();
try {
const response = await callGoogleGenerateContentRaw(config, messages, options);
const resultText = response.choices?.[0]?.message?.content;
completeLLMApiLog(logId, {
httpStatus: 200,
duration: Date.now() - startTime,
resultType: 'text',
resultText: resultText ? truncate(resultText, 1000) : undefined,
});
return response;
} catch (error: any) {
failLLMApiLog(logId, {
duration: Date.now() - startTime,
errorMessage: error.message || String(error),
});
throw error;
}
}

101
packages/drawnix/src/utils/gemini-api/message-utils.test.ts Normal file
View File

@@ -0,0 +1,101 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
appendImagePartsToLastUserMessage,
buildImagePartsFromChatAttachments,
normalizeImageUrlForMultimodalInput,
countImageParts,
} from './message-utils';
import type { GeminiMessage } from './types';
afterEach(() => {
vi.restoreAllMocks();
});
describe('message-utils', () => {
it('builds image parts from chat attachments that already use data urls', async () => {
const imageParts = await buildImagePartsFromChatAttachments([
{
id: 'att-1',
name: 'example.png',
type: 'image/png',
size: 0,
data: 'data:image/png;base64,ZmFrZQ==',
isBlob: false,
},
{
id: 'att-2',
name: 'notes.txt',
type: 'text/plain',
size: 0,
data: 'hello',
isBlob: false,
},
]);
expect(imageParts).toHaveLength(1);
expect(imageParts[0]).toEqual({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,ZmFrZQ==',
},
});
});
it('appends image parts to the last user message only', () => {
const messages: GeminiMessage[] = [
{
role: 'system',
content: [{ type: 'text', text: 'system' }],
},
{
role: 'user',
content: [{ type: 'text', text: 'first user' }],
},
{
role: 'assistant',
content: [{ type: 'text', text: 'assistant' }],
},
{
role: 'user',
content: [{ type: 'text', text: 'last user' }],
},
];
const updatedMessages = appendImagePartsToLastUserMessage(messages, [
{
type: 'image_url',
image_url: {
url: 'data:image/png;base64,ZmFrZQ==',
},
},
]);
expect(countImageParts(updatedMessages)).toBe(1);
expect(updatedMessages[3].content).toEqual([
{ type: 'text', text: 'last user' },
{
type: 'image_url',
image_url: {
url: 'data:image/png;base64,ZmFrZQ==',
},
},
]);
expect(updatedMessages[1].content).toEqual([
{ type: 'text', text: 'first user' },
]);
});
it('converts local cached image paths into data urls before sending', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(new Blob(['fake-image'], { type: 'image/png' }), {
status: 200,
}) as Response
);
const normalized = await normalizeImageUrlForMultimodalInput(
'/__aitu_cache__/image/example.png'
);
expect(normalized.startsWith('data:image/png;base64,')).toBe(true);
});
});

251
packages/drawnix/src/utils/gemini-api/message-utils.ts Normal file
View File

@@ -0,0 +1,251 @@
import type { Attachment } from '../../types/chat.types';
import type { GeminiMessage, GeminiMessagePart } from './types';
import { fileToBase64 } from './utils';
export type MultimodalAttachmentInput = File | Attachment;
function blobToDataUrl(blob: Blob): Promise<string> {
if (typeof FileReader === 'undefined') {
return blob.arrayBuffer().then((buffer) => {
const bytes = new Uint8Array(buffer);
const mimeType = blob.type || 'application/octet-stream';
let base64: string;
if (typeof Buffer !== 'undefined') {
base64 = Buffer.from(bytes).toString('base64');
} else {
let binary = '';
bytes.forEach((byte) => {
binary += String.fromCharCode(byte);
});
base64 = btoa(binary);
}
return `data:${mimeType};base64,${base64}`;
});
}
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject(new Error('读取图片数据失败'));
reader.readAsDataURL(blob);
});
}
function isImageMimeType(mimeType?: string): boolean {
return Boolean(mimeType && mimeType.toLowerCase().startsWith('image/'));
}
function isFile(value: unknown): value is File {
return typeof File !== 'undefined' && value instanceof File;
}
function isAbsoluteHttpUrl(url: string): boolean {
return /^https?:\/\//i.test(url);
}
function isLocalRelativeImageUrl(url: string): boolean {
if (
url.startsWith('/__aitu_cache__/') ||
url.startsWith('/asset-library/') ||
url.startsWith('/') ||
url.startsWith('./') ||
url.startsWith('../')
) {
return true;
}
if (!isAbsoluteHttpUrl(url) && !/^[a-z][a-z0-9+.-]*:/i.test(url)) {
return true;
}
if (typeof globalThis.location?.origin === 'string') {
return url.startsWith(globalThis.location.origin);
}
return false;
}
export async function normalizeImageUrlForMultimodalInput(
url: string
): Promise<string> {
const normalizedUrl = url.trim();
if (!normalizedUrl) {
throw new Error('图片地址不能为空');
}
if (normalizedUrl.startsWith('data:image/')) {
return normalizedUrl;
}
if (normalizedUrl.startsWith('blob:') || isLocalRelativeImageUrl(normalizedUrl)) {
const response = await fetch(normalizedUrl);
if (!response.ok) {
throw new Error(`读取本地图片失败: ${response.status}`);
}
const blob = await response.blob();
return blobToDataUrl(blob);
}
return normalizedUrl;
}
export async function buildImagePartsFromUrls(
urls: string[],
maxImageCount: number = Number.POSITIVE_INFINITY
): Promise<GeminiMessagePart[]> {
const selectedUrls = urls.filter(Boolean).slice(0, maxImageCount);
return Promise.all(
selectedUrls.map(async (url) => ({
type: 'image_url' as const,
image_url: {
url: await normalizeImageUrlForMultimodalInput(url),
},
}))
);
}
export async function buildImagePartsFromFiles(
files: File[],
maxImageCount: number = Number.POSITIVE_INFINITY
): Promise<GeminiMessagePart[]> {
const imageFiles = files
.filter((file) => isImageMimeType(file.type))
.slice(0, maxImageCount);
return Promise.all(
imageFiles.map(async (file) => ({
type: 'image_url' as const,
image_url: {
url: await fileToBase64(file),
},
}))
);
}
export function countImageAttachmentInputs(
attachments: MultimodalAttachmentInput[] = []
): number {
return attachments.filter((attachment) =>
isFile(attachment)
? isImageMimeType(attachment.type)
: isImageMimeType(attachment.type)
).length;
}
export async function buildImagePartsFromAttachmentInputs(
attachments: MultimodalAttachmentInput[] = [],
maxImageCount: number = Number.POSITIVE_INFINITY
): Promise<GeminiMessagePart[]> {
const imageAttachments = attachments
.filter((attachment) =>
isFile(attachment)
? isImageMimeType(attachment.type)
: isImageMimeType(attachment.type)
)
.slice(0, maxImageCount);
return Promise.all(
imageAttachments.map(async (attachment) => ({
type: 'image_url' as const,
image_url: {
url: isFile(attachment)
? await fileToBase64(attachment)
: await normalizeImageUrlForMultimodalInput(attachment.data),
},
}))
);
}
export async function buildImagePartsFromChatAttachments(
attachments: Attachment[] = [],
maxImageCount: number = Number.POSITIVE_INFINITY
): Promise<GeminiMessagePart[]> {
const imageAttachments = attachments
.filter((attachment) => isImageMimeType(attachment.type))
.slice(0, maxImageCount);
return Promise.all(
imageAttachments.map(async (attachment) => ({
type: 'image_url' as const,
image_url: {
url: await normalizeImageUrlForMultimodalInput(attachment.data),
},
}))
);
}
export function countImageParts(messages: GeminiMessage[]): number {
return messages.reduce(
(total, message) =>
total +
message.content.filter((part) => part.type === 'image_url').length,
0
);
}
export function hasImageParts(messages: GeminiMessage[]): boolean {
return countImageParts(messages) > 0;
}
export function appendImagePartsToLastUserMessage(
messages: GeminiMessage[],
imageParts: GeminiMessagePart[]
): GeminiMessage[] {
if (imageParts.length === 0) {
return messages;
}
const lastUserMessageIndex = [...messages]
.map((message, index) => ({ message, index }))
.reverse()
.find(({ message }) => message.role === 'user')?.index;
if (lastUserMessageIndex == null) {
return [
...messages,
{
role: 'user',
content: imageParts,
},
];
}
return messages.map((message, index) =>
index === lastUserMessageIndex
? {
...message,
content: [...message.content, ...imageParts],
}
: message
);
}
/**
* 将 File 对象转换为 inline_data 消息部分
* 支持视频、音频、PDF 等任意文件类型
*/
export async function buildInlineDataPart(
file: File
): Promise<GeminiMessagePart> {
const dataUrl = await blobToDataUrl(file);
const commaIndex = dataUrl.indexOf(',');
return {
type: 'inline_data',
mimeType: file.type || 'application/octet-stream',
data: commaIndex >= 0 ? dataUrl.slice(commaIndex + 1) : dataUrl,
};
}
/**
* 将远程文件 URI（如 YouTube URL）转换为 file_uri 消息部分
*/
export function buildFileUriPart(uri: string): GeminiMessagePart {
return {
type: 'file_uri',
fileUri: uri,
};
}

164
packages/drawnix/src/utils/gemini-api/services.test.ts Normal file
View File

@@ -0,0 +1,164 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
import { normalizeAspectRatio } from './services';
afterEach(() => {
vi.restoreAllMocks();
vi.resetModules();
});
describe('normalizeAspectRatio', () => {
it('preserves canonical Gemini aspect ratio enums', () => {
expect(normalizeAspectRatio('21x9')).toBe('21:9');
expect(normalizeAspectRatio('16x9')).toBe('16:9');
expect(normalizeAspectRatio('9x16')).toBe('9:16');
});
it('normalizes pixel sizes to reduced aspect ratios', () => {
expect(normalizeAspectRatio('1280x720')).toBe('16:9');
expect(normalizeAspectRatio('1024x1792')).toBe('4:7');
});
it('returns ratio strings as-is', () => {
expect(normalizeAspectRatio('21:9')).toBe('21:9');
expect(normalizeAspectRatio('auto')).toBeUndefined();
});
});
describe('generateImageWithGemini local provider payload', () => {
it('sends no auth header and forwards local image extension params', async () => {
const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => ({
ok: true,
json: async () => ({
data: [{ url: 'http://127.0.0.1:8080/output.png' }],
}),
text: async () => '',
}));
vi.stubGlobal('fetch', fetchMock);
vi.doMock('../settings-manager', () => ({
settingsManager: {
waitForInitialization: vi.fn(async () => {}),
},
providerPricingCacheSettings: {
get: () => [],
update: vi.fn(async () => {}),
addListener: vi.fn(),
removeListener: vi.fn(),
},
providerProfilesSettings: {
get: () => [
{
id: 'local-image',
name: 'Local Image',
providerType: 'openai-compatible',
baseUrl: 'http://127.0.0.1:8080/v1',
apiKey: '',
authType: 'none',
imageApiCompatibility: 'openai-compatible-basic',
enabled: true,
capabilities: {
supportsModelsEndpoint: true,
supportsText: false,
supportsImage: true,
supportsVideo: false,
supportsAudio: false,
supportsTools: false,
},
},
],
},
providerCatalogsSettings: {
get: () => [
{
profileId: 'local-image',
discoveredAt: Date.now(),
discoveredModels: [
{
id: 'flux.1-schnell-local',
label: 'FLUX.1 schnell local',
type: 'image',
vendor: 'FLUX',
},
],
selectedModelIds: ['flux.1-schnell-local'],
},
],
},
createModelRef: vi.fn((profileId, modelId) =>
profileId || modelId ? { profileId, modelId } : null
),
resolveInvocationRoute: vi.fn(() => ({
routeType: 'image',
modelId: 'flux.1-schnell-local',
profileId: 'local-image',
profileName: 'Local Image',
providerType: 'openai-compatible',
baseUrl: 'http://127.0.0.1:8080/v1',
apiKey: '',
source: 'preset',
})),
resolveInvocationPlanFromRoute: vi.fn(() => ({
provider: {
profileId: 'local-image',
profileName: 'Local Image',
providerType: 'openai-compatible',
baseUrl: 'http://127.0.0.1:8080/v1',
apiKey: '',
authType: 'none',
},
modelRef: {
profileId: 'local-image',
modelId: 'flux.1-schnell-local',
},
binding: {
id: 'local-image-binding',
profileId: 'local-image',
modelId: 'flux.1-schnell-local',
operation: 'image',
protocol: 'openai.images.generations',
requestSchema: 'openai.image.basic-json',
responseSchema: 'openai.image.data',
submitPath: '/images/generations',
priority: 320,
confidence: 'high',
source: 'template',
},
})),
}));
vi.doMock('./auth', () => ({
validateAndEnsureConfig: vi.fn(async (config) => config),
}));
vi.doMock('../../services/media-executor/llm-api-logger', () => ({
startLLMApiLog: vi.fn(() => 'log-id'),
completeLLMApiLog: vi.fn(),
failLLMApiLog: vi.fn(),
}));
const { generateImageWithGemini } = await import('./services');
await generateImageWithGemini('local prompt', {
model: 'flux.1-schnell-local',
size: '1024x1024',
params: {
negative_prompt: 'blur',
steps: 4,
cfg_scale: 1.5,
seed: 42,
},
});
const init = fetchMock.mock.calls[0]?.[1] as RequestInit;
expect(init.headers).not.toMatchObject({
Authorization: expect.any(String),
});
expect(JSON.parse(String(init.body))).toMatchObject({
model: 'flux.1-schnell-local',
prompt: 'Generate an image: local prompt',
size: '1024x1024',
negative_prompt: 'blur',
steps: 4,
cfg_scale: 1.5,
seed: 42,
});
});
});

681
packages/drawnix/src/utils/gemini-api/services.ts Normal file
View File

@@ -0,0 +1,681 @@
/**
* Gemini API 服务函数
*/
import {
ImageInput,
GeminiMessage,
VideoGenerationOptions,
ProcessedContent,
GeminiResponse,
} from './types';
import {
DEFAULT_CONFIG,
VIDEO_DEFAULT_CONFIG,
shouldUseNonStreamMode,
} from './config';
import { prepareImageData, processMixedContent } from './utils';
import {
callApiWithRetry,
callGoogleGenerateContentRaw,
callApiStreamRaw,
callVideoApiStreamRaw,
} from './apiCalls';
import {
resolveInvocationRoute,
settingsManager,
type ModelRef,
} from '../settings-manager';
import {
providerTransport,
resolveInvocationPlanFromRoute,
type ResolvedProviderContext,
type ProviderAuthStrategy,
} from '../../services/provider-routing';
import { IMAGE_GENERATION_TIMEOUT_MS } from '../../constants/TASK_CONSTANTS';
import { validateAndEnsureConfig } from './auth';
import {
startLLMApiLog,
completeLLMApiLog,
failLLMApiLog,
type LLMApiLog,
} from '../../services/media-executor/llm-api-logger';
import { normalizeImageDataUrl, truncate } from '@aitu/utils';
function inferAuthTypeFromRoute(
route: ReturnType<typeof resolveInvocationRoute>
): ProviderAuthStrategy {
return 'bearer';
}
function buildProviderContextFromConfig(config: {
apiKey: string;
baseUrl: string;
authType?: ProviderAuthStrategy;
providerType?: string;
extraHeaders?: Record<string, string>;
provider?: ResolvedProviderContext | null;
}): ResolvedProviderContext {
return (
config.provider || {
profileId: 'runtime',
profileName: 'Runtime',
providerType: config.providerType || 'custom',
baseUrl: config.baseUrl,
apiKey: config.apiKey,
authType: config.authType || 'bearer',
extraHeaders: config.extraHeaders,
}
);
}
function buildRuntimeConfig(
routeType: 'text' | 'image' | 'video',
routeModel: string | ModelRef | null | undefined,
fallbackModelName: string,
defaults:
| Partial<typeof DEFAULT_CONFIG>
| Partial<typeof VIDEO_DEFAULT_CONFIG>
) {
const route = resolveInvocationRoute(routeType, routeModel);
const plan = resolveInvocationPlanFromRoute(routeType, routeModel);
return {
route,
plan,
config: {
...defaults,
apiKey: route.apiKey,
baseUrl: route.baseUrl,
modelName: route.modelId || fallbackModelName,
authType: plan?.provider.authType || inferAuthTypeFromRoute(route),
providerType:
plan?.provider.providerType || route.providerType || 'custom',
extraHeaders: plan?.provider.extraHeaders,
protocol: plan?.binding.protocol || null,
binding: plan?.binding || null,
provider: plan?.provider || null,
},
};
}
export function normalizeAspectRatio(size?: string): string | undefined {
if (!size || size === 'auto') {
return undefined;
}
const normalizedSize = size.trim();
const canonicalAspectRatioMap: Record<string, string> = {
'1x1': '1:1',
'2x3': '2:3',
'3x2': '3:2',
'3x4': '3:4',
'4x3': '4:3',
'4x5': '4:5',
'5x4': '5:4',
'9x16': '9:16',
'16x9': '16:9',
'21x9': '21:9',
};
const normalizedLower = normalizedSize.toLowerCase();
if (canonicalAspectRatioMap[normalizedLower]) {
return canonicalAspectRatioMap[normalizedLower];
}
if (size.includes(':')) {
return normalizedSize;
}
if (!size.includes('x')) {
return undefined;
}
const [wRaw, hRaw] = size.split('x');
const width = Number(wRaw);
const height = Number(hRaw);
if (!width || !height) {
return undefined;
}
const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b));
const divisor = gcd(width, height);
return `${width / divisor}:${height / divisor}`;
}
function normalizeGoogleImageSize(
quality?: '1k' | '2k' | '4k'
): '1K' | '2K' | '4K' | undefined {
if (!quality) {
return undefined;
}
return quality.toUpperCase() as '1K' | '2K' | '4K';
}
function normalizeGoogleImageResult(content: string): {
data: Array<{ b64_json?: string; url?: string }>;
} {
const base64Matches = Array.from(
content.matchAll(/data:([^;]+);base64,([A-Za-z0-9+/=]+)/g)
);
const urlMatches = Array.from(content.matchAll(/https?:\/\/[^\s<>"')]+/g));
return {
data: [
...base64Matches.map((match) => ({
b64_json: match[2],
})),
...urlMatches.map((match) => ({
url: match[0],
})),
],
};
}
/**
* 调用 Gemini API 进行图像生成
* 使用专用的 /v1/images/generations 接口
* 不再依赖 SW 任务队列，直接在主线程 fetch
*/
export async function generateImageWithGemini(
prompt: string,
options: {
size?: string;
image?: string | string[]; // 支持单图或多图
response_format?: 'url' | 'b64_json';
omitDefaultResponseFormat?: boolean;
quality?: '1k' | '2k' | '4k';
count?: number;
params?: Record<string, unknown>;
model?: string; // 支持指定模型
modelRef?: ModelRef | null;
} = {}
): Promise<any> {
// 等待设置管理器初始化完成
await settingsManager.waitForInitialization();
const routeModel = options.modelRef || options.model;
const route = resolveInvocationRoute('image', routeModel);
const modelName =
route.modelId ||
DEFAULT_CONFIG.modelName ||
'gemini-3-pro-image-preview-vip';
return generateImageDirect(prompt, options, modelName, routeModel);
}
// generateImageViaSW 已移除 - 不再依赖 SW 任务队列
/**
* 使用 fetch 生成图片
*/
async function generateImageDirect(
prompt: string,
options: {
size?: string;
image?: string | string[];
response_format?: 'url' | 'b64_json';
omitDefaultResponseFormat?: boolean;
quality?: '1k' | '2k' | '4k';
count?: number;
params?: Record<string, unknown>;
model?: string;
modelRef?: ModelRef | null;
},
modelName: string,
routeModel?: string | ModelRef | null
): Promise<any> {
const { config: runtimeConfig } = buildRuntimeConfig(
'image',
routeModel || modelName,
modelName,
DEFAULT_CONFIG
);
const startTime = Date.now();
// 开始记录 LLM API 调用（降级模式）
const referenceImages = options.image
? Array.isArray(options.image)
? options.image
: [options.image]
: undefined;
const logId = startLLMApiLog({
endpoint: '/images/generations',
model: modelName,
taskType: 'image',
prompt,
hasReferenceImages: referenceImages && referenceImages.length > 0,
referenceImageCount: referenceImages?.length,
});
try {
const validatedConfig = await validateAndEnsureConfig(runtimeConfig);
if (validatedConfig.protocol === 'google.generateContent') {
const content = [
{
type: 'text' as const,
text: prompt,
},
...(options.image
? Array.isArray(options.image)
? options.image
: [options.image]
: []
).map((url) => ({
type: 'image_url' as const,
image_url: {
url,
},
})),
];
const response = await callGoogleGenerateContentRaw(
validatedConfig,
[
{
role: 'user',
content,
},
],
{
stream: false,
generationConfig: {
responseModalities: ['IMAGE'],
imageConfig: {
...(normalizeAspectRatio(options.size)
? { aspectRatio: normalizeAspectRatio(options.size) }
: {}),
...(normalizeGoogleImageSize(options.quality)
? { imageSize: normalizeGoogleImageSize(options.quality) }
: {}),
},
},
}
);
const duration = Date.now() - startTime;
const normalizedResult = normalizeGoogleImageResult(
response.choices[0]?.message?.content || ''
);
completeLLMApiLog(logId, {
httpStatus: 200,
duration,
resultType: 'image',
resultCount: normalizedResult.data.length || 1,
});
return normalizedResult;
}
const headers = {
'Content-Type': 'application/json',
};
// 构建请求体 - 强调生成图片
const enhancedPrompt = `Generate an image: ${prompt}`;
const data: any = {
model: validatedConfig.modelName || 'gemini-3-pro-image-preview-vip',
prompt: enhancedPrompt,
};
if (options.response_format) {
data.response_format = options.response_format;
} else if (!options.omitDefaultResponseFormat) {
data.response_format = 'url'; // 默认返回 url
}
// size 参数可选，不传则由 API 自动决定（对应 auto）
if (options.size && options.size !== 'auto') {
data.size = options.size;
}
// image 参数可选（单图或多图）
if (options.image) {
data.image = options.image;
}
// quality 参数可选
if (options.quality) {
data.quality = options.quality;
}
if (
typeof options.count === 'number' &&
Number.isFinite(options.count) &&
options.count > 1
) {
data.n = Math.min(Math.max(Math.round(options.count), 1), 10);
}
const localImageParamMap: Record<string, string> = {
negative_prompt: 'negative_prompt',
negativePrompt: 'negative_prompt',
steps: 'steps',
step: 'steps',
cfg_scale: 'cfg_scale',
cfgScale: 'cfg_scale',
seed: 'seed',
};
Object.entries(localImageParamMap).forEach(([sourceKey, targetKey]) => {
const value = options.params?.[sourceKey];
if (value !== undefined && value !== null && value !== '') {
data[targetKey] = value;
}
});
const response = await providerTransport.send(
buildProviderContextFromConfig(validatedConfig),
{
path: '/images/generations',
method: 'POST',
headers,
body: JSON.stringify(data),
timeoutMs: IMAGE_GENERATION_TIMEOUT_MS,
}
);
if (!response.ok) {
const errorText = await response.text();
console.error('[ImageAPI] Request failed:', response.status, errorText);
const duration = Date.now() - startTime;
failLLMApiLog(logId, {
httpStatus: response.status,
duration,
errorMessage: errorText.substring(0, 500),
});
const error = new Error(
`图片生成请求失败: ${response.status} - ${errorText}`
);
(error as any).apiErrorBody = errorText;
(error as any).httpStatus = response.status;
throw error;
}
const result = await response.json();
const duration = Date.now() - startTime;
// 提取结果 URL
const resultUrl = result.data?.[0]?.url || result.data?.[0]?.b64_json;
const normalizedResultUrl =
typeof resultUrl === 'string'
? normalizeImageDataUrl(resultUrl)
: undefined;
completeLLMApiLog(logId, {
httpStatus: response.status,
duration,
resultType: 'image',
resultCount: result.data?.length || 1,
resultUrl: normalizedResultUrl?.substring(0, 200),
});
return result;
} catch (error: any) {
const duration = Date.now() - startTime;
// 如果错误还没被记录（非 HTTP 错误）
if (!error.httpStatus) {
failLLMApiLog(logId, {
duration,
errorMessage: error.message || 'Image generation failed',
});
}
throw error;
}
}
/**
* 调用 Gemini API 进行视频生成
*/
export async function generateVideoWithGemini(
prompt: string,
image: ImageInput | null,
options: VideoGenerationOptions = {}
): Promise<{
response: GeminiResponse;
processedContent: ProcessedContent;
}> {
// 等待设置管理器初始化完成
await settingsManager.waitForInitialization();
const { config } = buildRuntimeConfig(
'video',
null,
VIDEO_DEFAULT_CONFIG.modelName || 'veo3',
VIDEO_DEFAULT_CONFIG
);
const validatedConfig = await validateAndEnsureConfig(config);
// 准备图片数据（现在是可选的）
let imageContent;
if (image) {
try {
// console.log('处理视频生成源图片...');
const imageData = await prepareImageData(image);
imageContent = {
type: 'image_url' as const,
image_url: {
url: imageData,
},
};
// console.log('视频生成源图片处理完成');
} catch (error) {
console.error('处理源图片时出错:', error);
throw error;
}
} else {
// console.log('无源图片，使用纯文本生成视频');
}
// 构建视频生成专用的提示词（根据是否有图片使用不同提示词）
const videoPrompt = image
? `Generate a video based on this image and description: "${prompt}"`
: `Generate a video based on this description: "${prompt}"`;
// 构建消息内容（只有在有图片时才包含图片）
const contentList =
image && imageContent
? [{ type: 'text' as const, text: videoPrompt }, imageContent]
: [{ type: 'text' as const, text: videoPrompt }];
const messages: GeminiMessage[] = [
{
role: 'user',
content: contentList,
},
];
// console.log('开始调用视频生成API...');
// 使用专用的视频生成流式调用
const response = await callVideoApiStreamRaw(
validatedConfig,
messages,
options
);
// 处理响应内容
const responseContent = response.choices[0]?.message?.content || '';
const processedContent = processMixedContent(responseContent);
return {
response,
processedContent,
};
}
/**
* 调用 Gemini API 进行聊天对话（支持图片输入）
*/
export async function chatWithGemini(
prompt: string,
images: ImageInput[] = [],
onChunk?: (content: string) => void
): Promise<{
response: GeminiResponse;
processedContent: ProcessedContent;
}> {
// 等待设置管理器初始化完成
await settingsManager.waitForInitialization();
const { config } = buildRuntimeConfig(
images.length > 0 ? 'image' : 'text',
null,
DEFAULT_CONFIG.modelName || 'gemini-2.0-flash',
DEFAULT_CONFIG
);
const validatedConfig = await validateAndEnsureConfig(config);
// 准备图片数据
const imageContents = [];
for (let i = 0; i < images.length; i++) {
try {
// console.log(`处理第 ${i + 1} 张图片...`);
const imageData = await prepareImageData(images[i]);
imageContents.push({
type: 'image_url' as const,
image_url: {
url: imageData,
},
});
} catch (error) {
console.error(`处理第 ${i + 1} 张图片时出错:`, error);
throw error;
}
}
// 构建消息内容
const contentList = [
{ type: 'text' as const, text: prompt },
...imageContents,
];
const messages: GeminiMessage[] = [
{
role: 'user',
content: contentList,
},
];
// console.log(`共发送 ${imageContents.length} 张图片到 Gemini API`);
// 根据模型选择流式或非流式调用
let response: GeminiResponse;
const modelName = validatedConfig.modelName || '';
if (shouldUseNonStreamMode(modelName)) {
// 某些模型（如 seedream）在流式模式下可能返回不完整响应，使用非流式调用
// console.log(`模型 ${modelName} 使用非流式调用确保响应完整`);
response = await callApiWithRetry(validatedConfig, messages);
// Non-stream mode simulates one chunk at the end if callback is provided
if (onChunk && response.choices[0]?.message?.content) {
onChunk(response.choices[0].message.content);
}
} else if (images.length > 0 || onChunk) {
// 其他模型：图文混合或明确要求流式（提供了 onChunk）使用流式调用
// console.log('使用流式调用');
response = await callApiStreamRaw(validatedConfig, messages, onChunk);
} else {
// 纯文本且无流式回调，可以使用非流式调用
response = await callApiWithRetry(validatedConfig, messages);
}
// 处理响应内容
const responseContent = response.choices[0]?.message?.content || '';
const processedContent = processMixedContent(responseContent);
return {
response,
processedContent,
};
}
/**
* sendChatWithGemini 的日志元数据
*/
export interface SendChatLogMeta {
taskType?: LLMApiLog['taskType'];
taskId?: string;
prompt?: string;
}
/**
* 发送多轮对话消息
* @param messages 消息列表
* @param onChunk 流式回调
* @param signal 取消信号
* @param temporaryModel 临时模型引用（仅在当前会话中使用，不影响全局设置）
* @param logMeta 日志元数据，不传则默认 taskType='chat'
*/
export async function sendChatWithGemini(
messages: GeminiMessage[],
onChunk?: (content: string) => void,
signal?: AbortSignal,
temporaryModel?: string | ModelRef | null,
logMeta?: SendChatLogMeta
): Promise<GeminiResponse> {
// 等待设置管理器初始化完成
const t0 = Date.now();
await settingsManager.waitForInitialization();
const { config } = buildRuntimeConfig(
'text',
temporaryModel || null,
'gpt-4o-mini',
DEFAULT_CONFIG
);
const t1 = Date.now();
const validatedConfig = await validateAndEnsureConfig(config);
// --- LLM API 日志 ---
const firstTextContent = messages[0]?.content;
const firstTextPart = Array.isArray(firstTextContent)
? firstTextContent.find((p) => p.type === 'text')
: undefined;
const autoPrompt =
logMeta?.prompt ||
(firstTextPart && 'text' in firstTextPart ? firstTextPart.text : undefined);
const logId = startLLMApiLog({
endpoint: config.baseUrl || '/chat/completions',
model: config.modelName || 'unknown',
taskType: logMeta?.taskType || 'chat',
prompt: autoPrompt,
taskId: logMeta?.taskId,
});
const startTime = Date.now();
try {
let resultText = '';
// 只有显式需要流式回调的聊天链路才走流式；
// 结构化 JSON / 分析类场景应保持非流式以获得完整响应。
let response: GeminiResponse;
if (onChunk) {
response = await callApiStreamRaw(
validatedConfig,
messages,
(chunk) => {
resultText += chunk;
onChunk(chunk);
},
signal
);
} else {
response = await callApiWithRetry(validatedConfig, messages);
resultText = response.choices?.[0]?.message?.content || '';
}
completeLLMApiLog(logId, {
httpStatus: 200,
duration: Date.now() - startTime,
resultType: 'text',
resultText: resultText ? truncate(resultText, 1000) : undefined,
});
return response;
} catch (error: any) {
failLLMApiLog(logId, {
duration: Date.now() - startTime,
errorMessage: error.message || String(error),
});
throw error;
}
}

88
packages/drawnix/src/utils/gemini-api/types.ts Normal file
View File

@@ -0,0 +1,88 @@
/**
* Gemini API 类型定义
*/
import type {
ProviderAuthStrategy,
ProviderModelBinding,
ResolvedProviderContext,
} from '../../services/provider-routing/types';
export interface GeminiConfig {
apiKey: string;
baseUrl: string;
modelName?: string;
timeout?: number;
authType?: ProviderAuthStrategy;
providerType?: string;
extraHeaders?: Record<string, string>;
protocol?: string | null;
binding?: ProviderModelBinding | null;
provider?: ResolvedProviderContext | null;
}
export interface ImageInput {
file?: File;
base64?: string;
url?: string;
}
export type GeminiMessagePart =
| {
type: 'text';
text?: string;
}
| {
type: 'image_url';
image_url?: {
url: string;
};
}
| {
/** base64 内联数据（视频/音频/PDF 等） */
type: 'inline_data';
mimeType: string;
data: string;
}
| {
/** 远程文件 URI（如 YouTube URL） */
type: 'file_uri';
fileUri: string;
};
export interface GeminiMessage {
role: 'user' | 'assistant' | 'system';
content: GeminiMessagePart[];
}
export interface VideoGenerationOptions {
max_tokens?: number;
temperature?: number;
top_p?: number;
presence_penalty?: number;
frequency_penalty?: number;
}
export interface GeminiResponse {
choices: Array<{
message: {
role: string;
content: string;
};
}>;
}
export interface ProcessedContent {
textContent: string;
images: Array<{
type: 'base64' | 'url';
data: string;
index: number;
}>;
videos?: Array<{
type: 'url';
data: string;
index: number;
}>;
originalContent: string;
}

183
packages/drawnix/src/utils/gemini-api/utils.ts Normal file
View File

@@ -0,0 +1,183 @@
/**
* Gemini API 工具函数
*/
import { ImageInput, ProcessedContent } from './types';
/**
* 将文件转换为 base64 格式
*/
export async function fileToBase64(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const result = reader.result as string;
resolve(result);
};
reader.onerror = () => reject(new Error('文件读取失败'));
reader.readAsDataURL(file);
});
}
/**
* 准备图片数据，转换为 API 所需格式
*/
export async function prepareImageData(image: ImageInput): Promise<string> {
if (image.file) {
return await fileToBase64(image.file);
} else if (image.base64) {
// 确保 base64 数据包含正确的前缀
if (image.base64.startsWith('data:')) {
return image.base64;
} else {
return `data:image/png;base64,${image.base64}`;
}
} else if (image.url) {
// 对于 URL，直接返回（API 可能支持 URL 格式）
return image.url;
} else {
throw new Error('无效的图片输入：必须提供 file、base64 或 url');
}
}
/**
* 将 base64 数据转换为 Blob URL
*/
export function base64ToBlobUrl(base64Data: string, mimeType: string = 'image/png'): string {
const byteCharacters = atob(base64Data);
const byteNumbers = new Array(byteCharacters.length);
for (let i = 0; i < byteCharacters.length; i++) {
byteNumbers[i] = byteCharacters.charCodeAt(i);
}
const byteArray = new Uint8Array(byteNumbers);
const blob = new Blob([byteArray], { type: mimeType });
return URL.createObjectURL(blob);
}
/**
* 处理混合内容（文字、base64图片、URL图片、视频链接）
*/
export function processMixedContent(content: string): ProcessedContent {
// 查找 base64 图片
const base64Pattern = /data:image\/[^;]+;base64,([A-Za-z0-9+/=]+)/g;
const base64Matches = Array.from(content.matchAll(base64Pattern));
// 优先查找 Markdown 格式的图片 ![alt](url)
// 这能正确提取带查询参数的完整 URL
const markdownImagePattern = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/gi;
const markdownImageMatches = Array.from(content.matchAll(markdownImagePattern));
// 查找图片 URL 链接（支持带查询参数的 URL）
// Original: /https?:\/\/[^\s<>"'\]]+\.(png|jpg|jpeg|gif|webp)/gi
const imageUrlPattern = /https?:\/\/[^\s<>"'\])]+\.(png|jpg|jpeg|gif|webp)(\?[^\s<>"'\])]*)?/gi;
const imageUrlMatches = Array.from(content.matchAll(imageUrlPattern));
// 查找视频 URL 链接（包括markdown格式）
const videoUrlPatterns = [
// 匹配markdown链接中的视频URL：[▶️ 在线观看](url) 或 [⏬ 下载视频](url)
/\[(?:▶️\s*在线观看|⏬\s*下载视频|.*?观看.*?|.*?下载.*?)\]\(([^)]+\.(?:mp4|avi|mov|wmv|flv|webm|mkv)(?:\?[^)]*)?)\)/gi,
// 直接的视频URL
/https?:\/\/[^\s<>"'\]]+\.(?:mp4|avi|mov|wmv|flv|webm|mkv)(?:\?[^\s<>"'\]]*)?/gi,
// 特定域名的视频链接（如filesystem.site）
/https?:\/\/filesystem\.site\/[^\s<>"'\]]+/gi
];
let textContent = content;
const images: ProcessedContent['images'] = [];
const videos: Array<{ type: 'url'; data: string; index: number }> = [];
let imageIndex = 1;
let videoIndex = 1;
// 处理 base64 图片
for (const match of base64Matches) {
const fullMatch = match[0];
const base64Data = match[1];
images.push({
type: 'base64',
data: base64Data,
index: imageIndex,
});
textContent = textContent.replace(fullMatch, `[图片 ${imageIndex}]`);
imageIndex++;
}
// 用于避免重复处理同一 URL
const processedUrls = new Set<string>();
// 优先处理 Markdown 格式的图片 ![alt](url) - 能正确提取带查询参数的完整 URL
for (const match of markdownImageMatches) {
const fullMatch = match[0]; // ![alt](url)
const url = match[1]; // 括号内的完整 URL
if (!processedUrls.has(url)) {
processedUrls.add(url);
images.push({
type: 'url',
data: url,
index: imageIndex,
});
textContent = textContent.replace(fullMatch, `[图片 ${imageIndex}]`);
imageIndex++;
}
}
// 处理普通图片 URL（排除已处理的）
for (const match of imageUrlMatches) {
const url = match[0];
if (!processedUrls.has(url)) {
processedUrls.add(url);
images.push({
type: 'url',
data: url,
index: imageIndex,
});
textContent = textContent.replace(url, `[图片 ${imageIndex}]`);
imageIndex++;
}
}
// 处理视频 URL（按优先级顺序）
for (const pattern of videoUrlPatterns) {
const matches = Array.from(content.matchAll(pattern));
for (const match of matches) {
let videoUrl: string;
if (match.length > 1 && match[1]) {
// markdown链接格式，提取括号内的URL
videoUrl = match[1];
} else {
// 直接的URL
videoUrl = match[0];
}
// 清理URL末尾可能的标点符号
videoUrl = videoUrl.replace(/[.,;!?]*$/, '');
// 检查是否已经添加过这个视频URL
const alreadyExists = videos.some(v => v.data === videoUrl);
if (!alreadyExists) {
videos.push({
type: 'url',
data: videoUrl,
index: videoIndex,
});
// 替换原文中的内容
textContent = textContent.replace(match[0], `[视频 ${videoIndex}]`);
videoIndex++;
}
}
}
return {
textContent,
images,
videos: videos.length > 0 ? videos : undefined,
originalContent: content,
};
}