diff --git a/backend/package/yuxi/models/providers/builtin.py b/backend/package/yuxi/models/providers/builtin.py index ac73637e8..6d9f04929 100644 --- a/backend/package/yuxi/models/providers/builtin.py +++ b/backend/package/yuxi/models/providers/builtin.py @@ -38,7 +38,7 @@ "embedding_base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings", "rerank_base_url": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks", "api_key_env": "DASHSCOPE_API_KEY", - "capabilities": ["chat", "embedding", "rerank"], + "capabilities": ["chat", "embedding", "rerank", "image"], "models_endpoint": "https://dashscope.aliyuncs.com/compatible-mode/v1/models", "enabled_models": [ { diff --git a/backend/package/yuxi/models/providers/cache.py b/backend/package/yuxi/models/providers/cache.py index 03b095593..9fcf30a02 100644 --- a/backend/package/yuxi/models/providers/cache.py +++ b/backend/package/yuxi/models/providers/cache.py @@ -26,7 +26,7 @@ class ModelInfo: provider_id: str model_id: str - model_type: str # chat / embedding / rerank + model_type: str # chat / embedding / rerank / image display_name: str # 运行时配置 diff --git a/backend/package/yuxi/models/providers/service.py b/backend/package/yuxi/models/providers/service.py index bb63f68a8..ebf09ea0e 100644 --- a/backend/package/yuxi/models/providers/service.py +++ b/backend/package/yuxi/models/providers/service.py @@ -19,7 +19,7 @@ ) from yuxi.storage.postgres.models_business import ModelProvider -VALID_MODEL_TYPES = {"chat", "embedding", "rerank"} +VALID_MODEL_TYPES = {"chat", "embedding", "rerank", "image"} VALID_MODEL_SOURCES = {"manual", "remote"} VALID_PROVIDER_TYPES = {"openai", "anthropic", "gemini", "openrouter"} OPENAI_COMPATIBLE_REQUEST_BODY_PROVIDER_TYPES = {"openai", "openrouter"} @@ -54,7 +54,7 @@ def _normalize_model_item(model: dict[str, Any]) -> dict[str, Any]: model_type = str(model.get("type") or "unknown").strip() if model_type not in VALID_MODEL_TYPES: - raise ValueError(f"启用模型 {model_id} 的 type 必须是 chat、embedding 或 rerank") + raise ValueError(f"启用模型 {model_id} 的 type 必须是 chat、embedding、rerank 或 image") # source 区分手动添加 vs 远端拉取,用于跳过远端清单存在性的视觉警告。 source = str(model.get("source") or "remote").strip() @@ -457,6 +457,11 @@ async def test_model_status_by_spec(spec: str) -> dict: "model_type": "rerank", } + # 图像生成模型不支持 OpenAI 兼容 chat 接口, 走 DashScope 原生 + # multimodal-generation 接口测试。 + if info.model_type == "image": + return await _test_image_generation_model(spec, info) + from yuxi.models.chat import select_model model = select_model(model_spec=spec) @@ -467,3 +472,60 @@ async def test_model_status_by_spec(spec: str) -> dict: return {"spec": spec, "status": "unavailable", "message": "响应无效", "model_type": "chat"} except Exception as e: return {"spec": spec, "status": "error", "message": str(e), "model_type": info.model_type} + + +async def _test_image_generation_model(spec: str, info) -> dict: + """用 DashScope 原生 multimodal-generation 接口测试图像生成模型。 + + 官方文档: qwen-image 系列不支持 compatible-mode, content 必须是 + ``[{"text": ...}]`` 数组, 图片在 ``output.choices[0].message.content[0].image``。 + + 该协议目前只有 DashScope 系供应商提供;其它供应商配置 image 类型时无法测试, + 显式报告"暂不支持",而不是把 DashScope 专用路径拼到它的 base_url 上。 + """ + import httpx + + api_key = getattr(info, "api_key", "") or "" + if not api_key: + return {"spec": spec, "status": "error", "message": "供应商未配置 API Key", "model_type": "image"} + + base = (getattr(info, "base_url", "") or "").rstrip("/") + # DashScope 兼容模式域名换原生 API 域名(同主机, 不同路径前缀)。 + if "compatible-mode" in base: + base = base.split("/compatible-mode")[0] + if not base: + base = "https://dashscope.aliyuncs.com" + if "dashscope" not in base: + return { + "spec": spec, + "status": "unavailable", + "message": "当前仅支持 DashScope 图像模型的连接测试", + "model_type": "image", + } + url = f"{base}/api/v1/services/aigc/multimodal-generation/generation" + + payload = { + "model": info.model_id, + "input": {"messages": [{"role": "user", "content": [{"text": "a red circle"}]}]}, + # 不显式传 size:不同 Qwen-Image 型号支持的分辨率集合不同(如 max/plus 只接受 + # 文档列出的尺寸),交给模型默认值,避免测试因参数非法而误判模型不可用。 + "parameters": {"prompt_extend": False, "watermark": False, "n": 1}, + } + async with httpx.AsyncClient(timeout=120) as client: + resp = await client.post(url, json=payload, headers={"Authorization": f"Bearer {api_key}"}) + if resp.status_code != 200: + detail = resp.text[:200] + return { + "spec": spec, + "status": "unavailable", + "message": f"HTTP {resp.status_code}: {detail}", + "model_type": "image", + } + + data = resp.json() + choices = (data.get("output") or {}).get("choices") or [] + content = (choices[0].get("message") or {}).get("content") if choices else None + image_url = next((c.get("image") for c in content or [] if isinstance(c, dict) and c.get("image")), None) + if image_url: + return {"spec": spec, "status": "available", "message": "连接正常(已生成测试图片)", "model_type": "image"} + return {"spec": spec, "status": "unavailable", "message": f"响应缺少图片: {str(data)[:150]}", "model_type": "image"} diff --git a/backend/package/yuxi/storage/postgres/models_business.py b/backend/package/yuxi/storage/postgres/models_business.py index 43e50cc09..6ebfd6407 100644 --- a/backend/package/yuxi/storage/postgres/models_business.py +++ b/backend/package/yuxi/storage/postgres/models_business.py @@ -813,7 +813,7 @@ class ModelProvider(Base): api_key_env = Column(String(128), nullable=True, comment="API Key 环境变量名") api_key = Column(String(500), nullable=True, comment="直接配置的 API Key") - capabilities = Column(JSON, nullable=False, default=list, comment="支持能力:chat/embedding/rerank") + capabilities = Column(JSON, nullable=False, default=list, comment="支持能力:chat/embedding/rerank/image") enabled_models = Column(JSON, nullable=False, default=list, comment="已启用模型配置对象") headers_json = Column(JSON, nullable=True, comment="额外请求头") extra_json = Column(JSON, nullable=True, comment="扩展配置") diff --git a/backend/test/unit/services/test_model_provider_service.py b/backend/test/unit/services/test_model_provider_service.py index 6d6e99703..5f441339e 100644 --- a/backend/test/unit/services/test_model_provider_service.py +++ b/backend/test/unit/services/test_model_provider_service.py @@ -1,3 +1,4 @@ +import json import os from types import SimpleNamespace @@ -352,3 +353,89 @@ def test_normalize_payload_allows_model_type_within_capabilities(): sources = [model["source"] for model in payload["enabled_models"]] assert types == ["chat", "embedding"] assert sources == ["manual", "manual"] + + +def test_normalize_payload_accepts_image_model_type(): + """image 是正式模型类型,provider 声明该能力后可写入图像生成模型。""" + payload = _normalize_payload( + { + "provider_id": "image-provider", + "display_name": "Image Provider", + "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", + "capabilities": ["chat", "image"], + "enabled_models": [{"id": "qwen-image-3.0", "type": "image", "source": "manual"}], + } + ) + + assert payload["enabled_models"][0]["type"] == "image" + + +def test_normalize_payload_rejects_image_model_without_capability(): + """provider 未声明 image 能力时,拒绝写入 image 类型模型。""" + with pytest.raises(ValueError, match="不在 provider 能力"): + _normalize_payload( + { + "provider_id": "chat-only", + "display_name": "Chat Only", + "base_url": "https://example.com/v1", + "capabilities": ["chat"], + "enabled_models": [{"id": "qwen-image-3.0", "type": "image"}], + } + ) + + +def test_normalize_remote_model_preserves_image_type(): + """远端模型清单返回 image 类型时,归一化保留 image 而非兜底成 chat。""" + model = _normalize_remote_model({"id": "qwen-image-3.0", "type": "image", "name": "Qwen Image"}) + + assert model["id"] == "qwen-image-3.0" + assert model["type"] == "image" + + +def _image_model_info(*, base_url: str, model_id: str = "qwen-image-3.0"): + from types import SimpleNamespace + + return SimpleNamespace( + provider_id="alibaba-cn", + model_id=model_id, + model_type="image", + display_name=model_id, + api_key="sk-test", + base_url=base_url, + provider_type="openai", + spec=f"alibaba-cn:{model_id}", + ) + + +@pytest.mark.asyncio +async def test_image_test_uses_native_endpoint_without_size_parameter(httpx_mock): + """图像模型测试走 DashScope 原生接口,且不硬编码 size(型号支持集合不同)。""" + from yuxi.models.providers.service import _test_image_generation_model + + httpx_mock.add_response( + url="https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", + json={"output": {"choices": [{"message": {"content": [{"image": "https://example.test/a.png"}]}}]}}, + ) + + info = _image_model_info(base_url="https://dashscope.aliyuncs.com/compatible-mode/v1") + result = await _test_image_generation_model("alibaba-cn:qwen-image-3.0", info) + + assert result["status"] == "available" + request = httpx_mock.get_requests()[-1] + assert request.url.path == "/api/v1/services/aigc/multimodal-generation/generation" + body = json.loads(request.content) + assert "size" not in body["parameters"] + assert body["input"]["messages"][0]["content"] == [{"text": "a red circle"}] + + +@pytest.mark.asyncio +async def test_image_test_reports_unsupported_provider_without_sending_request(httpx_mock): + """非 DashScope 供应商的 image 模型:显式报告暂不支持,不得按其 base_url 拼 DashScope 路径。""" + from yuxi.models.providers.service import _test_image_generation_model + + info = _image_model_info(base_url="https://api.example.com/v1/images") + result = await _test_image_generation_model("example:flux-pro", info) + + assert result["status"] == "unavailable" + assert "DashScope" in result["message"] + assert httpx_mock.get_requests() == [] diff --git a/docs/develop-guides/decisions/implemented/2026-09-08-model-type-image.md b/docs/develop-guides/decisions/implemented/2026-09-08-model-type-image.md new file mode 100644 index 000000000..03ab181bd --- /dev/null +++ b/docs/develop-guides/decisions/implemented/2026-09-08-model-type-image.md @@ -0,0 +1,40 @@ +# 模型类型系统新增 image 类型 + +状态:implemented +类型:feature +Owner:backend/package/yuxi/models/providers/service.py + +## 问题 + +图像生成模型(如 DashScope 的 qwen-image 系列)不支持 OpenAI 兼容 chat 接口,只支持 DashScope 原生 multimodal-generation 接口(`content` 必须是 `[{"text": ...}]` 数组)。但 Yuxi 的模型类型系统只有 chat / embedding / rerank 三种(`VALID_MODEL_TYPES`),图像生成模型被兜底登记为 `chat`,导致两个缺陷: + +1. 模型测试按 chat 接口调用,报 `Input should be a valid list: input.messages.0.content`; +2. `get_all_specs("chat")` 把图像生成模型混入纯文本对话智能体的可选模型列表,误选后运行时报同样错误。 + +## 决策 + +给模型类型系统正式新增第四种 `image` 类型: + +- `VALID_MODEL_TYPES` 加入 `"image"`,`_normalize_model_item` 与 `_normalize_remote_model` 随之接受并保留 image 类型,不再兜底成 chat; +- DashScope builtin provider 的 `capabilities` 加入 `"image"`; +- 前端 ModelProviderManagePanel 的 capabilities 多选、类型 tab、type 下拉均支持「图像生成」; +- `test_model_status_by_spec` 以 `info.model_type == "image"` 走原生接口测试,不再靠 model_id 字符串匹配; +- 图像模型测试**按供应商协议分流**:原生 multimodal-generation 协议当前只有 DashScope 系供应商提供,非 DashScope 的 image 模型显式返回「暂不支持」,不把 DashScope 专用路径拼到它的 base_url 上; +- 测试请求**不显式传 `size`**:不同 Qwen-Image 型号支持的分辨率集合不同(max/plus 只接受文档列出的尺寸),传固定值会让正常模型因参数非法被判不可用;交给模型默认值。 + +`get_all_specs("chat")`、`model_type == "chat"` 等纯文本模型的分流逻辑不变,image 类型天然不命中这些分支,纯文本模型行为完全不受影响。 + +## 替代方案 + +- 仅靠 model_id 字符串匹配特判(原临时修复):无法让 UI 正确显示「图像生成」,纯文本模型列表仍混入图像模型,且靠猜模型命名,新模型名会漏判;拒绝为正式方案。 +- 自动数据迁移脚本:仓库无 alembic 迁移机制,自动改写用户配置数据违背 `ensure_builtin_model_providers_in_db` 的「不覆盖已编辑配置」契约;拒绝。历史数据改为在 UI 中一次性手动纠正(勾选 image 能力 + 改 qwen-image 的 type)。 + +## 后果 + +新增 image 类型后,新添加的图像生成模型正确落位,纯文本对话智能体的模型列表不再混入图像模型。历史手动添加的 qwen-image 已通过 UI 把 DashScope 的「能力」勾选 image 并将两个 qwen-image 模型的 type 改为「图像生成」,无需 model_id 兜底。DashScope 图像模型调用仍走 `_test_image_generation_model` 的原生接口(内部将 compatible-mode 域名切回原生域名)。 + +## 验证 + +- `backend/test/unit/services/test_model_provider_service.py` 新增 5 用例:image 类型可写入、无 image 能力时拒绝、远端归一化保留 image 类型,以及断言**实际 HTTP payload** 的 2 个(原生 endpoint + 不含 `size`;非 DashScope 供应商不发请求并显式报告不支持)。 +- `docker compose exec api uv run --group test pytest test/unit/services/test_model_provider_service.py -q` 通过。 +- 浏览器回归:模型管理页 DashScope 能力可勾选 image,qwen-image 测试按钮显示「连接正常」,纯文本 chat 模型列表不再混入图像模型。 diff --git a/web/src/components/model-management/ModelProviderManagePanel.vue b/web/src/components/model-management/ModelProviderManagePanel.vue index a88adb33f..fecb635aa 100644 --- a/web/src/components/model-management/ModelProviderManagePanel.vue +++ b/web/src/components/model-management/ModelProviderManagePanel.vue @@ -54,6 +54,12 @@ const MODALITY_DISPLAY = { pdf: { icon: FileText, label: 'PDF 文档输入' } } const REQUEST_BODY_OVERRIDES_PLACEHOLDER = '{\n "enable_thinking": false\n}' +const MODEL_TYPE_LABELS = { + chat: '对话', + embedding: '向量', + rerank: '重排', + image: '图像生成' +} // Provider form state const showProviderModal = ref(false) @@ -259,9 +265,10 @@ const remoteModelTypeOptions = computed(() => { }, {}) return [ { label: `全部 ${models.length}`, value: 'all' }, - { label: `对话 ${counts.chat || 0}`, value: 'chat' }, - { label: `向量 ${counts.embedding || 0}`, value: 'embedding' }, - { label: `重排 ${counts.rerank || 0}`, value: 'rerank' } + { label: `${MODEL_TYPE_LABELS.chat} ${counts.chat || 0}`, value: 'chat' }, + { label: `${MODEL_TYPE_LABELS.embedding} ${counts.embedding || 0}`, value: 'embedding' }, + { label: `${MODEL_TYPE_LABELS.rerank} ${counts.rerank || 0}`, value: 'rerank' }, + { label: `${MODEL_TYPE_LABELS.image} ${counts.image || 0}`, value: 'image' } ] }) @@ -269,8 +276,8 @@ const remoteModelTypeOptions = computed(() => { // 旧数据 capabilities 为空时回退到全集,保持现状 const editingModelTypeOptions = computed(() => { const caps = currentProviderForModels.value?.capabilities - const types = Array.isArray(caps) && caps.length ? caps : ['chat', 'embedding', 'rerank'] - return types.map((c) => ({ value: c, label: c })) + const types = Array.isArray(caps) && caps.length ? caps : ['chat', 'embedding', 'rerank', 'image'] + return types.map((c) => ({ value: c, label: MODEL_TYPE_LABELS[c] || c })) }) const parseJsonObject = (text, label) => { @@ -963,6 +970,7 @@ defineExpose({ chat embedding rerank + image