Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/package/yuxi/models/providers/builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"embedding_base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings",
"rerank_base_url": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
"api_key_env": "DASHSCOPE_API_KEY",
"capabilities": ["chat", "embedding", "rerank"],
"capabilities": ["chat", "embedding", "rerank", "image"],
"models_endpoint": "https://dashscope.aliyuncs.com/compatible-mode/v1/models",
"enabled_models": [
{
Expand Down
2 changes: 1 addition & 1 deletion backend/package/yuxi/models/providers/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class ModelInfo:

provider_id: str
model_id: str
model_type: str # chat / embedding / rerank
model_type: str # chat / embedding / rerank / image
display_name: str

# 运行时配置
Expand Down
66 changes: 64 additions & 2 deletions backend/package/yuxi/models/providers/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
)
from yuxi.storage.postgres.models_business import ModelProvider

VALID_MODEL_TYPES = {"chat", "embedding", "rerank"}
VALID_MODEL_TYPES = {"chat", "embedding", "rerank", "image"}
VALID_MODEL_SOURCES = {"manual", "remote"}
VALID_PROVIDER_TYPES = {"openai", "anthropic", "gemini", "openrouter"}
OPENAI_COMPATIBLE_REQUEST_BODY_PROVIDER_TYPES = {"openai", "openrouter"}
Expand Down Expand Up @@ -54,7 +54,7 @@ def _normalize_model_item(model: dict[str, Any]) -> dict[str, Any]:

model_type = str(model.get("type") or "unknown").strip()
if model_type not in VALID_MODEL_TYPES:
raise ValueError(f"启用模型 {model_id} 的 type 必须是 chat、embedding 或 rerank")
raise ValueError(f"启用模型 {model_id} 的 type 必须是 chat、embedding、rerankimage")

# source 区分手动添加 vs 远端拉取,用于跳过远端清单存在性的视觉警告。
source = str(model.get("source") or "remote").strip()
Expand Down Expand Up @@ -457,6 +457,11 @@ async def test_model_status_by_spec(spec: str) -> dict:
"model_type": "rerank",
}

# 图像生成模型不支持 OpenAI 兼容 chat 接口, 走 DashScope 原生
# multimodal-generation 接口测试。
if info.model_type == "image":
return await _test_image_generation_model(spec, info)

from yuxi.models.chat import select_model

model = select_model(model_spec=spec)
Expand All @@ -467,3 +472,60 @@ async def test_model_status_by_spec(spec: str) -> dict:
return {"spec": spec, "status": "unavailable", "message": "响应无效", "model_type": "chat"}
except Exception as e:
return {"spec": spec, "status": "error", "message": str(e), "model_type": info.model_type}


async def _test_image_generation_model(spec: str, info) -> dict:
"""用 DashScope 原生 multimodal-generation 接口测试图像生成模型。

官方文档: qwen-image 系列不支持 compatible-mode, content 必须是
``[{"text": ...}]`` 数组, 图片在 ``output.choices[0].message.content[0].image``。

该协议目前只有 DashScope 系供应商提供;其它供应商配置 image 类型时无法测试,
显式报告"暂不支持",而不是把 DashScope 专用路径拼到它的 base_url 上。
"""
import httpx

api_key = getattr(info, "api_key", "") or ""
if not api_key:
return {"spec": spec, "status": "error", "message": "供应商未配置 API Key", "model_type": "image"}

base = (getattr(info, "base_url", "") or "").rstrip("/")
# DashScope 兼容模式域名换原生 API 域名(同主机, 不同路径前缀)。
if "compatible-mode" in base:
base = base.split("/compatible-mode")[0]
if not base:
base = "https://dashscope.aliyuncs.com"
if "dashscope" not in base:
return {
"spec": spec,
"status": "unavailable",
"message": "当前仅支持 DashScope 图像模型的连接测试",
"model_type": "image",
}
url = f"{base}/api/v1/services/aigc/multimodal-generation/generation"

payload = {
"model": info.model_id,
"input": {"messages": [{"role": "user", "content": [{"text": "a red circle"}]}]},
# 不显式传 size:不同 Qwen-Image 型号支持的分辨率集合不同(如 max/plus 只接受
# 文档列出的尺寸),交给模型默认值,避免测试因参数非法而误判模型不可用。
"parameters": {"prompt_extend": False, "watermark": False, "n": 1},
}
async with httpx.AsyncClient(timeout=120) as client:
resp = await client.post(url, json=payload, headers={"Authorization": f"Bearer {api_key}"})
if resp.status_code != 200:
detail = resp.text[:200]
return {
"spec": spec,
"status": "unavailable",
"message": f"HTTP {resp.status_code}: {detail}",
"model_type": "image",
}

data = resp.json()
choices = (data.get("output") or {}).get("choices") or []
content = (choices[0].get("message") or {}).get("content") if choices else None
image_url = next((c.get("image") for c in content or [] if isinstance(c, dict) and c.get("image")), None)
if image_url:
return {"spec": spec, "status": "available", "message": "连接正常(已生成测试图片)", "model_type": "image"}
return {"spec": spec, "status": "unavailable", "message": f"响应缺少图片: {str(data)[:150]}", "model_type": "image"}
2 changes: 1 addition & 1 deletion backend/package/yuxi/storage/postgres/models_business.py
Original file line number Diff line number Diff line change
Expand Up @@ -813,7 +813,7 @@ class ModelProvider(Base):
api_key_env = Column(String(128), nullable=True, comment="API Key 环境变量名")
api_key = Column(String(500), nullable=True, comment="直接配置的 API Key")

capabilities = Column(JSON, nullable=False, default=list, comment="支持能力:chat/embedding/rerank")
capabilities = Column(JSON, nullable=False, default=list, comment="支持能力:chat/embedding/rerank/image")
enabled_models = Column(JSON, nullable=False, default=list, comment="已启用模型配置对象")
headers_json = Column(JSON, nullable=True, comment="额外请求头")
extra_json = Column(JSON, nullable=True, comment="扩展配置")
Expand Down
87 changes: 87 additions & 0 deletions backend/test/unit/services/test_model_provider_service.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
import os
from types import SimpleNamespace

Expand Down Expand Up @@ -352,3 +353,89 @@ def test_normalize_payload_allows_model_type_within_capabilities():
sources = [model["source"] for model in payload["enabled_models"]]
assert types == ["chat", "embedding"]
assert sources == ["manual", "manual"]


def test_normalize_payload_accepts_image_model_type():
"""image 是正式模型类型,provider 声明该能力后可写入图像生成模型。"""
payload = _normalize_payload(
{
"provider_id": "image-provider",
"display_name": "Image Provider",
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
"capabilities": ["chat", "image"],
"enabled_models": [{"id": "qwen-image-3.0", "type": "image", "source": "manual"}],
}
)

assert payload["enabled_models"][0]["type"] == "image"


def test_normalize_payload_rejects_image_model_without_capability():
"""provider 未声明 image 能力时,拒绝写入 image 类型模型。"""
with pytest.raises(ValueError, match="不在 provider 能力"):
_normalize_payload(
{
"provider_id": "chat-only",
"display_name": "Chat Only",
"base_url": "https://example.com/v1",
"capabilities": ["chat"],
"enabled_models": [{"id": "qwen-image-3.0", "type": "image"}],
}
)


def test_normalize_remote_model_preserves_image_type():
"""远端模型清单返回 image 类型时,归一化保留 image 而非兜底成 chat。"""
model = _normalize_remote_model({"id": "qwen-image-3.0", "type": "image", "name": "Qwen Image"})

assert model["id"] == "qwen-image-3.0"
assert model["type"] == "image"


def _image_model_info(*, base_url: str, model_id: str = "qwen-image-3.0"):
from types import SimpleNamespace

return SimpleNamespace(
provider_id="alibaba-cn",
model_id=model_id,
model_type="image",
display_name=model_id,
api_key="sk-test",
base_url=base_url,
provider_type="openai",
spec=f"alibaba-cn:{model_id}",
)


@pytest.mark.asyncio
async def test_image_test_uses_native_endpoint_without_size_parameter(httpx_mock):
"""图像模型测试走 DashScope 原生接口,且不硬编码 size(型号支持集合不同)。"""
from yuxi.models.providers.service import _test_image_generation_model

httpx_mock.add_response(
url="https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation",
json={"output": {"choices": [{"message": {"content": [{"image": "https://example.test/a.png"}]}}]}},
)

info = _image_model_info(base_url="https://dashscope.aliyuncs.com/compatible-mode/v1")
result = await _test_image_generation_model("alibaba-cn:qwen-image-3.0", info)

assert result["status"] == "available"
request = httpx_mock.get_requests()[-1]
assert request.url.path == "/api/v1/services/aigc/multimodal-generation/generation"
body = json.loads(request.content)
assert "size" not in body["parameters"]
assert body["input"]["messages"][0]["content"] == [{"text": "a red circle"}]


@pytest.mark.asyncio
async def test_image_test_reports_unsupported_provider_without_sending_request(httpx_mock):
"""非 DashScope 供应商的 image 模型:显式报告暂不支持,不得按其 base_url 拼 DashScope 路径。"""
from yuxi.models.providers.service import _test_image_generation_model

info = _image_model_info(base_url="https://api.example.com/v1/images")
result = await _test_image_generation_model("example:flux-pro", info)

assert result["status"] == "unavailable"
assert "DashScope" in result["message"]
assert httpx_mock.get_requests() == []
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# 模型类型系统新增 image 类型

状态:implemented
类型:feature
Owner:backend/package/yuxi/models/providers/service.py

## 问题

图像生成模型(如 DashScope 的 qwen-image 系列)不支持 OpenAI 兼容 chat 接口,只支持 DashScope 原生 multimodal-generation 接口(`content` 必须是 `[{"text": ...}]` 数组)。但 Yuxi 的模型类型系统只有 chat / embedding / rerank 三种(`VALID_MODEL_TYPES`),图像生成模型被兜底登记为 `chat`,导致两个缺陷:

1. 模型测试按 chat 接口调用,报 `Input should be a valid list: input.messages.0.content`;
2. `get_all_specs("chat")` 把图像生成模型混入纯文本对话智能体的可选模型列表,误选后运行时报同样错误。

## 决策

给模型类型系统正式新增第四种 `image` 类型:

- `VALID_MODEL_TYPES` 加入 `"image"`,`_normalize_model_item` 与 `_normalize_remote_model` 随之接受并保留 image 类型,不再兜底成 chat;
- DashScope builtin provider 的 `capabilities` 加入 `"image"`;
- 前端 ModelProviderManagePanel 的 capabilities 多选、类型 tab、type 下拉均支持「图像生成」;
- `test_model_status_by_spec` 以 `info.model_type == "image"` 走原生接口测试,不再靠 model_id 字符串匹配;
- 图像模型测试**按供应商协议分流**:原生 multimodal-generation 协议当前只有 DashScope 系供应商提供,非 DashScope 的 image 模型显式返回「暂不支持」,不把 DashScope 专用路径拼到它的 base_url 上;
- 测试请求**不显式传 `size`**:不同 Qwen-Image 型号支持的分辨率集合不同(max/plus 只接受文档列出的尺寸),传固定值会让正常模型因参数非法被判不可用;交给模型默认值。

`get_all_specs("chat")`、`model_type == "chat"` 等纯文本模型的分流逻辑不变,image 类型天然不命中这些分支,纯文本模型行为完全不受影响。

## 替代方案

- 仅靠 model_id 字符串匹配特判(原临时修复):无法让 UI 正确显示「图像生成」,纯文本模型列表仍混入图像模型,且靠猜模型命名,新模型名会漏判;拒绝为正式方案。
- 自动数据迁移脚本:仓库无 alembic 迁移机制,自动改写用户配置数据违背 `ensure_builtin_model_providers_in_db` 的「不覆盖已编辑配置」契约;拒绝。历史数据改为在 UI 中一次性手动纠正(勾选 image 能力 + 改 qwen-image 的 type)。

## 后果

新增 image 类型后,新添加的图像生成模型正确落位,纯文本对话智能体的模型列表不再混入图像模型。历史手动添加的 qwen-image 已通过 UI 把 DashScope 的「能力」勾选 image 并将两个 qwen-image 模型的 type 改为「图像生成」,无需 model_id 兜底。DashScope 图像模型调用仍走 `_test_image_generation_model` 的原生接口(内部将 compatible-mode 域名切回原生域名)。

## 验证

- `backend/test/unit/services/test_model_provider_service.py` 新增 5 用例:image 类型可写入、无 image 能力时拒绝、远端归一化保留 image 类型,以及断言**实际 HTTP payload** 的 2 个(原生 endpoint + 不含 `size`;非 DashScope 供应商不发请求并显式报告不支持)。
- `docker compose exec api uv run --group test pytest test/unit/services/test_model_provider_service.py -q` 通过。
- 浏览器回归:模型管理页 DashScope 能力可勾选 image,qwen-image 测试按钮显示「连接正常」,纯文本 chat 模型列表不再混入图像模型。
18 changes: 13 additions & 5 deletions web/src/components/model-management/ModelProviderManagePanel.vue
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,12 @@ const MODALITY_DISPLAY = {
pdf: { icon: FileText, label: 'PDF 文档输入' }
}
const REQUEST_BODY_OVERRIDES_PLACEHOLDER = '{\n "enable_thinking": false\n}'
const MODEL_TYPE_LABELS = {
chat: '对话',
embedding: '向量',
rerank: '重排',
image: '图像生成'
}

// Provider form state
const showProviderModal = ref(false)
Expand Down Expand Up @@ -259,18 +265,19 @@ const remoteModelTypeOptions = computed(() => {
}, {})
return [
{ label: `全部 ${models.length}`, value: 'all' },
{ label: `对话 ${counts.chat || 0}`, value: 'chat' },
{ label: `向量 ${counts.embedding || 0}`, value: 'embedding' },
{ label: `重排 ${counts.rerank || 0}`, value: 'rerank' }
{ label: `${MODEL_TYPE_LABELS.chat} ${counts.chat || 0}`, value: 'chat' },
{ label: `${MODEL_TYPE_LABELS.embedding} ${counts.embedding || 0}`, value: 'embedding' },
{ label: `${MODEL_TYPE_LABELS.rerank} ${counts.rerank || 0}`, value: 'rerank' },
{ label: `${MODEL_TYPE_LABELS.image} ${counts.image || 0}`, value: 'image' }
]
})

// Model Config Modal 的 type 下拉选项:基于 provider.capabilities 限定
// 旧数据 capabilities 为空时回退到全集,保持现状
const editingModelTypeOptions = computed(() => {
const caps = currentProviderForModels.value?.capabilities
const types = Array.isArray(caps) && caps.length ? caps : ['chat', 'embedding', 'rerank']
return types.map((c) => ({ value: c, label: c }))
const types = Array.isArray(caps) && caps.length ? caps : ['chat', 'embedding', 'rerank', 'image']
return types.map((c) => ({ value: c, label: MODEL_TYPE_LABELS[c] || c }))
})

const parseJsonObject = (text, label) => {
Expand Down Expand Up @@ -963,6 +970,7 @@ defineExpose({
<a-select-option value="chat">chat</a-select-option>
<a-select-option value="embedding">embedding</a-select-option>
<a-select-option value="rerank">rerank</a-select-option>
<a-select-option value="image">image</a-select-option>
</a-select>
</label>

Expand Down
Loading