Ollama上的MiniCPM-V版MCP视觉工具
通过 Ollama 上的 MiniCPM-V，借助 MCP 提供展示描述、OCR 以及图像比对工具，从而使 Cursor 和 Claude Desktop 能够完全在本地对截图进行处理。
MCP主机的本地视觉工具
只需部署一个能够处理图像描述、文档OCR识别以及图像间对比功能的MCP服务器即可。任何支持MCP的主机——如Cursor、Claude Desktop等——都能直接分析截图、收据和UI差异，而无需每次都自行开发私有HTTP API。
其强大功能源自Ollama提供的MiniCPM-V 4.6：拥有约13亿个参数，下载体积小，同时支持文本和图像处理，并具备较大的上下文窗口。该模型专为边缘设备优化了视觉功能。使用演示非常简单：只需将截图放入集成开发环境，然后询问有哪些变化；智能体就会直接进行对比分析，而无需调用云端视觉服务。
学习目标
你将加载视觉模型，了解为何MCP比一次性REST接口更适合作为工具的底层架构，在服务器和演示环境中共享同一个Ollama后端，实现三个功能明确的工具，进行终端操作演示，最后连接Cursor与Claude Desktop。
为何智能体需要视觉工具
当证据为PNG或扫描后的PDF文件时，纯文本智能体会无法正常工作。虽然云视觉API可用，但需要额外管理密钥、数据输出策略以及相应的SDK。通过MCP发布工具可以将模型选择和文件系统操作留在你的程序内部，而编辑者仅能看到架构定义和类型化的参数。
通过Ollama使用MiniCPM-V
ollama pull minicpm-v4.6
ollama run minicpm-v4.6 "Describe this image" --image ./photo.jpg
如果该提示能生成合理的描述文字，就说明运行环境已准备好支持MCP。
快速入门步骤
克隆指南文件夹，创建虚拟环境，并安装所需依赖：
cd guides/minicpm-v-mcp-server
python -m venv .venv && source .venv/bin/activate
pip install -r requirements.txt
cp .env.example .env
ollama pull minicpm-v4.6
python examples/generate_fixtures.py
python examples/agent_demo.py
生成合成测试数据，这样演示就不必依赖专有的截图：
generate_fixtures.py
#!/usr/bin/env python3
"""Generate sample images for MCP demos (no external assets required)."""
from __future__ import annotations
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
FIXTURES = Path(__file__).resolve().parent / "fixtures"
def _font(size: int):
for name in ("DejaVuSans.ttf", "/System/Library/Fonts/Supplemental/Arial.ttf"):
try:
return ImageFont.truetype(name, size)
except OSError:
continue
return ImageFont.load_default()
def receipt() -> None:
img = Image.new("RGB", (480, 640), "#fafafa")
d = ImageDraw.Draw(img)
f, fs = _font(22), _font(16)
d.text((40, 40), "COFFEE BEAN Co.", fill="#1a1a1a", font=f)
d.text((40, 90), "123 Main St · San Francisco", fill="#555", font=fs)
lines = [
"Latte (Oat) $5.50",
"Croissant $4.25",
"Tip $1.00",
"─────────────────────────",
"TOTAL $10.75",
"Card **** 4242",
"2026-06-23 09:14 AM",
]
y = 150
for line in lines:
d.text((40, y), line, fill="#222", font=fs)
y += 36
img.save(FIXTURES / "sample_receipt.png")
def diagram_v1() -> None:
img = Image.new("RGB", (640, 400), "#0f172a")
d = ImageDraw.Draw(img)
f = _font(18)
d.rounded_rectangle((40, 80, 200, 160), radius=12, fill="#1e3a5f", outline="#38bdf8")
d.text((70, 110), "API", fill="#e2e8f0", font=f)
d.rounded_rectangle((420, 80, 580, 160), radius=12, fill="#14532d", outline="#4ade80")
d.text((440, 110), "Qdrant", fill="#e2e8f0", font=f)
d.line((200, 120, 420, 120), fill="#94a3b8", width=3)
d.text((250, 200), "v1 — sync pipeline", fill="#94a3b8", font=f)
img.save(FIXTURES / "diagram_v1.png")
def diagram_v2() -> None:
img = Image.new("RGB", (640, 400), "#0f172a")
d = ImageDraw.Draw(img)
f = _font(18)
d.rounded_rectangle((40, 80, 200, 160), radius=12, fill="#1e3a5f", outline="#38bdf8")
d.text((60, 110), "LitServe", fill="#e2e8f0", font=f)
d.rounded_rectangle((250, 60, 410, 140), radius=12, fill="#4c1d95", outline="#a78bfa")
d.text((270, 90), "CrewAI", fill="#e2e8f0", font=f)
d.rounded_rectangle((420, 80, 580, 160), radius=12, fill="#14532d", outline="#4ade80")
d.text((440, 110), "Qdrant", fill="#e2e8f0", font=f)
d.line((200, 120, 250, 100), fill="#94a3b8", width=3)
d.line((410, 100, 420, 120), fill="#94a3b8", width=3)
d.text((200, 200), "v2 — agentic pipeline", fill="#fbbf24", font=f)
img.save(FIXTURES / "diagram_v2.png")
def main() -> None:
FIXTURES.mkdir(parents=True, exist_ok=True)
receipt()
diagram_v1()
diagram_v2()
print(f"Fixtures written to {FIXTURES}")
if __name__ == "__main__":
main()
端到端智能体演示脚本使用了与MCP服务器提供的相同工具：
agent_demo.py
#!/usr/bin/env python3
"""End-to-end demo — MiniCPM-V MCP vision tools (works offline with OLLAMA_MOCK=1).
Simulates what Cursor / Claude Desktop sees when the agent calls vision tools.
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT))
import vision_backend as vb # noqa: E402
from server import compare_images, describe_image, ocr_document # noqa: E402
FIXTURES = ROOT / "fixtures"
def _banner(title: str) -> None:
print(f"\n{'=' * 60}\n {title}\n{'=' * 60}")
def _parse(result: str) -> dict:
return json.loads(result)
def main() -> None:
os.environ.setdefault("OLLAMA_VISION_MODEL", "minicpm-v4.6")
if not FIXTURES.exists() or not list(FIXTURES.glob("*.png")):
from generate_fixtures import main as gen # noqa: E402
gen()
status = vb.health_check()
_banner("MiniCPM-V 4.6 Vision MCP — Agent Demo")
print(f"Model: {vb.VISION_MODEL} · Mode: {status.get('mode', '?')}")
if not status.get("ok") and not vb.MOCK:
print("\n⚠️ Ollama offline — re-run with OLLAMA_MOCK=1 or start Ollama.\n")
# Scenario 1 — describe screenshot
_banner("Scenario 1 — describe_image")
print('[Tool: describe_image] path=fixtures/diagram_v2.png')
r1 = _parse(describe_image(str(FIXTURES / "diagram_v2.png"), "What services are shown?"))
print("\n## Architecture summary\n")
print(r1.get("result", r1.get("error", r1)))
# Scenario 2 — OCR receipt
_banner("Scenario 2 — ocr_document")
print('[Tool: ocr_document] path=fixtures/sample_receipt.png')
r2 = _parse(ocr_document(str(FIXTURES / "sample_receipt.png")))
print("\n## Receipt OCR\n")
print(r2.get("result", r2.get("error", r2)))
# Scenario 3 — compare before/after
_banner("Scenario 3 — compare_images")
print("[Tool: compare_images] v1 → v2 pipeline diagrams")
r3 = _parse(
compare_images(
str(FIXTURES / "diagram_v1.png"),
str(FIXTURES / "diagram_v2.png"),
focus="new components and labels",
)
)
print("\n## Visual diff\n")
print(r3.get("result", r3.get("error", r3)))
_banner("Done — wire examples/server.py into Cursor MCP settings")
print("See examples/cursor_mcp.json.example")
if __name__ == "__main__":
main()
共享视觉后端
集中处理Ollama调用，避免MCP与演示脚本出现偏差：
vision_backend.py
"""Ollama vision backend for MiniCPM-V 4.6 — shared by MCP server and demos."""
from __future__ import annotations
import base64
import os
from pathlib import Path
import httpx
OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
VISION_MODEL = os.environ.get("OLLAMA_VISION_MODEL", "minicpm-v4.6")
MOCK = os.environ.get("OLLAMA_MOCK", "0") == "1"
SUPPORTED_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp"}
class VisionError(Exception):
pass
def _encode_image(path: Path) -> str:
if not path.is_file():
raise VisionError(f"Image not found: {path}")
if path.suffix.lower() not in SUPPORTED_SUFFIXES:
raise VisionError(f"Unsupported image type: {path.suffix}")
return base64.b64encode(path.read_bytes()).decode("ascii")
def _mock_response(prompt: str, image_count: int) -> str:
return (
f"[mock {VISION_MODEL}] Processed {image_count} image(s).\n"
f"Prompt preview: {prompt[:120]}…\n"
"Set OLLAMA_MOCK=0 and run `ollama pull minicpm-v4.6` for live inference."
)
def chat_vision(prompt: str, image_paths: list[Path], *, timeout: float = 120.0) -> str:
"""Send a vision chat request to Ollama."""
if MOCK:
return _mock_response(prompt, len(image_paths))
images_b64 = [_encode_image(p) for p in image_paths]
payload = {
"model": VISION_MODEL,
"messages": [{"role": "user", "content": prompt, "images": images_b64}],
"stream": False,
}
try:
with httpx.Client(timeout=timeout) as client:
resp = client.post(f"{OLLAMA_HOST}/api/chat", json=payload)
resp.raise_for_status()
data = resp.json()
except httpx.ConnectError as exc:
raise VisionError(
f"Cannot reach Ollama at {OLLAMA_HOST}. Start Ollama and run: ollama pull {VISION_MODEL}"
) from exc
except httpx.HTTPStatusError as exc:
raise VisionError(f"Ollama error {exc.response.status_code}: {exc.response.text[:300]}") from exc
message = data.get("message") or {}
content = message.get("content", "").strip()
if not content:
raise VisionError("Empty response from Ollama")
return content
def health_check() -> dict:
"""Return model + connectivity status for demos."""
if MOCK:
return {"ok": True, "mode": "mock", "model": VISION_MODEL}
try:
with httpx.Client(timeout=5.0) as client:
resp = client.get(f"{OLLAMA_HOST}/api/tags")
resp.raise_for_status()
tags = {m.get("name", "").split(":")[0] for m in resp.json().get("models", [])}
base = VISION_MODEL.split(":")[0]
return {
"ok": base in tags or VISION_MODEL in tags,
"mode": "live",
"model": VISION_MODEL,
"ollama_host": OLLAMA_HOST,
}
except Exception as exc: # noqa: BLE001 — demo helper
return {"ok": False, "mode": "offline", "model": VISION_MODEL, "error": str(exc)}
“带图片的聊天”功能的典型数据格式：
payload = {
"model": VISION_MODEL, # minicpm-v4.6
"messages": [{"role": "user", "content": prompt, "images": images_b64}],
"stream": False,
}
环境模板：
.env.example
# Ollama host (default local)
OLLAMA_HOST=http://127.0.0.1:11434
# Vision model — 1.6 GB, text + image, 256K context
OLLAMA_VISION_MODEL=minicpm-v4.6
# Set to 1 to run agent_demo without Ollama (offline smoke test)
OLLAMA_MOCK=0
将主机URL设置为localhost或局域网网关，并明确指定视觉模型的名称。
工具界面
describe_image可汇总单个文件中的内容以便分类处理。 ocr_document能从收据、表格和幻灯片中提取文本。 compare_images可分析两次界面截图的差异，用于回答“哪些内容发生了变化？”的问题。
注册这些工具的服务器入口点：
server.py
#!/usr/bin/env python3
"""MiniCPM-V MCP server — vision tools for Cursor, Claude Desktop, and Hermes.
Exposes three tools over MCP:
describe_image(path, question?) → general image understanding
ocr_document(path) → structured text extraction
compare_images(path_a, path_b, focus?) → side-by-side visual diff
Powered by MiniCPM-V 4.6 via Ollama (~1.6 GB, text + image, 256K context).
Run: python examples/server.py # stdio transport (for MCP hosts)
"""
from __future__ import annotations
import json
from pathlib import Path
from mcp.server.fastmcp import FastMCP
try:
from . import vision_backend as vb
except ImportError: # pragma: no cover
import vision_backend as vb # type: ignore
mcp = FastMCP("minicpm-vision")
DESCRIBE_DEFAULT = (
"Describe this image in detail. Include objects, text visible, layout, "
"colors, and anything notable for a developer reviewing a screenshot."
)
OCR_PROMPT = (
"Extract all readable text from this document or screenshot. "
"Preserve structure with markdown headings and bullet lists where appropriate. "
"If tables are present, format them as markdown tables."
)
COMPARE_DEFAULT = (
"Compare these two images. List similarities and differences. "
"Note UI changes, text changes, and layout shifts."
)
def _resolve(path: str) -> Path:
p = Path(path).expanduser().resolve()
if not p.is_file():
raise FileNotFoundError(f"Not a file: {p}")
return p
def _tool_result(text: str, **meta) -> str:
return json.dumps({"result": text, **meta}, indent=2)
@mcp.tool()
def describe_image(path: str, question: str = "") -> str:
"""Describe or answer questions about a single image using MiniCPM-V 4.6.
Args:
path: Absolute or relative path to a PNG, JPG, WEBP, or GIF file.
question: Optional specific question about the image. Leave empty for
a general description.
Returns JSON with the model's answer and metadata.
"""
try:
img = _resolve(path)
prompt = question.strip() or DESCRIBE_DEFAULT
answer = vb.chat_vision(prompt, [img])
return _tool_result(answer, tool="describe_image", path=str(img), model=vb.VISION_MODEL)
except (FileNotFoundError, vb.VisionError) as exc:
return json.dumps({"error": str(exc)}, indent=2)
@mcp.tool()
def ocr_document(path: str) -> str:
"""OCR a document, receipt, whiteboard photo, or screenshot to markdown text.
Args:
path: Absolute or relative path to the image file.
Returns JSON with extracted text in markdown format.
"""
try:
img = _resolve(path)
answer = vb.chat_vision(OCR_PROMPT, [img])
return _tool_result(answer, tool="ocr_document", path=str(img), model=vb.VISION_MODEL)
except (FileNotFoundError, vb.VisionError) as exc:
return json.dumps({"error": str(exc)}, indent=2)
@mcp.tool()
def compare_images(path_a: str, path_b: str, focus: str = "") -> str:
"""Compare two images and report visual differences.
Args:
path_a: Path to the first image (e.g. before screenshot).
path_b: Path to the second image (e.g. after screenshot).
focus: Optional aspect to focus on (e.g. "navigation bar", "error message").
Returns JSON with a structured comparison.
"""
try:
a, b = _resolve(path_a), _resolve(path_b)
prompt = COMPARE_DEFAULT
if focus.strip():
prompt += f"\n\nFocus especially on: {focus.strip()}"
answer = vb.chat_vision(prompt, [a, b])
return _tool_result(
answer,
tool="compare_images",
path_a=str(a),
path_b=str(b),
model=vb.VISION_MODEL,
)
except (FileNotFoundError, vb.VisionError) as exc:
return json.dumps({"error": str(exc)}, indent=2)
@mcp.resource("minicpm-vision://model")
def model_info() -> str:
"""Capability hint: which vision model and host this server uses."""
status = vb.health_check()
return json.dumps(
{
"model": vb.VISION_MODEL,
"ollama_host": vb.OLLAMA_HOST,
"tools": ["describe_image", "ocr_document", "compare_images"],
"status": status,
},
indent=2,
)
if __name__ == "__main__":
mcp.run(transport="stdio")
演示循环
生成测试数据并运行智能体演示：
python examples/generate_fixtures.py
python examples/agent_demo.py
agent_demo.py
#!/usr/bin/env python3
"""End-to-end demo — MiniCPM-V MCP vision tools (works offline with OLLAMA_MOCK=1).
Simulates what Cursor / Claude Desktop sees when the agent calls vision tools.
"""
from __future__ import annotations
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT))
import vision_backend as vb # noqa: E402
from server import compare_images, describe_image, ocr_document # noqa: E402
FIXTURES = ROOT / "fixtures"
def _banner(title: str) -> None:
print(f"\n{'=' * 60}\n {title}\n{'=' * 60}")
def _parse(result: str) -> dict:
return json.loads(result)
def main() -> None:
os.environ.setdefault("OLLAMA_VISION_MODEL", "minicpm-v4.6")
if not FIXTURES.exists() or not list(FIXTURES.glob("*.png")):
from generate_fixtures import main as gen # noqa: E402
gen()
status = vb.health_check()
_banner("MiniCPM-V 4.6 Vision MCP — Agent Demo")
print(f"Model: {vb.VISION_MODEL} · Mode: {status.get('mode', '?')}")
if not status.get("ok") and not vb.MOCK:
print("\n⚠️ Ollama offline — re-run with OLLAMA_MOCK=1 or start Ollama.\n")
# Scenario 1 — describe screenshot
_banner("Scenario 1 — describe_image")
print('[Tool: describe_image] path=fixtures/diagram_v2.png')
r1 = _parse(describe_image(str(FIXTURES / "diagram_v2.png"), "What services are shown?"))
print("\n## Architecture summary\n")
print(r1.get("result", r1.get("error", r1)))
# Scenario 2 — OCR receipt
_banner("Scenario 2 — ocr_document")
print('[Tool: ocr_document] path=fixtures/sample_receipt.png')
r2 = _parse(ocr_document(str(FIXTURES / "sample_receipt.png")))
print("\n## Receipt OCR\n")
print(r2.get("result", r2.get("error", r2)))
# Scenario 3 — compare before/after
_banner("Scenario 3 — compare_images")
print("[Tool: compare_images] v1 → v2 pipeline diagrams")
r3 = _parse(
compare_images(
str(FIXTURES / "diagram_v1.png"),
str(FIXTURES / "diagram_v2.png"),
focus="new components and labels",
)
)
print("\n## Visual diff\n")
print(r3.get("result", r3.get("error", r3)))
_banner("Done — wire examples/server.py into Cursor MCP settings")
print("See examples/cursor_mcp.json.example")
if __name__ == "__main__":
main()
预期的工具调用方式如下：
[Tool: describe_image] path=fixtures/diagram_v2.png
[Tool: ocr_document] path=fixtures/sample_receipt.png
[Tool: compare_images] v1 → v2 pipeline diagrams
没有 GPU 的 CI 可以强制返回模拟响应：
OLLAMA_MOCK=1 python examples/agent_demo.py
主机配置
mcpServers 的设置项：
{
"mcpServers": {
"minicpm-vision": {
"command": "python",
"args": ["examples/server.py"],
"cwd": "/absolute/path/to/guides/minicpm-v-mcp-server",
"env": {
"OLLAMA_VISION_MODEL": "minicpm-v4.6",
"OLLAMA_HOST": "http://127.0.0.1:11434"
}
}
}
}
Claude Desktop 的对应配置：
{
"mcpServers": {
"minicpm-vision": {
"command": "python",
"args": ["/absolute/path/to/guides/minicpm-v-mcp-server/examples/server.py"],
"env": {
"OLLAMA_VISION_MODEL": "minicpm-v4.6",
"OLLAMA_HOST": "http://127.0.0.1:11434"
}
}
}
}
两者都将 command 指向你的虚拟环境中的 Python，并通过环境变量传递 Ollama 的配置参数。
操作注意事项
请确保所有测试用例都处于已提交状态，对 OLLAMA_MOCK 进行文档记录，并在 README 中明确指定具体的 Ollama 版本号。这样虽然会让新成员的接入流程和 PR 审核变得繁琐，但正是打造团队真正愿意信任的视觉工具所追求的目标。
总结
一个简洁的本地 VLM 结合 MCP 就足以让截图成为一流的智能体输入方式。建议采用统一的后端，搭配三个功能完善的工具，并保持主机配置的简洁性。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。
记录端口、模型标签和模拟标志，以确保持续集成环境与笔记本电脑的行为一致。