From 87e670dd16861a32cd74cbf8345f9de6ce2ba03a Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 00:53:05 +0800 Subject: [PATCH 1/6] feat: add audited real model provider integration --- .env.example | 23 +- .github/workflows/ci.yml | 30 ++ data_agent/model_adapters/base.py | 28 +- .../model_adapters/openai_compatible.py | 333 ++++++++++++------ data_agent/model_adapters/output_schemas.py | 64 ++++ data_agent/model_adapters/profiles.py | 2 +- data_agent/model_adapters/prompts.py | 6 +- data_agent/model_adapters/redaction.py | 2 + data_agent/process.py | 234 ++++++------ data_agent/ui/app.py | 2 + data_agent/ui/preview.py | 3 +- data_agent/ui/security.py | 2 + model_profiles.yaml.example | 71 ++-- scripts/audit_secret_leaks.py | 79 +++++ scripts/run_real_api_check.py | 212 +++++++++++ tests/test_model_provider_contracts.py | 196 +++++++++++ tests/test_model_provider_mock.py | 45 ++- tests/test_real_api_tools.py | 54 +++ 18 files changed, 1111 insertions(+), 275 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 data_agent/model_adapters/output_schemas.py create mode 100644 scripts/audit_secret_leaks.py create mode 100644 scripts/run_real_api_check.py create mode 100644 tests/test_model_provider_contracts.py create mode 100644 tests/test_real_api_tools.py diff --git a/.env.example b/.env.example index 1a4eeae..ba3a70f 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,12 @@ -BEST_MODEL_BASE_URL= -BEST_MODEL_API_KEY= -BEST_MODEL_NAME= +# Public defaults are safe to commit. API keys and the private Ark endpoint stay blank. +DEEPSEEK_TEXT_BASE_URL=https://api.deepseek.com +DEEPSEEK_TEXT_API_KEY= +DEEPSEEK_TEXT_MODEL=deepseek-v4-pro -FAST_MODEL_BASE_URL= -FAST_MODEL_API_KEY= -FAST_MODEL_NAME= +VOLCENGINE_VISION_BASE_URL=https://ark.cn-beijing.volces.com/api/v3 +VOLCENGINE_VISION_API_KEY= +VOLCENGINE_VISION_MODEL= -VISION_MODEL_BASE_URL= -VISION_MODEL_API_KEY= -VISION_MODEL_NAME= - -OCR_MODEL_BASE_URL= -OCR_MODEL_API_KEY= -OCR_MODEL_NAME= +SILICONFLOW_OCR_BASE_URL=https://api.siliconflow.cn/v1 +SILICONFLOW_OCR_API_KEY= +SILICONFLOW_OCR_MODEL=PaddlePaddle/PaddleOCR-VL-1.5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5ebe879 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + offline-test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install project + run: | + python -m pip install --upgrade pip + python -m pip install -e '.[dev]' + - name: Compile + run: python -m compileall -q data_agent scripts + - name: Offline tests + env: + DATA_AGENT_DEMO_INBOX: "" + run: python -m pytest -q diff --git a/data_agent/model_adapters/base.py b/data_agent/model_adapters/base.py index c707cfa..b70a2fc 100644 --- a/data_agent/model_adapters/base.py +++ b/data_agent/model_adapters/base.py @@ -2,7 +2,7 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, Field @@ -21,6 +21,23 @@ class ModelProfile(BaseModel): cost_tier: str = "medium" supports_vision: bool = False supports_json: bool = True + endpoint_path: str = "/chat/completions" + input_modalities: list[str] = Field(default_factory=lambda: ["text"]) + json_mode: Literal["required", "preferred", "disabled"] = "preferred" + image_detail: Literal["auto", "low", "high"] = "high" + max_output_tokens: int = 2048 + thinking_mode: Literal["enabled", "disabled", "provider_default"] = "provider_default" + + def effective_input_modalities(self) -> list[str]: + modalities = list(self.input_modalities) + if self.supports_vision and "image" not in modalities: + modalities.append("image") + return modalities + + def effective_json_mode(self) -> Literal["required", "preferred", "disabled"]: + if not self.supports_json: + return "disabled" + return self.json_mode class TaskContext(BaseModel): @@ -58,3 +75,12 @@ class ModelResult(BaseModel): created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) schema_version: str = "model_result_v1" prompt_version: str = "" + requires_review: bool = False + input_metadata: dict[str, Any] = Field(default_factory=dict) + + +class ModelExecution(BaseModel): + """Auditable result of one routed role, including failed cloud attempts.""" + + attempts: list[ModelResult] = Field(default_factory=list) + selected_result: ModelResult diff --git a/data_agent/model_adapters/openai_compatible.py b/data_agent/model_adapters/openai_compatible.py index 5e4244f..70091dd 100644 --- a/data_agent/model_adapters/openai_compatible.py +++ b/data_agent/model_adapters/openai_compatible.py @@ -1,177 +1,280 @@ -"""OpenAI-compatible provider: HTTP calls to chat/completions endpoint.""" +"""Audited OpenAI-compatible Chat Completions provider implementation.""" from __future__ import annotations import base64 import json +import re import time from pathlib import Path from typing import Any import requests +from pydantic import ValidationError from .base import ModelProfile, ModelResult, TaskContext +from .output_schemas import validate_role_output from .prompts import get_prompt_for_role -from .redaction import redact_dict, redact_string, sanitize_model_output_json +from .redaction import redact_dict, redact_string, sanitize_forbidden_keys_deep + + +_MIME_TYPES = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", +} +_MAX_IMAGE_BYTES = 20 * 1024 * 1024 def _encode_image(image_path: str) -> str: path = Path(image_path) - ext = path.suffix.lower().lstrip(".") - mime_map = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg", "gif": "image/gif", "webp": "image/webp"} - mime = mime_map.get(ext, "image/png") - with open(path, "rb") as f: - b64 = base64.b64encode(f.read()).decode("utf-8") - return f"data:{mime};base64,{b64}" + if not path.is_file(): + raise ValueError("Image input is missing or is not a regular file") + size = path.stat().st_size + if size <= 0: + raise ValueError("Image input is empty") + if size > _MAX_IMAGE_BYTES: + raise ValueError(f"Image input exceeds {_MAX_IMAGE_BYTES} byte limit") + mime = _MIME_TYPES.get(path.suffix.lower()) + if mime is None: + raise ValueError(f"Unsupported image type: {path.suffix.lower() or ''}") + return f"data:{mime};base64,{base64.b64encode(path.read_bytes()).decode('ascii')}" -def call_openai_compatible( +def _join_endpoint(base_url: str, endpoint_path: str) -> str: + base = base_url.rstrip("/") + path = "/" + endpoint_path.strip("/") + if base.endswith(path): + return base + return base + path + + +def build_chat_request( profile: ModelProfile, ctx: TaskContext, - env: dict[str, str], + model_name: str, + *, + base_url: str = "", + text_input: str = "", image_path: str = "", -) -> ModelResult: - start = time.time() - base_url = env.get("base_url", "").rstrip("/") - api_key = env.get("api_key", "") - model_name = env.get("model", "") - - if not base_url or not api_key or not model_name: - return ModelResult( - success=False, - role=profile.role, - provider=profile.provider, - mode=ctx.model_mode, - error="Model not configured: missing base_url, api_key, or model env var.", - fallback_used=True, - fallback_from=profile.name, - prompt_version="", - ) - +) -> tuple[str, dict[str, Any]]: + """Build a provider request without headers or secrets.""" system_msg, user_prompt = get_prompt_for_role(profile.role) + messages: list[dict[str, Any]] = [{"role": "system", "content": system_msg}] + modalities = profile.effective_input_modalities() - messages: list[dict[str, Any]] = [ - {"role": "system", "content": system_msg}, - ] - - if profile.supports_vision and ctx.has_image and image_path: - img_url = _encode_image(image_path) + if image_path: + if "image" not in modalities or not ctx.has_image: + raise ValueError(f"Profile '{profile.name}' does not accept image input") messages.append({ "role": "user", "content": [ {"type": "text", "text": user_prompt}, - {"type": "image_url", "image_url": {"url": img_url}}, + { + "type": "image_url", + "image_url": { + "url": _encode_image(image_path), + "detail": profile.image_detail, + }, + }, ], }) else: + if ctx.has_text: + if not text_input.strip(): + raise ValueError("Observation text input is empty") + user_prompt = f"{user_prompt}\n\n\n{text_input}\n" messages.append({"role": "user", "content": user_prompt}) - payload = { + payload: dict[str, Any] = { "model": model_name, "messages": messages, "temperature": 0.0, + "max_tokens": profile.max_output_tokens, } - if profile.supports_json: + if profile.effective_json_mode() in {"required", "preferred"}: payload["response_format"] = {"type": "json_object"} + if profile.thinking_mode != "provider_default": + payload["thinking"] = {"type": profile.thinking_mode} + endpoint = _join_endpoint(base_url, profile.endpoint_path) if base_url else profile.endpoint_path + return endpoint, payload - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}", - } - try: - resp = requests.post( - f"{base_url}/chat/completions", - json=payload, - headers=headers, - timeout=profile.timeout_seconds, - ) - latency_ms = int((time.time() - start) * 1000) +def _content_to_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and item.get("type") in {"text", "output_text"}: + value = item.get("text", "") + if isinstance(value, str): + parts.append(value) + return "\n".join(parts) + return "" - if resp.status_code != 200: - redacted_text = redact_string(resp.text[:500], {api_key}) if api_key else redact_string(resp.text[:500]) - return ModelResult( - success=False, - role=profile.role, - provider=profile.provider, - model=model_name, - mode=ctx.model_mode, - error=f"HTTP {resp.status_code}: {redacted_text}", - latency_ms=latency_ms, - prompt_version="", - ) - - raw = resp.json() - redacted_raw = redact_dict(raw, {api_key} if api_key else None) - - content = "" - choices = raw.get("choices", []) - if choices: - content = choices[0].get("message", {}).get("content", "") - - output_json: dict[str, Any] = {} - parse_error = "" + +def _extract_json_object(text: str) -> dict[str, Any]: + stripped = text.strip() + if not stripped: + raise ValueError("empty_content") + candidates = [stripped] + fenced = re.fullmatch(r"```(?:json)?\s*(.*?)\s*```", stripped, flags=re.DOTALL | re.IGNORECASE) + if fenced: + candidates.insert(0, fenced.group(1).strip()) + for candidate in candidates: try: - if content.strip(): - output_json = json.loads(content) - except json.JSONDecodeError as e: - parse_error = f"Invalid JSON from model: {e}" + value = json.loads(candidate) + if isinstance(value, str): + value = json.loads(value) + if isinstance(value, dict): + return value + except (json.JSONDecodeError, TypeError): + pass - output_json, forbidden_keys = sanitize_model_output_json(output_json) - warnings: list[str] = [] - if forbidden_keys: - warnings.append("model_output_excluded_from_conclusion") + decoder = json.JSONDecoder() + for index, char in enumerate(stripped): + if char != "{": + continue + try: + value, _ = decoder.raw_decode(stripped[index:]) + if isinstance(value, dict): + return value + except json.JSONDecodeError: + continue + raise ValueError("invalid_json_content") - token_usage: dict[str, Any] = {} - usage = raw.get("usage", {}) - if usage: - token_usage = {"prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"), "total_tokens": usage.get("total_tokens")} - success = bool(output_json) and not parse_error +def _parse_response(raw: Any) -> tuple[dict[str, Any], str, list[str], bool]: + if not isinstance(raw, dict): + raise ValueError("response_body_not_object") + choices = raw.get("choices") + if not isinstance(choices, list) or not choices: + raise ValueError("empty_choices") + choice = choices[0] + if not isinstance(choice, dict): + raise ValueError("invalid_choice") + message = choice.get("message") + if not isinstance(message, dict): + raise ValueError("missing_message") + # reasoning_content is deliberately not a final-content fallback. + content = _content_to_text(message.get("content")) + if not content: + content = _content_to_text(message.get("final_content")) + parsed = _extract_json_object(content) + parsed, forbidden = sanitize_forbidden_keys_deep(parsed) + warnings: list[str] = [] + requires_review = False + if forbidden: + warnings.append("model_output_excluded_from_conclusion") + requires_review = True + if choice.get("finish_reason") == "length": + warnings.append("model_output_truncated") + requires_review = True + return parsed, content, warnings, requires_review + + +def call_openai_compatible( + profile: ModelProfile, + ctx: TaskContext, + env: dict[str, str], + image_path: str = "", + text_input: str = "", +) -> ModelResult: + start = time.monotonic() + base_url = env.get("base_url", "").rstrip("/") + api_key = env.get("api_key", "") + model_name = env.get("model", "") + secrets = {api_key} if api_key else None + def failure(error: str, *, latency_ms: int = 0, warnings: list[str] | None = None) -> ModelResult: return ModelResult( - success=success, + success=False, role=profile.role, provider=profile.provider, model=model_name, mode=ctx.model_mode, input_type="image" if image_path else "text", - output_json=output_json, - raw_text=redact_string(content, {api_key} if api_key else None), - raw_response=redacted_raw, - confidence=0.8 if success else 0.0, - warnings=warnings, - error=parse_error, + error=redact_string(error, secrets), + warnings=warnings or [], latency_ms=latency_ms, - token_usage=token_usage, - prompt_version="v1", + prompt_version="v2", + requires_review=True, ) - except requests.Timeout: - latency_ms = int((time.time() - start) * 1000) - return ModelResult( - success=False, - role=profile.role, - provider=profile.provider, - model=model_name, - mode=ctx.model_mode, - error=f"Request timed out after {profile.timeout_seconds}s", - fallback_used=True, - fallback_from=profile.name, - latency_ms=latency_ms, - prompt_version="", + if not base_url or not api_key or not model_name: + return failure("Model not configured: missing base_url, api_key, or model env var.") + + try: + endpoint, payload = build_chat_request( + profile, ctx, model_name, base_url=base_url, text_input=text_input, image_path=image_path ) - except requests.RequestException as e: - latency_ms = int((time.time() - start) * 1000) - redacted_err = redact_string(str(e), {api_key}) if api_key else redact_string(str(e)) + except (OSError, ValueError) as exc: + return failure(f"Invalid model input: {exc}") + + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} + try: + resp = requests.post(endpoint, json=payload, headers=headers, timeout=profile.timeout_seconds) + latency_ms = int((time.monotonic() - start) * 1000) + if resp.status_code != 200: + body = redact_string(getattr(resp, "text", "")[:500], secrets) + return failure(f"HTTP {resp.status_code}: {body}", latency_ms=latency_ms) + try: + raw = resp.json() + except (ValueError, json.JSONDecodeError): + return failure("Response body is not JSON", latency_ms=latency_ms) + redacted_raw = redact_dict(raw, secrets) if isinstance(raw, dict) else {} + try: + parsed, content, warnings, requires_review = _parse_response(raw) + except ValueError as exc: + failed = failure(f"Model response validation failed: {exc}", latency_ms=latency_ms) + failed.raw_response = redacted_raw + return failed + try: + output_json = validate_role_output(profile.role, ctx.data_type, parsed) + except ValidationError as exc: + failed = failure(f"Model response schema validation failed: {exc}", latency_ms=latency_ms, warnings=warnings) + failed.raw_response = redacted_raw + failed.raw_text = redact_string(content, secrets) + return failed + requires_review = requires_review or bool(output_json.get("requires_review")) + usage = raw.get("usage") if isinstance(raw, dict) else None + token_usage: dict[str, Any] = {} + if isinstance(usage, dict): + token_usage = { + "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": usage.get("completion_tokens"), + "total_tokens": usage.get("total_tokens"), + } + else: + warnings.append("token_usage_unavailable") return ModelResult( - success=False, + success=True, role=profile.role, provider=profile.provider, model=model_name, mode=ctx.model_mode, - error=f"Request error: {redacted_err}", - fallback_used=True, - fallback_from=profile.name, + input_type="image" if image_path else "text", + output_json=output_json, + raw_text=redact_string(content, secrets), + raw_response=redacted_raw, + confidence=float(output_json.get("confidence", 0.0)), + warnings=warnings, latency_ms=latency_ms, - prompt_version="", + token_usage=token_usage, + prompt_version="v2", + requires_review=requires_review, + ) + except requests.Timeout: + return failure( + f"Request timed out after {profile.timeout_seconds}s", + latency_ms=int((time.monotonic() - start) * 1000), + ) + except requests.RequestException as exc: + return failure( + f"Request error: {exc}", + latency_ms=int((time.monotonic() - start) * 1000), ) diff --git a/data_agent/model_adapters/output_schemas.py b/data_agent/model_adapters/output_schemas.py new file mode 100644 index 0000000..ae1865c --- /dev/null +++ b/data_agent/model_adapters/output_schemas.py @@ -0,0 +1,64 @@ +"""Strict role-level schemas for model-assisted extraction outputs.""" +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class ExtractionBase(BaseModel): + model_config = ConfigDict(extra="allow", strict=True) + + requires_review: bool + confidence: float = Field(ge=0.0, le=1.0) + uncertainties: list[str] + + +class ObservationExtraction(ExtractionBase): + factual_observations: list[str] + trend_statements: list[str] + interpretation_candidates: list[str] + operator_notes: list[str] + sample_ids: list[str] + time_expressions: list[str] + phenomenon_types: list[str] + + +class ChartVisionExtraction(ExtractionBase): + image_kind: str + chart_type: str + x_axis_label: str + y_axis_label: str + detected_units: list[str] + legend_text: list[str] + visible_series_count: int = Field(ge=0) + visible_peak_candidates: list[Any] + + +class SurfaceVisionExtraction(ExtractionBase): + image_kind: str + detected_objects: list[str] + visible_features: list[str] + scale_bar_text: str + annotation_text: list[str] + + +class OcrExtraction(ExtractionBase): + text_blocks: list[str] + detected_units: list[str] + axis_candidates: list[str] + unreadable_regions: list[str] + + +def validate_role_output(role: str, data_type: str, output: dict[str, Any]) -> dict[str, Any]: + if role in {"fast", "observation"}: + schema = ObservationExtraction + elif role == "ocr": + schema = OcrExtraction + elif role == "vision" and data_type == "visual_image": + schema = SurfaceVisionExtraction + elif role == "vision": + schema = ChartVisionExtraction + else: + raise ValueError(f"No output schema registered for role '{role}'") + return schema.model_validate(output).model_dump(mode="json") diff --git a/data_agent/model_adapters/profiles.py b/data_agent/model_adapters/profiles.py index afa4e51..6b1b798 100644 --- a/data_agent/model_adapters/profiles.py +++ b/data_agent/model_adapters/profiles.py @@ -12,7 +12,7 @@ def load_profiles(config_path: Optional[Path] = None) -> dict[str, ModelProfile]: if config_path is None: - config_path = Path("model_profiles.yaml") + config_path = Path(os.environ.get("DATA_AGENT_MODEL_PROFILES", "model_profiles.yaml")) if not config_path.exists(): return {} with open(config_path, "r", encoding="utf-8") as f: diff --git a/data_agent/model_adapters/prompts.py b/data_agent/model_adapters/prompts.py index e8abee6..2e1d6b5 100644 --- a/data_agent/model_adapters/prompts.py +++ b/data_agent/model_adapters/prompts.py @@ -21,6 +21,9 @@ - legend_text - visible_series_count - text_blocks (all visible text) +- axis_candidates (list of axis label candidates) +- unreadable_regions (list of ambiguous or unreadable areas) +- uncertainties (list of ambiguity descriptions) - requires_review (true if any text is ambiguous) - confidence (0.0-1.0) @@ -71,12 +74,13 @@ OBSERVATION_TEXT_PROMPT = """Analyze this observation text and extract: - factual_observations (list of observed facts) -- trend_or_statements (trend-like descriptions) +- trend_statements (trend-like descriptions) - interpretation_candidates (phrases suggesting speculation, with uncertainty markers like "可能", "或许", "大概") - operator_notes - sample_ids - time_expressions - phenomenon_types +- uncertainties (list of ambiguity descriptions) - requires_review - confidence (0.0-1.0) diff --git a/data_agent/model_adapters/redaction.py b/data_agent/model_adapters/redaction.py index da5d047..c62d1e2 100644 --- a/data_agent/model_adapters/redaction.py +++ b/data_agent/model_adapters/redaction.py @@ -13,6 +13,8 @@ def _get_known_secret_values() -> set[str]: for env_var in ( "BEST_MODEL_API_KEY", "FAST_MODEL_API_KEY", "VISION_MODEL_API_KEY", "OCR_MODEL_API_KEY", + "DEEPSEEK_TEXT_API_KEY", "VOLCENGINE_VISION_API_KEY", + "SILICONFLOW_OCR_API_KEY", ): val = os.environ.get(env_var, "") if val: diff --git a/data_agent/process.py b/data_agent/process.py index 2ab55b7..8549268 100644 --- a/data_agent/process.py +++ b/data_agent/process.py @@ -31,7 +31,7 @@ from .processors.observation_text import process_observation_text from .processors.visual_image import process_visual_image from .processors.metadata import process_metadata -from .model_adapters.base import TaskContext, ModelResult +from .model_adapters.base import TaskContext, ModelExecution, ModelResult from .model_adapters.router import route_model_calls, get_fallback_chain from .model_adapters.profiles import load_profiles, resolve_profile_env, is_profile_available from .model_adapters.stubs import STUB_REGISTRY @@ -142,6 +142,36 @@ def _get_profiles() -> dict: return load_profiles() +_MAX_OBSERVATION_BYTES = 1024 * 1024 +_MAX_OBSERVATION_CHARS = 20_000 + + +def _read_observation_input(path: Path) -> tuple[str, dict, list[str], str]: + """Read bounded UTF-8 observation text without leaking the local path.""" + try: + size = path.stat().st_size + if size <= 0: + return "", {"original_bytes": size, "sent_characters": 0}, [], "Observation text input is empty" + with open(path, "rb") as f: + raw = f.read(_MAX_OBSERVATION_BYTES + 1) + byte_truncated = len(raw) > _MAX_OBSERVATION_BYTES + raw = raw[:_MAX_OBSERVATION_BYTES] + text = raw.decode("utf-8") + except UnicodeDecodeError: + return "", {"original_bytes": 0, "sent_characters": 0}, [], "Observation text is not valid UTF-8" + except OSError as exc: + return "", {"original_bytes": 0, "sent_characters": 0}, [], f"Cannot read observation text: {exc.__class__.__name__}" + char_truncated = len(text) > _MAX_OBSERVATION_CHARS + text = text[:_MAX_OBSERVATION_CHARS] + truncated = byte_truncated or char_truncated or size > len(raw) + warnings = ["input_truncated"] if truncated else [] + return text, { + "original_bytes": size, + "sent_characters": len(text), + "truncated": truncated, + }, warnings, "" + + def _call_model_roles( task_dir: Path, conn, @@ -178,103 +208,94 @@ def _call_model_roles( profiles = _get_profiles() image_path_arg = str(file_path) if has_image and file_path.exists() else "" + text_input = "" + input_metadata: dict = {} + input_warnings: list[str] = [] + input_error = "" + if has_text: + text_input, input_metadata, input_warnings, input_error = _read_observation_input(file_path) for role in roles: - result = _execute_model_role(role, profiles, ctx, image_path_arg) - if result is None: + if input_error: + failed = ModelResult( + success=False, role=role, provider="none", mode=model_mode, + error=input_error, requires_review=True, input_metadata=input_metadata, + ) + execution = ModelExecution(attempts=[failed], selected_result=failed) + else: + execution = _execute_model_role(role, profiles, ctx, image_path_arg, text_input) + if execution is None: continue - - prefix = f"run_{run_short}__" if run_short else "" - output_name = f"{prefix}model_result_{role}.json" - output_path = task_dir / "derived" / output_name - - result_dict = result.model_dump(mode="json") - result_dict = sanitize_and_redact_model_result(result_dict) - with open(output_path, "w", encoding="utf-8") as f: - json.dump(result_dict, f, ensure_ascii=False, indent=2) - - model_derived = DataObject( - task_id=tid, - data_type=DataType.MODEL_RESULT, - subtype=f"model_result_{role}", - confidence=result.confidence, - derived_from=[l1_obj.object_id], - lifecycle=LifecycleLevel.L2, - data_schema={ - "output_file": output_name, - "role": role, - "provider": result.provider, - "model": result.model, - "mode": result.mode, - "fallback_used": result.fallback_used, - "prompt_version": result.prompt_version, - }, - ) - derived_objects.append(model_derived) - - model_run = ProcessingRun( - task_id=tid, - tool_name=f"model:{role}", - tool_version="0.1.0", - input_data_ids=[l1_obj.object_id], - output_data_ids=[model_derived.object_id], - parameters={ - "provider": result.provider, - "model": result.model, - "role": role, - "mode": result.mode, - "fallback_used": result.fallback_used, - }, - status=ProcessingStatus.SUCCEEDED if result.success else ProcessingStatus.FAILED, - warnings=result.warnings, - errors=[result.error] if result.error else [], - ) - model_runs.append(model_run) - - if result.fallback_used: - flags.append(QualityFlag( - task_id=tid, - severity="warning", - target_type="model_result", - target_id=model_derived.object_id, - message=f"fallback_used: Model role '{role}' used fallback from '{result.fallback_from}'.", - evidence=str(result.warnings), - requires_review=False, - confidence=result.confidence, - )) - if not result.success: - flags.append(QualityFlag( - task_id=tid, - severity="warning", - target_type="model_result", - target_id=model_derived.object_id, - message=f"model_unavailable: {role} returned error: {result.error}", - evidence=str(result.error), - requires_review=False, - confidence=0.0, - )) - if result.confidence < 0.5: - flags.append(QualityFlag( - task_id=tid, - severity="info", - target_type="model_result", - target_id=model_derived.object_id, - message=f"low_confidence_model_output: {role} confidence={result.confidence}.", - evidence=str(result.output_json), - requires_review=True, - confidence=result.confidence, + attempts = execution.attempts or [execution.selected_result] + for attempt_index, result in enumerate(attempts): + result.input_metadata.update(input_metadata) + for warning in input_warnings: + if warning not in result.warnings: + result.warnings.append(warning) + if input_warnings: + result.requires_review = True + + selected = attempt_index == len(attempts) - 1 + result_key = role if selected else f"{role}_cloud_attempt" + prefix = f"run_{run_short}__" if run_short else "" + output_name = f"{prefix}model_result_{result_key}.json" + output_path = task_dir / "derived" / output_name + result_dict = sanitize_and_redact_model_result(result.model_dump(mode="json")) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(result_dict, f, ensure_ascii=False, indent=2) + + model_derived = DataObject( + task_id=tid, data_type=DataType.MODEL_RESULT, + subtype=f"model_result_{result_key}", confidence=result.confidence, + derived_from=[l1_obj.object_id], lifecycle=LifecycleLevel.L2, + data_schema={ + "output_file": output_name, "role": role, "provider": result.provider, + "model": result.model, "mode": result.mode, + "fallback_used": result.fallback_used, "prompt_version": result.prompt_version, + "attempt_kind": "selected" if selected else "cloud_attempt", + }, + ) + derived_objects.append(model_derived) + model_runs.append(ProcessingRun( + task_id=tid, tool_name=f"model:{role}", tool_version="0.2.0", + input_data_ids=[l1_obj.object_id], output_data_ids=[model_derived.object_id], + parameters={ + "provider": result.provider, "model": result.model, "role": role, + "mode": result.mode, "fallback_used": result.fallback_used, + "attempt_kind": "selected" if selected else "cloud_attempt", + }, + status=ProcessingStatus.SUCCEEDED if result.success else ProcessingStatus.FAILED, + warnings=result.warnings, errors=[result.error] if result.error else [], )) - for w in result.warnings: - if w == "model_output_excluded_from_conclusion": + + if result.fallback_used: flags.append(QualityFlag( - task_id=tid, - severity="warning", - target_type="model_result", + task_id=tid, severity="warning", target_type="model_result", target_id=model_derived.object_id, - message="model_output_excluded_from_conclusion: Forbidden output keys removed.", - requires_review=True, - confidence=0.5, + message=f"fallback_used: Model role '{role}' used fallback from '{result.fallback_from}'.", + evidence=str(result.warnings), requires_review=True, confidence=result.confidence, )) + if not result.success: + flags.append(QualityFlag( + task_id=tid, severity="warning", target_type="model_result", + target_id=model_derived.object_id, + message=f"model_unavailable: {role} returned error: {result.error}", + evidence=str(result.error), requires_review=True, confidence=0.0, + )) + if result.confidence < 0.5: + flags.append(QualityFlag( + task_id=tid, severity="info", target_type="model_result", + target_id=model_derived.object_id, + message=f"low_confidence_model_output: {role} confidence={result.confidence}.", + evidence=str(result.output_json), requires_review=True, confidence=result.confidence, + )) + for warning in result.warnings: + if warning in {"model_output_excluded_from_conclusion", "model_output_truncated", "input_truncated"}: + flags.append(QualityFlag( + task_id=tid, severity="warning", target_type="model_result", + target_id=model_derived.object_id, message=f"{warning}: Model result requires review.", + requires_review=True, confidence=result.confidence, + )) return derived_objects, model_runs, flags @@ -284,17 +305,20 @@ def _execute_model_role( profiles: dict, ctx: TaskContext, image_path: str = "", -) -> ModelResult | None: + text_input: str = "", +) -> ModelExecution | None: if ctx.model_mode == "local": stub_func = STUB_REGISTRY.get(role) or STUB_REGISTRY.get("local_stub") if stub_func: - return stub_func(ctx) + result = stub_func(ctx) + return ModelExecution(attempts=[result], selected_result=result) return None if ctx.model_mode == "cloud": if role in profiles and is_profile_available(profiles[role]): env = resolve_profile_env(profiles[role]) - return call_openai_compatible(profiles[role], ctx, env, image_path) + result = call_openai_compatible(profiles[role], ctx, env, image_path, text_input) + return ModelExecution(attempts=[result], selected_result=result) result = ModelResult( success=False, role=role, @@ -303,26 +327,28 @@ def _execute_model_role( error=f"Model profile '{role}' not configured or unavailable.", fallback_used=False, ) - return result + return ModelExecution(attempts=[result], selected_result=result) if ctx.model_mode == "auto": if role in profiles and is_profile_available(profiles[role]): env = resolve_profile_env(profiles[role]) - result = call_openai_compatible(profiles[role], ctx, env, image_path) + result = call_openai_compatible(profiles[role], ctx, env, image_path, text_input) if result.success: - return result + return ModelExecution(attempts=[result], selected_result=result) fallback_chain = get_fallback_chain(role, profiles) - fallback_result = result - fallback_result.fallback_used = True - fallback_result.fallback_from = role for fallback_role in fallback_chain: stub_func = STUB_REGISTRY.get(fallback_role) if stub_func: fb = stub_func(ctx) - fb.fallback_from = role + fb.fallback_from = f"{profiles[role].provider}:{profiles[role].name}" fb.fallback_used = True - return fb - return fallback_result + return ModelExecution(attempts=[result, fb], selected_result=fb) + return ModelExecution(attempts=[result], selected_result=result) + unavailable = ModelResult( + success=False, role=role, provider="none", mode="auto", + error=f"Model profile '{role}' not configured or unavailable.", + requires_review=True, + ) fallback_chain = get_fallback_chain(role, profiles) for fallback_role in fallback_chain: stub_func = STUB_REGISTRY.get(fallback_role) @@ -330,7 +356,7 @@ def _execute_model_role( result = stub_func(ctx) result.fallback_from = role result.fallback_used = True - return result + return ModelExecution(attempts=[unavailable, result], selected_result=result) return None return None diff --git a/data_agent/ui/app.py b/data_agent/ui/app.py index 7e248f5..e7967c9 100644 --- a/data_agent/ui/app.py +++ b/data_agent/ui/app.py @@ -361,6 +361,8 @@ def _error_redacted(exc: Exception) -> str: st.caption(f"Token usage: {audit['token_usage']}") if audit.get("schema_version"): st.caption(f"Schema: {audit['schema_version']} | Prompt: {audit['prompt_version']}") + if audit.get("input_metadata"): + st.caption(f"Input metadata: {audit['input_metadata']}") with st.expander("Risk", expanded=False): if risk.get("error"): diff --git a/data_agent/ui/preview.py b/data_agent/ui/preview.py index 989c3ff..a94c59d 100644 --- a/data_agent/ui/preview.py +++ b/data_agent/ui/preview.py @@ -117,12 +117,13 @@ def preview_model_result(path: Path) -> dict[str, Any] | None: "token_usage": data.get("token_usage", {}), "schema_version": data.get("schema_version", ""), "prompt_version": data.get("prompt_version", ""), + "input_metadata": data.get("input_metadata", {}), } risk = { "warnings": data.get("warnings", []), "error": data.get("error", ""), - "requires_review": output.get("requires_review", False), + "requires_review": data.get("requires_review", output.get("requires_review", False)), "ocr_unavailable": output.get("ocr_unavailable", False), "vision_unavailable": output.get("vision_unavailable", False), } diff --git a/data_agent/ui/security.py b/data_agent/ui/security.py index 0d0e8a0..1f70ac3 100644 --- a/data_agent/ui/security.py +++ b/data_agent/ui/security.py @@ -13,6 +13,8 @@ _KEY_ENV_NAMES = { "BEST_MODEL_API_KEY", "FAST_MODEL_API_KEY", "VISION_MODEL_API_KEY", "OCR_MODEL_API_KEY", + "DEEPSEEK_TEXT_API_KEY", "VOLCENGINE_VISION_API_KEY", + "SILICONFLOW_OCR_API_KEY", } diff --git a/model_profiles.yaml.example b/model_profiles.yaml.example index b667584..1bbcb1b 100644 --- a/model_profiles.yaml.example +++ b/model_profiles.yaml.example @@ -1,56 +1,59 @@ profiles: fast: role: fast - provider: openai_compatible - base_url_env: FAST_MODEL_BASE_URL - api_key_env: FAST_MODEL_API_KEY - model_env: FAST_MODEL_NAME + provider: deepseek + base_url_env: DEEPSEEK_TEXT_BASE_URL + api_key_env: DEEPSEEK_TEXT_API_KEY + model_env: DEEPSEEK_TEXT_MODEL + endpoint_path: /chat/completions + input_modalities: [text] + json_mode: required + thinking_mode: disabled + max_output_tokens: 2048 enabled: true priority: 10 - fallback: ["local_stub"] - timeout_seconds: 45 - cost_tier: low - supports_vision: false - supports_json: true - - best: - role: best - provider: openai_compatible - base_url_env: BEST_MODEL_BASE_URL - api_key_env: BEST_MODEL_API_KEY - model_env: BEST_MODEL_NAME - enabled: true - priority: 10 - fallback: ["fast", "local_stub"] - timeout_seconds: 90 - cost_tier: high + fallback: [local_stub] + timeout_seconds: 60 + cost_tier: medium supports_vision: false supports_json: true vision: role: vision - provider: openai_compatible_vision - base_url_env: VISION_MODEL_BASE_URL - api_key_env: VISION_MODEL_API_KEY - model_env: VISION_MODEL_NAME + provider: volcengine_ark + base_url_env: VOLCENGINE_VISION_BASE_URL + api_key_env: VOLCENGINE_VISION_API_KEY + model_env: VOLCENGINE_VISION_MODEL + endpoint_path: /chat/completions + input_modalities: [text, image] + json_mode: disabled + image_detail: high + thinking_mode: provider_default + max_output_tokens: 2048 enabled: true priority: 10 - fallback: ["local_stub"] + fallback: [local_stub] timeout_seconds: 90 cost_tier: medium supports_vision: true - supports_json: true + supports_json: false ocr: role: ocr - provider: openai_compatible_vision - base_url_env: OCR_MODEL_BASE_URL - api_key_env: OCR_MODEL_API_KEY - model_env: OCR_MODEL_NAME + provider: siliconflow + base_url_env: SILICONFLOW_OCR_BASE_URL + api_key_env: SILICONFLOW_OCR_API_KEY + model_env: SILICONFLOW_OCR_MODEL + endpoint_path: /chat/completions + input_modalities: [text, image] + json_mode: disabled + image_detail: high + thinking_mode: provider_default + max_output_tokens: 2048 enabled: true priority: 10 - fallback: ["local_ocr_stub", "local_stub"] - timeout_seconds: 60 + fallback: [local_ocr_stub, local_stub] + timeout_seconds: 90 cost_tier: medium supports_vision: true - supports_json: true + supports_json: false diff --git a/scripts/audit_secret_leaks.py b/scripts/audit_secret_leaks.py new file mode 100644 index 0000000..8a03f2c --- /dev/null +++ b/scripts/audit_secret_leaks.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Scan repository and evidence artifacts for exact configured secret values.""" +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import zipfile +from pathlib import Path + + +SECRET_NAMES = ( + "DEEPSEEK_TEXT_API_KEY", + "VOLCENGINE_VISION_API_KEY", + "SILICONFLOW_OCR_API_KEY", +) + + +def _git_paths(repo: Path, *args: str) -> list[Path]: + result = subprocess.run(["git", *args, "-z"], cwd=repo, capture_output=True, check=True) + return [repo / item.decode("utf-8", "surrogateescape") for item in result.stdout.split(b"\0") if item] + + +def _safe_read(path: Path) -> bytes: + try: + return path.read_bytes() + except OSError: + return b"" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--workspace", type=Path) + parser.add_argument("--zip", dest="zip_path", type=Path) + args = parser.parse_args() + repo = args.repo.resolve() + secrets = {name: os.environ.get(name, "").encode() for name in SECRET_NAMES} + configured = {name: value for name, value in secrets.items() if value} + + sources: list[tuple[str, str, bytes]] = [] + for path in _git_paths(repo, "ls-files"): + if path.is_file(): + sources.append(("git_tracked", str(path.relative_to(repo)), _safe_read(path))) + for path in _git_paths(repo, "ls-files", "--others", "--exclude-standard"): + if path.is_file(): + sources.append(("git_untracked", str(path.relative_to(repo)), _safe_read(path))) + diff = subprocess.run(["git", "diff", "--binary"], cwd=repo, capture_output=True, check=True).stdout + sources.append(("git_diff", "working_tree", diff)) + if args.workspace and args.workspace.exists(): + for path in args.workspace.rglob("*"): + if path.is_file(): + sources.append(("workspace", str(path), _safe_read(path))) + if args.zip_path and args.zip_path.is_file(): + with zipfile.ZipFile(args.zip_path) as archive: + for name in archive.namelist(): + if not name.endswith("/"): + sources.append(("zip", name, archive.read(name))) + + failed = False + for env_name in SECRET_NAMES: + value = configured.get(env_name) + if not value: + print(f"{env_name}: not configured") + continue + matches = [(kind, label) for kind, label, content in sources if value in content] + if matches: + failed = True + print(f"{env_name}: exact match found ({len(matches)} location(s))") + for kind, label in matches: + print(f" {kind}: {label}") + else: + print(f"{env_name}: no exact match found") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_real_api_check.py b/scripts/run_real_api_check.py new file mode 100644 index 0000000..327e3fe --- /dev/null +++ b/scripts/run_real_api_check.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Run safe, synthetic, opt-in real-provider smoke checks.""" +from __future__ import annotations + +import argparse +import json +import os +import sqlite3 +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +import matplotlib.pyplot as plt +import requests +import yaml +from PIL import Image, ImageDraw + +from data_agent.db import init_db +from data_agent.export import export_task +from data_agent.ingest import ingest_inbox +from data_agent.model_adapters.redaction import redact_string +from data_agent.process import process_single_task +from data_agent.validation import validate_task + + +SCENARIOS = ("deepseek-text", "volcengine-vision", "siliconflow-ocr", "auto-fallback") +CONFIG = { + "deepseek-text": { + "role": "fast", "provider": "deepseek", "prefix": "DEEPSEEK_TEXT", + "modalities": ["text"], "json_mode": "required", "thinking_mode": "disabled", + }, + "volcengine-vision": { + "role": "vision", "provider": "volcengine_ark", "prefix": "VOLCENGINE_VISION", + "modalities": ["text", "image"], "json_mode": "disabled", "thinking_mode": "provider_default", + }, + "siliconflow-ocr": { + "role": "ocr", "provider": "siliconflow", "prefix": "SILICONFLOW_OCR", + "modalities": ["text", "image"], "json_mode": "disabled", "thinking_mode": "provider_default", + }, +} + + +def _required(prefix: str) -> tuple[dict[str, str], list[str]]: + names = [f"{prefix}_BASE_URL", f"{prefix}_API_KEY", f"{prefix}_MODEL"] + values = {name: os.environ.get(name, "") for name in names} + return values, [name for name, value in values.items() if not value] + + +def _preflight_models(values: dict[str, str], requested_model: str) -> bool: + base = values[next(name for name in values if name.endswith("_BASE_URL"))].rstrip("/") + key = values[next(name for name in values if name.endswith("_API_KEY"))] + try: + response = requests.get( + f"{base}/models", headers={"Authorization": f"Bearer {key}"}, timeout=30 + ) + if response.status_code != 200: + return False + body = response.json() + models = body.get("data", []) if isinstance(body, dict) else [] + return any(isinstance(item, dict) and item.get("id") == requested_model for item in models) + except (requests.RequestException, ValueError): + return False + + +def _write_profile(path: Path, spec: dict[str, object], prefix: str) -> None: + role = str(spec["role"]) + config = {"profiles": {role: { + "role": role, "provider": spec["provider"], + "base_url_env": f"{prefix}_BASE_URL", "api_key_env": f"{prefix}_API_KEY", + "model_env": f"{prefix}_MODEL", "endpoint_path": "/chat/completions", + "input_modalities": spec["modalities"], "json_mode": spec["json_mode"], + "image_detail": "high", "thinking_mode": spec["thinking_mode"], + "max_output_tokens": 2048, "enabled": True, + "fallback": ["local_ocr_stub", "local_stub"] if role == "ocr" else ["local_stub"], + "timeout_seconds": 90, "supports_vision": "image" in spec["modalities"], + "supports_json": spec["json_mode"] != "disabled", + }}} + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + + +def _make_input(inbox: Path, scenario: str) -> None: + if scenario in {"deepseek-text", "auto-fallback"}: + (inbox / "observation_smoke.txt").write_text( + "2026-01-01 10:00，样品 SYN-01 表面可见轻微浑浊。可能与温度变化有关。操作员备注：仅为合成测试。", + encoding="utf-8", + ) + elif scenario == "volcengine-vision": + x = [1, 2, 3, 4, 5] + fig, ax = plt.subplots(figsize=(6, 4)) + ax.plot(x, [1, 3, 2, 4, 3], label="Synthetic A") + ax.plot(x, [2, 2, 3, 3, 4], label="Synthetic B") + ax.set(title="Synthetic Material Chart", xlabel="Time (s)", ylabel="Signal (a.u.)") + ax.legend() + fig.tight_layout() + fig.savefig(inbox / "ftir_chart_smoke.png") + plt.close(fig) + else: + image = Image.new("RGB", (900, 300), "white") + draw = ImageDraw.Draw(image) + draw.text((40, 50), "Synthetic OCR: Sample SYN-01", fill="black") + draw.text((40, 120), "Wavelength 550 nm", fill="black") + draw.text((40, 190), "Signal 0.82 a.u.", fill="black") + image.save(inbox / "uvvis_chart_ocr.png") + + +def _find_result(workspace: Path, task_id: str, role: str, provider: str) -> tuple[Path | None, dict]: + for path in sorted((workspace / "tasks" / task_id / "derived").glob(f"*model_result_{role}.json"), reverse=True): + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("provider") == provider: + return path, data + return None, {} + + +def _run_one(scenario: str) -> int: + workspace = Path(tempfile.mkdtemp(prefix=f"material-agent-{scenario}-")) + inbox = workspace / "synthetic_inbox" + inbox.mkdir() + _make_input(inbox, scenario) + + if scenario == "auto-fallback": + spec = CONFIG["deepseek-text"] + prefix = "DATA_AGENT_FALLBACK_TEST" + os.environ.update({ + f"{prefix}_BASE_URL": "https://deterministic.invalid", + f"{prefix}_API_KEY": "synthetic-test-key", + f"{prefix}_MODEL": "synthetic-model", + }) + else: + spec = CONFIG[scenario] + prefix = str(spec["prefix"]) + values, missing = _required(prefix) + if missing: + print(f"scenario={scenario} status=SKIPPED reason=missing_environment workspace={workspace}") + return 2 + requested = values[f"{prefix}_MODEL"] + if scenario in {"deepseek-text", "siliconflow-ocr"}: + available = _preflight_models(values, requested) + print(f"scenario={scenario} provider={spec['provider']} requested_model={'available' if available else 'unavailable'}") + if not available: + print(f"scenario={scenario} status=FAIL reason=model_preflight_failed workspace={workspace}") + return 1 + + profile_path = workspace / "model_profiles.yaml" + _write_profile(profile_path, spec, prefix) + os.environ["DATA_AGENT_MODEL_PROFILES"] = str(profile_path) + conn = init_db(workspace) + [task_id] = ingest_inbox(inbox, workspace, conn) + conn.close() + + if scenario == "auto-fallback": + import data_agent.model_adapters.openai_compatible as adapter + original_post = adapter.requests.post + adapter.requests.post = lambda *args, **kwargs: SimpleNamespace(status_code=429, text="synthetic rate limit") + try: + process_single_task(workspace, task_id, "auto") + finally: + adapter.requests.post = original_post + provider = "local_stub" + role = "fast" + else: + process_single_task(workspace, task_id, "cloud") + provider = str(spec["provider"]) + role = str(spec["role"]) + + result_path, result = _find_result(workspace, task_id, role, provider) + validation = validate_task(workspace, task_id) + export = export_task(workspace, task_id) + conn = sqlite3.connect(workspace / "agent.sqlite") + row = conn.execute( + "SELECT run_id, status FROM processing_runs WHERE task_id=? AND tool_name=? ORDER BY created_at DESC LIMIT 1", + (task_id, f"model:{role}"), + ).fetchone() + conn.close() + status = "PASS" if result_path and result.get("success") and validation.status != "error" and export.success else "FAIL" + if scenario == "auto-fallback": + attempts = list((workspace / "tasks" / task_id / "derived").glob("*cloud_attempt.json")) + status = "PASS" if attempts and result.get("fallback_used") and validation.status != "error" and export.success else "FAIL" + print(" ".join([ + f"scenario={scenario}", f"provider={provider}", f"model={result.get('model', '')}", + f"status={status}", f"task_id={task_id}", f"run_id={row[0] if row else ''}", + f"run_status={row[1] if row else ''}", f"result_path={result_path or ''}", + f"latency_ms={result.get('latency_ms', 0)}", f"token_usage_available={bool(result.get('token_usage'))}", + f"fallback={bool(result.get('fallback_used'))}", f"validation={validation.status}", + f"export={export.success}", f"workspace={workspace}", + ])) + return 0 if status == "PASS" else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scenario", required=True, choices=(*SCENARIOS, "all")) + args = parser.parse_args() + scenarios = SCENARIOS if args.scenario == "all" else (args.scenario,) + results = [_run_one(scenario) for scenario in scenarios] + if 1 in results: + return 1 + if 2 in results: + return 2 + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"status=FAIL error={redact_string(str(exc))}") + raise SystemExit(1) diff --git a/tests/test_model_provider_contracts.py b/tests/test_model_provider_contracts.py new file mode 100644 index 0000000..dbb44f1 --- /dev/null +++ b/tests/test_model_provider_contracts.py @@ -0,0 +1,196 @@ +"""Provider payload, response parsing, and audited fallback contracts.""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from data_agent.db import init_db +from data_agent.ingest import ingest_inbox +from data_agent.model_adapters.base import ModelExecution, ModelProfile, ModelResult, TaskContext +from data_agent.model_adapters.openai_compatible import build_chat_request, call_openai_compatible +from data_agent.process import _execute_model_role, process_single_task + + +def observation_output(**overrides): + value = { + "factual_observations": ["sample visible"], "trend_statements": [], + "interpretation_candidates": [], "operator_notes": [], "sample_ids": ["S-01"], + "time_expressions": [], "phenomenon_types": [], "uncertainties": [], + "requires_review": False, "confidence": 0.9, + } + value.update(overrides) + return value + + +def ocr_output(**overrides): + value = { + "text_blocks": ["Absorbance"], "detected_units": ["a.u."], + "axis_candidates": ["Wavelength"], "unreadable_regions": [], + "uncertainties": [], "requires_review": False, "confidence": 0.92, + } + value.update(overrides) + return value + + +def profile(role="fast", provider="deepseek", image=False, json_mode="required"): + return ModelProfile( + name=role, role=role, provider=provider, base_url_env="URL", api_key_env="KEY", + model_env="MODEL", supports_vision=image, supports_json=json_mode != "disabled", + input_modalities=["text", "image"] if image else ["text"], json_mode=json_mode, + thinking_mode="disabled" if provider == "deepseek" else "provider_default", + ) + + +def env(model="deepseek-v4-pro"): + return {"base_url": "https://provider.example/v1", "api_key": "test-secret", "model": model} + + +def response(content, *, usage=True, finish_reason="stop"): + resp = MagicMock() + resp.status_code = 200 + body = {"choices": [{"finish_reason": finish_reason, "message": {"content": content}}]} + if usage: + body["usage"] = {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3} + resp.json.return_value = body + return resp + + +def test_deepseek_payload_contains_real_text_and_json_mode(): + p = profile() + ctx = TaskContext(task_id="t", data_type="descriptive_observation_text", has_text=True, model_mode="cloud") + endpoint, payload = build_chat_request(p, ctx, "deepseek-v4-pro", base_url="https://api.deepseek.com/", text_input="S-01 became cloudy") + assert endpoint == "https://api.deepseek.com/chat/completions" + assert payload["model"] == "deepseek-v4-pro" + assert payload["response_format"] == {"type": "json_object"} + assert payload["thinking"] == {"type": "disabled"} + assert "" in payload["messages"][1]["content"] + assert "S-01 became cloudy" in payload["messages"][1]["content"] + + +@pytest.mark.parametrize("provider_name,model", [ + ("volcengine_ark", "ep-test"), + ("siliconflow", "PaddlePaddle/PaddleOCR-VL-1.5"), +]) +def test_multimodal_provider_payload(provider_name, model, tmp_path): + image = tmp_path / "chart.png" + image.write_bytes(b"valid-test-bytes") + p = profile("ocr", provider_name, image=True, json_mode="disabled") + ctx = TaskContext(task_id="t", data_type="chart_image_input", has_image=True, model_mode="cloud") + endpoint, payload = build_chat_request(p, ctx, model, base_url="https://provider.example/v1", image_path=str(image)) + assert endpoint == "https://provider.example/v1/chat/completions" + assert payload["model"] == model + assert "response_format" not in payload + parts = payload["messages"][1]["content"] + image_part = next(part for part in parts if part["type"] == "image_url") + assert image_part["image_url"]["url"].startswith("data:image/png;base64,") + assert image_part["image_url"]["detail"] == "high" + + +@pytest.mark.parametrize("wrapped", [ + lambda value: json.dumps(value), + lambda value: "```json\n" + json.dumps(value) + "\n```", + lambda value: "Result follows: " + json.dumps(value) + " End.", + lambda value: json.dumps(json.dumps(value)), +]) +def test_robust_json_extraction(wrapped): + p = profile() + ctx = TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud") + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=response(wrapped(observation_output()))): + result = call_openai_compatible(p, ctx, env()) + assert result.success + assert result.output_json["sample_ids"] == ["S-01"] + + +def test_content_array_and_missing_usage(): + content = [{"type": "text", "text": json.dumps(observation_output())}] + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=response(content, usage=False)): + result = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert result.success + assert "token_usage_unavailable" in result.warnings + + +def test_reasoning_content_is_not_used_as_final_content(): + resp = response("") + resp.json.return_value["choices"][0]["message"]["reasoning_content"] = json.dumps(observation_output()) + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=resp): + result = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert not result.success + assert "empty_content" in result.error + + +@pytest.mark.parametrize("status", [400, 401, 429, 500]) +def test_http_failures_are_explicit_and_redacted(status): + resp = MagicMock(status_code=status, text="Bearer test-secret") + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=resp): + result = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert not result.success + assert f"HTTP {status}" in result.error + assert "test-secret" not in result.error + + +def test_schema_missing_field_and_finish_length(): + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=response('{"confidence": 0.5}')): + result = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert not result.success + assert "schema validation" in result.error + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=response(json.dumps(observation_output()), finish_reason="length")): + truncated = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert truncated.success and truncated.requires_review + assert "model_output_truncated" in truncated.warnings + + +def test_response_body_not_json_and_timeout(): + bad = MagicMock(status_code=200) + bad.json.side_effect = ValueError("bad") + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=bad): + result = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert not result.success and "not JSON" in result.error + with patch("data_agent.model_adapters.openai_compatible.requests.post", side_effect=requests.Timeout): + timed = call_openai_compatible(profile(), TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="cloud"), env()) + assert not timed.success and "timed out" in timed.error + + +def test_auto_execution_preserves_failed_cloud_attempt(monkeypatch): + p = profile() + monkeypatch.setenv("URL", "https://provider.example/v1") + monkeypatch.setenv("KEY", "test-secret") + monkeypatch.setenv("MODEL", "deepseek-v4-pro") + failed = ModelResult(success=False, role="fast", provider="deepseek", mode="auto", error="HTTP 429", requires_review=True) + with patch("data_agent.process.call_openai_compatible", return_value=failed): + execution = _execute_model_role("fast", {"fast": p}, TaskContext(task_id="t", data_type="descriptive_observation_text", model_mode="auto"), text_input="test") + assert execution is not None + assert len(execution.attempts) == 2 + assert not execution.attempts[0].success + assert execution.selected_result.fallback_used + assert execution.selected_result.provider.startswith("local") + + +def test_auto_persists_cloud_attempt_and_fallback(tmp_path): + inbox = tmp_path / "inbox" + workspace = tmp_path / "workspace" + inbox.mkdir() + workspace.mkdir() + (inbox / "observation_smoke.txt").write_text("S-01 became cloudy; 可能与温度有关", encoding="utf-8") + conn = init_db(workspace) + [task_id] = ingest_inbox(inbox, workspace, conn) + conn.close() + failed = ModelResult(success=False, role="fast", provider="deepseek", mode="auto", error="HTTP 429", requires_review=True) + fallback = ModelResult(success=True, role="fast", provider="local_stub", mode="local", output_json={"requires_review": True}, fallback_used=True, fallback_from="fast", requires_review=True) + execution = ModelExecution(attempts=[failed, fallback], selected_result=fallback) + with patch("data_agent.process._execute_model_role", return_value=execution): + assert process_single_task(workspace, task_id, "auto") + derived = workspace / "tasks" / task_id / "derived" + assert list(derived.glob("*model_result_fast_cloud_attempt.json")) + assert list(derived.glob("*model_result_fast.json")) + db = sqlite3.connect(workspace / "agent.sqlite") + statuses = [row[0] for row in db.execute("SELECT status FROM processing_runs WHERE tool_name='model:fast'")] + messages = [row[0] for row in db.execute("SELECT message FROM quality_flags")] + db.close() + assert "failed" in statuses and "succeeded" in statuses + assert any("model_unavailable" in message for message in messages) + assert any("fallback_used" in message for message in messages) diff --git a/tests/test_model_provider_mock.py b/tests/test_model_provider_mock.py index e011ee8..9cb9b4c 100644 --- a/tests/test_model_provider_mock.py +++ b/tests/test_model_provider_mock.py @@ -45,6 +45,38 @@ def _make_ctx() -> TaskContext: ) +def _observation_output(**overrides): + value = { + "factual_observations": ["test"], + "trend_statements": [], + "interpretation_candidates": [], + "operator_notes": [], + "sample_ids": [], + "time_expressions": [], + "phenomenon_types": [], + "uncertainties": [], + "requires_review": False, + "confidence": 0.9, + } + value.update(overrides) + return value + + +def _surface_output(**overrides): + value = { + "image_kind": "surface_photo", + "detected_objects": ["particle"], + "visible_features": ["uniform texture"], + "scale_bar_text": "", + "annotation_text": [], + "uncertainties": [], + "requires_review": False, + "confidence": 0.8, + } + value.update(overrides) + return value + + class TestProviderMockText: def test_success_with_valid_json(self): profile = _make_profile("fast") @@ -54,7 +86,7 @@ def test_success_with_valid_json(self): mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = { - "choices": [{"message": {"content": '{"factual_observations": ["test"], "confidence": 0.9}'}}], + "choices": [{"message": {"content": json.dumps(_observation_output())}}], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, } @@ -62,7 +94,8 @@ def test_success_with_valid_json(self): result = call_openai_compatible(profile, ctx, env) assert result.success - assert result.output_json == {"factual_observations": ["test"], "confidence": 0.9} + assert result.output_json["factual_observations"] == ["test"] + assert result.output_json["confidence"] == 0.9 assert result.model == "test-model-v1" assert result.token_usage["total_tokens"] == 15 @@ -81,7 +114,7 @@ def test_invalid_json(self): result = call_openai_compatible(profile, ctx, env) assert not result.success - assert "Invalid JSON" in result.error + assert "invalid_json_content" in result.error def test_http_error(self): profile = _make_profile("fast") @@ -183,7 +216,7 @@ def test_vision_success(self): mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = { - "choices": [{"message": {"content": '{"detected_objects": ["particle"], "confidence": 0.8}'}}], + "choices": [{"message": {"content": json.dumps(_surface_output())}}], } with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=mock_resp): @@ -211,7 +244,9 @@ def test_vision_forbidden_keys_removed(self): mock_resp = MagicMock() mock_resp.status_code = 200 mock_resp.json.return_value = { - "choices": [{"message": {"content": '{"detected_objects": [], "final_conclusion": "bad"}'}}], + "choices": [{"message": {"content": json.dumps(_surface_output( + detected_objects=[], final_conclusion="bad" + ))}}], } with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=mock_resp): diff --git a/tests/test_real_api_tools.py b/tests/test_real_api_tools.py new file mode 100644 index 0000000..dc4f490 --- /dev/null +++ b/tests/test_real_api_tools.py @@ -0,0 +1,54 @@ +"""Offline safety tests for the opt-in real API and secret-audit tools.""" +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[1] +PYTHON = REPO / ".venv" / "bin" / "python" + + +def _clean_provider_env() -> dict[str, str]: + env = os.environ.copy() + for prefix in ("DEEPSEEK_TEXT", "VOLCENGINE_VISION", "SILICONFLOW_OCR"): + for suffix in ("BASE_URL", "API_KEY", "MODEL"): + env.pop(f"{prefix}_{suffix}", None) + return env + + +def test_real_api_runner_skips_without_credentials(): + result = subprocess.run( + [str(PYTHON), "scripts/run_real_api_check.py", "--scenario", "deepseek-text"], + cwd=REPO, env=_clean_provider_env(), capture_output=True, text=True, + ) + assert result.returncode == 2 + assert "status=SKIPPED" in result.stdout + assert "Authorization" not in result.stdout + result.stderr + + +def test_auto_fallback_runner_is_offline_and_audited(): + result = subprocess.run( + [str(PYTHON), "scripts/run_real_api_check.py", "--scenario", "auto-fallback"], + cwd=REPO, env=_clean_provider_env(), capture_output=True, text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "status=PASS" in result.stdout + assert "fallback=True" in result.stdout + assert "validation=warn" in result.stdout + assert "export=True" in result.stdout + + +def test_secret_audit_detects_exact_value_without_printing_it(tmp_path): + secret = "unit-test-exact-secret-never-print" + (tmp_path / "result.json").write_text(f'{{"value":"{secret}"}}', encoding="utf-8") + env = _clean_provider_env() + env["DEEPSEEK_TEXT_API_KEY"] = secret + result = subprocess.run( + [str(PYTHON), "scripts/audit_secret_leaks.py", "--repo", str(REPO), "--workspace", str(tmp_path)], + cwd=REPO, env=env, capture_output=True, text=True, + ) + assert result.returncode == 1 + assert "exact match found" in result.stdout + assert secret not in result.stdout + result.stderr From 65ccb82a552d3de6cef40b4891f66ce7d28925e0 Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 00:53:14 +0800 Subject: [PATCH 2/6] docs: publish professional project and release status --- CURRENT_RELEASE_STATUS.md | 42 +++ EXPORT_CHECK.md | 2 + FINAL_CHECK.md | 2 + FRONTEND_CHECK.md | 2 + LICENSE | 21 ++ MODEL_LAYER_CHECK.md | 2 + README.md | 389 +++++++++++++-------------- REAL_API_CHECK.md | 63 ++--- VALIDATION_CHECK.md | 2 + docs/real_api_check_template.md | 142 +++------- docs/real_model_integration_audit.md | 27 ++ 11 files changed, 338 insertions(+), 356 deletions(-) create mode 100644 CURRENT_RELEASE_STATUS.md create mode 100644 LICENSE create mode 100644 docs/real_model_integration_audit.md diff --git a/CURRENT_RELEASE_STATUS.md b/CURRENT_RELEASE_STATUS.md new file mode 100644 index 0000000..d5ca06e --- /dev/null +++ b/CURRENT_RELEASE_STATUS.md @@ -0,0 +1,42 @@ +# Current Release Status + +**Release judgment: PARTIALLY READY** + +## Verified baseline + +- Date: 2026-07-13 +- Base commit: `135c98c` +- Branch: `codex/real-model-release-gate` +- Python: 3.11.15 +- Default offline tests: `262 passed, 52 skipped` +- Demo tests: skipped because `DATA_AGENT_DEMO_INBOX` is not configured +- Compile check: PASS +- Deterministic auto fallback: PASS +- Fallback package validation/export: WARN/PASS + +## Provider status + +| Provider | Role | Model | Offline contract | Real API | +|---|---|---|---|---| +| DeepSeek | observation text | `deepseek-v4-pro` | PASS | NOT RUN — rotated credentials missing | +| Volcengine Ark | chart/surface vision | local endpoint ID | PASS | NOT RUN — rotated credentials missing | +| SiliconFlow | OCR | `PaddlePaddle/PaddleOCR-VL-1.5` | PASS | NOT RUN — rotated credentials missing | + +## Mode and security status + +- `local`: PASS; zero network requests. +- `cloud`: offline request, response, schema, error, redaction, and persistence tests PASS. +- `auto`: cloud failure and fallback are separate auditable results; synthetic smoke PASS. +- `.env` and real `model_profiles.yaml` remain ignored and untracked. +- Exact-value scanner is implemented and tested without printing secret values. +- Real-key scan is NOT RUN because rotated keys are not configured. + +## Remaining release gates + +1. Configure newly rotated provider credentials locally. +2. Run all three real-provider smoke scenarios. +3. Validate and export each real-provider package. +4. Run exact-value audit against each workspace and ZIP. +5. Re-run the full suite and bind documentation to the final commit. + +The project is not yet `READY FOR PORTFOLIO README REWRITE`. diff --git a/EXPORT_CHECK.md b/EXPORT_CHECK.md index 49504f6..c0a2603 100644 --- a/EXPORT_CHECK.md +++ b/EXPORT_CHECK.md @@ -1,5 +1,7 @@ # Export Check +> Historical checkpoint. This report records the state at its original commit/date and is not the current release status. See `CURRENT_RELEASE_STATUS.md` for the latest verified state. + ## Current Status - **Date**: 2026-07-11 diff --git a/FINAL_CHECK.md b/FINAL_CHECK.md index 6821da2..0addd79 100644 --- a/FINAL_CHECK.md +++ b/FINAL_CHECK.md @@ -1,5 +1,7 @@ # FINAL_CHECK.md：最终验收报告 +> Historical checkpoint. This report records the state at its original commit/date and is not the current release status. See `CURRENT_RELEASE_STATUS.md` for the latest verified state. + ## 0. 验收日期与范围 - **日期**：2026-07-09 diff --git a/FRONTEND_CHECK.md b/FRONTEND_CHECK.md index d503f9a..2026dda 100644 --- a/FRONTEND_CHECK.md +++ b/FRONTEND_CHECK.md @@ -1,5 +1,7 @@ # FRONTEND_CHECK.md：前端操作闭环验收报告 +> Historical checkpoint. This report records the state at its original commit/date and is not the current release status. See `CURRENT_RELEASE_STATUS.md` for the latest verified state. + ## 1. 本轮目标 为材料研发数据处理 Agent 增加 Streamlit 本地前端操作闭环，并进行收口验收。 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..41d5c31 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Zander Kong + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MODEL_LAYER_CHECK.md b/MODEL_LAYER_CHECK.md index ea070b9..f2168ee 100644 --- a/MODEL_LAYER_CHECK.md +++ b/MODEL_LAYER_CHECK.md @@ -1,5 +1,7 @@ # Model Layer Check +> Historical checkpoint. This report records the state at its original commit/date and is not the current release status. See `CURRENT_RELEASE_STATUS.md` for the latest verified state. + ## 当前验收状态 ### 模型层历史 checkpoint（2026-07-09） diff --git a/README.md b/README.md index ba74a95..e62774d 100644 --- a/README.md +++ b/README.md @@ -1,281 +1,262 @@ -# Material R&D Data Processing Agent MVP +# Material R&D Data Processing Agent -本地材料研发数据处理 Agent，接收 demo 数据，生成可追溯 evidence package。 +面向材料研发数据整理、证据链复核与交付的本地优先数据处理 Agent。系统将分散的 CSV、光谱数据、图表截图、表面图像和实验观察文本组织为可追溯的 evidence package，并提供版本化处理、人工复核、完整性验证和安全导出。 -## 环境要求 +> 模型输出仅用于辅助提取可见事实、元数据和不确定性，不构成科研结论、机理解释或实验建议。 -- Python >= 3.10 +## 项目概览 -## 安装 +材料研发数据通常同时包含结构化表格、仪器导出、图片和自然语言记录。仅生成处理结果并不足以支持后续复核：还需要知道输入来自哪里、使用了什么处理步骤、哪些结果被替代、哪些内容需要人工确认，以及交付包是否完整。 -```bash -python3.10 -m venv .venv -.venv/bin/python -m pip install -e '.[dev]' +本项目围绕这条证据链实现完整工作流： + +```text +Upload → Ingest → Process → Review → Validate → Export ``` -## 当前验收状态 +核心能力： + +- 多类型研发数据识别与统一任务登记 +- L0–L3 生命周期管理和不可变原始数据归档 +- 每次运行生成独立 L2 结果，重跑不覆盖历史产物 +- `derived_from`、`replaces`、`replaced_by` 关系追踪 +- local / cloud / auto 三种模型模式和可审计 fallback +- SQLite 注册表与任务目录 evidence package 双重记录 +- Streamlit 操作界面与 Marimo 复核工作台 +- package validation、安全 ZIP 导出和样品级索引 +- API key、异常信息、模型原始响应和 UI 展示统一脱敏 + +## 系统架构 + +```mermaid +flowchart LR + A["CSV / Spectra / Images / Notes"] --> B["Ingest & Classification"] + B --> C["L1 Immutable Raw Archive"] + C --> D["Typed Processors"] + C --> E["Model Router"] + E --> F["Local Rules / Stub"] + E --> G["Configured Cloud Providers"] + D --> H["Versioned L2 Evidence"] + F --> H + G --> H + H --> I["Review & Quality Flags"] + I --> J["Package Validation"] + J --> K["Safe Review ZIP"] + B --> L["SQLite Registry"] + H --> L + I --> L +``` -- **基础 demo 闭环**：ingest → process → review → info -- **Model Service Layer 已实现**：可配置模型角色、路由、fallback -- **Streamlit Local UI 已实现**：6-tab 前端，支持 ingest/upload/process/review/查看 -- **pytest**：242 passed, 52 skipped（2026-07-11 Release Candidate；demo 集成测试需 DATA_AGENT_DEMO_INBOX 环境变量） -- **L2 版本化输出**：已实现（run 前缀） -- **Rerun replaces/replaced_by**：已实现 -- **Marimo 复核命令**：可生成 -- **Package Validation**：已实现（CLI `validate --task/--all` + UI Validate Package 按钮） -- **Package Export**：已实现（CLI `export --task` + UI Export Review Package 下载） -- **Sample Index**：已实现（CLI `index-samples` + UI Sample View tab） -- **FRONTEND_CHECK.md**：前端验收记录 -- **MODEL_LAYER_CHECK.md**：模型服务层验收记录（历史 checkpoint: 2026-07-09, 112 passed） -- **docs/ui_walkthrough.md**：UI 操作 walkthrough +任务目录保留可移植的文件证据，SQLite 负责跨任务查询和关系审计： -## 数据输入规范 +```text +task_XXXX/ +├── raw/ # L1 原始归档副本 +├── derived/ # 带 run 前缀的 L2 结果 +├── logs/ # runs、flags、relationships、validation +├── reviews/ # 人工复核记录 +└── manifest.json # package 索引 +``` -提交数据前请阅读 [Data Input Contract](docs/data_input_contract.md)，了解推荐的 CSV 格式、图像要求和观测文本规范，以提升自动提取准确率。 +## 数据与审计模型 -## 快速开始 +### 生命周期 -```bash -# 查看命令 -.venv/bin/python -m data_agent --help +| 层级 | 含义 | 约束 | +|---|---|---| +| L0 | 外部输入登记 | 记录来源和 checksum | +| L1 | 工作区原始归档 | 不修改、不覆盖 | +| L2 | 派生结果 | 每次运行创建新文件和对象 | +| L3 | 废弃、失败或被替代状态 | 保留历史，不物理删除证据 | -# 初始化 ingest -DATA_AGENT_DEMO_INBOX=/path/to/demo-inbox -.venv/bin/python -m data_agent ingest \ - --inbox "$DATA_AGENT_DEMO_INBOX" \ - --workspace work/check-ws +### 模型调用 -# 处理全部任务 -.venv/bin/python -m data_agent process \ - --workspace work/check-ws \ - --all \ - --models local +| 数据类型 | local | cloud / auto | +|---|---|---| +| 样品元数据、数值、原始光谱 | 本地确定性处理 | 本地确定性处理 | +| 图表截图 | local stub | OCR + Vision | +| 表面图像 | local vision stub | Vision + OCR | +| 观察文本 | 本地规则结果 | Text extraction | -# 查看任务信息 -.venv/bin/python -m data_agent info --workspace work/check-ws +`cloud` 记录云端调用的真实成功或失败；`auto` 在云端失败时执行本地 fallback，并分别保留云端失败 attempt 和最终 fallback evidence。失败记录不会被 fallback 覆盖或伪装为云端成功。 -# 审核任务 -.venv/bin/python -m data_agent review \ - --workspace work/check-ws \ - --task task_0001 \ - --action approve \ - --reviewer ZQ \ - --comment "demo approval" +当前 provider profile 模板对应： -# 打开 marimo 复核工作台 -.venv/bin/python -m data_agent open \ - --workspace work/check-ws \ - --task task_0001 +- DeepSeek `deepseek-v4-pro`：观察文本结构化提取 +- 火山引擎方舟视觉 endpoint：图表和表面图像观察 +- SiliconFlow `PaddlePaddle/PaddleOCR-VL-1.5`：图片文字提取 -# 重跑任务（验证 L2 版本化） -.venv/bin/python -m data_agent process \ - --workspace work/check-ws \ - --task task_0007 \ - --models local -``` +真实 provider 验证状态以 [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) 和 [REAL_API_CHECK.md](REAL_API_CHECK.md) 为准。没有真实调用证据时统一标记为 `NOT RUN`。 + +## 快速开始 + +### 环境 -## 运行测试 +- Python 3.10+ +- macOS、Linux，或支持 Python/SQLite 的等价环境 ```bash -.venv/bin/python -m pytest -q +python3.11 -m venv .venv +.venv/bin/python -m pip install --upgrade pip +.venv/bin/python -m pip install -e '.[dev]' ``` -## 完整验收命令 +### 本地工作流 ```bash -rm -rf work/check-ws - -.venv/bin/python -m pytest -q +export DATA_AGENT_DEMO_INBOX=/path/to/demo-inbox +export WORKSPACE=/tmp/material-agent-workspace -DATA_AGENT_DEMO_INBOX=/path/to/demo-inbox .venv/bin/python -m data_agent ingest \ --inbox "$DATA_AGENT_DEMO_INBOX" \ - --workspace work/check-ws + --workspace "$WORKSPACE" .venv/bin/python -m data_agent process \ - --workspace work/check-ws \ + --workspace "$WORKSPACE" \ --all \ --models local +.venv/bin/python -m data_agent info --workspace "$WORKSPACE" + .venv/bin/python -m data_agent review \ - --workspace work/check-ws \ + --workspace "$WORKSPACE" \ --task task_0001 \ --action approve \ - --reviewer ZQ \ - --comment "demo approval" - -.venv/bin/python -m data_agent info --workspace work/check-ws - -sqlite3 work/check-ws/agent.sqlite ' -select "tasks", count(*) from tasks -union all select "files", count(*) from files -union all select "data_objects", count(*) from data_objects -union all select "processing_runs", count(*) from processing_runs -union all select "quality_flags", count(*) from quality_flags -union all select "relationships", count(*) from relationships -union all select "reviews", count(*) from reviews; -' -``` - -## Local UI - -项目提供 Streamlit 本地 Web UI，可替代 CLI 进行日常操作。 - -### 启动 UI + --reviewer reviewer-id \ + --comment "Reviewed against source evidence" -```bash -# 推荐：CLI 快捷命令 -.venv/bin/python -m data_agent ui --workspace /tmp/material-agent-ui-ws - -# 打印命令（不启动） -.venv/bin/python -m data_agent ui --workspace /tmp/material-agent-ui-ws --print-command +.venv/bin/python -m data_agent validate \ + --workspace "$WORKSPACE" \ + --all -# 直接使用 Streamlit -.venv/bin/python -m streamlit run data_agent/ui/app.py +.venv/bin/python -m data_agent export \ + --workspace "$WORKSPACE" \ + --task task_0001 ``` -### Demo workspace 验收命令 - -```bash -# 准备工作区 -rm -rf /tmp/material-agent-ui-ws -mkdir -p /tmp/material-agent-ui-ws +输入文件命名、CSV 字段和图片要求见 [Data Input Contract](docs/data_input_contract.md)。 -# CLI ingest -.venv/bin/python -m data_agent ingest \ - --inbox "$DEMO_INBOX" \ - --workspace /tmp/material-agent-ui-ws +## 本地 UI 与复核工具 -# CLI process (local mode) -.venv/bin/python -m data_agent process \ - --workspace /tmp/material-agent-ui-ws \ - --all --models local +启动七个 tab 的 Streamlit 界面： -# 启动 UI -.venv/bin/python -m data_agent ui \ - --workspace /tmp/material-agent-ui-ws +```bash +.venv/bin/python -m data_agent ui --workspace "$WORKSPACE" ``` -### UI 功能 +界面包括： -- **workspace 输入**：选择或输入本地路径，支持 `DATA_AGENT_UI_WORKSPACE` 环境变量预设 -- **ingest / upload**：从 inbox 目录或直接上传文件 -- **task 列表**：按 display_status 筛选和排序，进入 workspace 后自动加载 -- **task detail**：查看 raw/derived/runs/flags/relationships/reviews - - raw CSV 预览前 50 行（表格形式） - - model_result 分区展示（Audit / Risk / Extracted Output / Raw Response） -- **review**：支持 target type（task / quality_flag / data_object / derived_file），可选 target id -- **model profiles**：查看配置状态（不泄露 API key） -- **marimo command**：查看 marimo 复核工作台启动命令 -- **错误脱敏**：所有 UI 错误提示经过安全脱敏，不泄露 API key、Bearer token 或环境变量值 +- Overview：任务、run、flag、review 和 model-result 汇总 +- Ingest：目录导入与文件上传 +- Tasks：任务筛选和状态查看 +- Task Detail：Basic / Advanced evidence 视图、处理、复核、验证与导出 +- Sample View：样品与任务的保守关联索引 +- Model Profiles：仅显示配置状态，不显示密钥值 +- Help：工作流、数据契约和复核入口 -### UI 模式 +生成 Marimo 复核命令： -UI 中可选择 `local` / `auto` / `cloud` 模式，行为与 CLI 一致： -- `local`：零网络调用 -- `auto`：优先云端、自动降级 -- `cloud`：仅云端 +```bash +.venv/bin/python -m data_agent open \ + --workspace "$WORKSPACE" \ + --task task_0001 \ + --print-command +``` -### 相关文档 +完整操作流程见 [UI Walkthrough](docs/ui_walkthrough.md)。 -- `FRONTEND_CHECK.md`：前端验收记录 -- `MODEL_LAYER_CHECK.md`：模型服务层验收记录 -- `docs/ui_walkthrough.md`：UI 操作 walkthrough -- `docs/data_input_contract.md`：数据输入规范 -- `REAL_API_CHECK.md`：真实 API 验证记录（需用户提供 API key） +## 云端模型配置 -## 项目结构 +云端模型是可选增强；local 模式不依赖网络或 API key。 -``` -data_agent/ - ui/ # Streamlit 前端 - schemas.py # Pydantic 数据模型 - db.py # SQLite 注册系统 - package.py # Evidence package 读写 - classify.py # 文件类型识别 - ingest.py # 输入文件注册 - process.py # 统一处理编排 - reviews.py # 审核记录 - reports.py # 处理报告生成 - processors/ # 各类型数据处理 - model_adapters/ # 模型适配器 (本地/云端) -marimo_apps/ # Marimo 复核工作台 -tests/ # 测试 +```bash +cp .env.example .env +cp model_profiles.yaml.example model_profiles.yaml ``` -## 生命周期模型 +`.env` 和 `model_profiles.yaml` 已被 Git 忽略。真实密钥只应通过本机环境变量或本地 secret manager 提供，不能写入源码、测试、报告、SQLite、截图或提交记录。 -- **L0**: 原始文件登记（inbox 源路径记录） -- **L1**: 不可变归档副本（raw/ 下文件） -- **L2**: 派生处理结果（derived/ 下带 run 前缀文件） -- **L3**: 废弃/失败/替代状态（标记，不删除） +```bash +set -a +source .env +set +a -## Model Service Layer +.venv/bin/python -m data_agent models check --verbose +``` -### 配置模型服务 +执行合成输入的 provider smoke test： -1. 复制模板文件： ```bash -cp .env.example .env -cp model_profiles.yaml.example model_profiles.yaml +.venv/bin/python scripts/run_real_api_check.py --scenario deepseek-text +.venv/bin/python scripts/run_real_api_check.py --scenario volcengine-vision +.venv/bin/python scripts/run_real_api_check.py --scenario siliconflow-ocr +.venv/bin/python scripts/run_real_api_check.py --scenario auto-fallback ``` -2. 编辑 `.env`，填入真实的 API 地址和密钥（**切勿提交到仓库**）： -```bash -BEST_MODEL_BASE_URL=https://api.openai.com/v1 -BEST_MODEL_API_KEY=sk-xxxxxxxx -BEST_MODEL_NAME=gpt-4 -# ... 其他模型类似 -``` +Runner 不接受命令行 key 参数；缺少环境变量时安全返回 `SKIPPED`。安全操作说明见 [Real API Check Template](docs/real_api_check_template.md)。 + +## 验证与安全门禁 + +默认测试完全离线： -3. 检查模型配置状态： ```bash -python3 -m data_agent models check --workspace work/check-ws -python3 -m data_agent models check --workspace work/check-ws --verbose +env -u DATA_AGENT_DEMO_INBOX .venv/bin/python -m pytest -q +.venv/bin/python -m compileall -q data_agent scripts +git diff --check ``` -### 三种模型模式 +当前离线基线：`262 passed, 52 skipped`。52 个 skip 为未配置 `DATA_AGENT_DEMO_INBOX` 时的 demo 集成测试，不代表真实 demo 流程已经在当前环境执行。 -| 模式 | 说明 | -|------|------| -| `local` | 仅使用本地规则和 stub，不发起任何网络请求（默认） | -| `cloud` | 尝试调用已配置的云端模型，失败时记录错误但不崩溃 | -| `auto` | 优先使用云端模型，自动降级至本地 stub（推荐生产环境） | +真实调用后，对仓库、workspace、SQLite 和 ZIP 执行精确密钥扫描： ```bash -python3 -m data_agent process --workspace work/check-ws --all --models local -python3 -m data_agent process --workspace work/check-ws --all --models cloud -python3 -m data_agent process --workspace work/check-ws --all --models auto +.venv/bin/python scripts/audit_secret_leaks.py \ + --repo . \ + --workspace "$SMOKE_WORKSPACE" \ + --zip "$EXPORTED_ZIP" ``` -### 模型角色与路由 +任何真实密钥进入 Git、workspace、报告、SQLite 或 ZIP 都属于发布阻塞问题。 -| 数据类型 | local 模式 | cloud/auto 模式 | -|----------|-----------|-----------------| -| 样品元数据 | 无模型 | 无模型 | -| 原始数值 | 无模型 | 无模型 | -| 原始光谱 | 无模型 | 无模型 | -| 图表截图 | local_stub | OCR + Vision | -| 表面照片 | local_vision_stub | Vision + OCR | -| 观测文本 | 无模型 | Fast | -| 结构化观测 | 无模型 | 无模型 | +## 关键工程约束 -### 无密钥降级 +- 原始文件和旧 L2 结果不可覆盖 +- validation 不静默修复业务数据 +- export 不把 ZIP 生成等同于 validation 通过 +- symlink、路径穿越和不安全归档成员会被拒绝 +- 模型输出经过角色级 schema 校验和递归禁止字段清理 +- `reasoning_content` 不作为最终模型输出 +- 低置信度、解析失败、截断和 fallback 必须进入人工复核流程 +- UI、Markdown、JSON、异常和导出内容使用统一脱敏规则 -当 `model_profiles.yaml` 缺失或 API 密钥未配置时： -- `local` 模式正常运行 -- `cloud` 模式记录 "model_unavailable"，处理继续 -- `auto` 模式自动降级至本地 stub +## 项目结构 + +```text +data_agent/ +├── model_adapters/ # provider profiles、请求、解析、schema、fallback、脱敏 +├── processors/ # 各数据类型的确定性处理器 +├── ui/ # Streamlit 操作与复核界面 +├── ingest.py # 输入登记和 L0→L1 归档 +├── process.py # 统一处理、模型调用和 evidence 编排 +├── validation.py # package 完整性与关系验证 +├── export.py # validation-aware 安全 ZIP 导出 +├── sample_index.py # workspace 样品索引 +└── schemas.py # 生命周期和审计对象 +marimo_apps/ # 交互式复核工作台 +scripts/ # 真实 API smoke 与安全扫描 +tests/ # 离线单元、集成、安全和 UI 测试 +docs/ # 数据契约、模型层、UI 与真实调用文档 +``` -### OpenAI 兼容端点（中国大陆用户示例） +## 状态与文档 -支持任何 OpenAI 兼容 API，如 DeepSeek、智谱 GLM、通义千问、Kimi 等。在 `.env` 中配置相应 endpoint 即可。 +- [Current Release Status](CURRENT_RELEASE_STATUS.md)：唯一当前发布状态入口 +- [Real API Check](REAL_API_CHECK.md)：真实 provider 调用证据 +- [Model Integration Audit](docs/real_model_integration_audit.md)：路由和审计链 +- [Data Input Contract](docs/data_input_contract.md)：输入数据规范 +- [UI Walkthrough](docs/ui_walkthrough.md)：界面操作流程 +- `FINAL_CHECK.md`、`MODEL_LAYER_CHECK.md`、`FRONTEND_CHECK.md`：历史 checkpoint -## 注意事项 +## License -- 不修改原始 demo 文件 -- 重跑不覆盖旧 L2，生成新 L2 + replaces relationship -- 不输出科研结论或机理解释 -- 云端模型为可选增强，本地流程必须可完整跑完 -- **严禁将真实 API Key 写入仓库文件**，仅存储在 `.env` -- `model_profiles.yaml` 和 `.env` 已加入 `.gitignore` +本项目采用 MIT License，详见 [LICENSE](LICENSE)。 diff --git a/REAL_API_CHECK.md b/REAL_API_CHECK.md index 8bcbdc0..76fbffa 100644 --- a/REAL_API_CHECK.md +++ b/REAL_API_CHECK.md @@ -1,48 +1,19 @@ # Real API Check -**Status: NOT RUN** - -This document records the status of real API verification. Core functionality (local mode, validation, export, sample index) is complete and tested. Real API verification requires user-provided API keys and is a separate manual gate. - -## Required Steps - -1. Configure `model_profiles.yaml` with valid provider entries. -2. Set environment variables for API keys (see `docs/real_api_check_template.md` for safe shell commands). -3. Run the three scenarios documented in `docs/real_api_check_template.md`. -4. Perform the key-safety audit after each scenario. -5. Record results below. - -## Text Call Results - -| Date | Provider | Model | Role | Status | Task ID | Model Result Path | Run ID | Flags | -|------|----------|-------|------|--------|---------|-------------------|--------|-------| -| *NOT RUN* | - | - | fast/text | - | - | - | - | - | - -## Vision/OCR Call Results - -| Date | Provider | Model | Role | Status | Task ID | Model Result Path | Run ID | Flags | -|------|----------|-------|------|--------|---------|-------------------|--------|-------| -| *NOT RUN* | - | - | vision/ocr | - | - | - | - | - | - -## Auto Fallback Results - -| Date | Provider | Model | Role | Status | Task ID | Model Result Path | Run ID | Flags | -|------|----------|-------|------|--------|---------|-------------------|--------|-------| -| *NOT RUN* | - | - | vision/ocr (auto) | - | - | - | - | - | - -## Key Safety Audit - -**Result: NOT RUN** - -## Completion Criteria - -- [ ] At least one text real call succeeded -- [ ] At least one vision/OCR call succeeded, or provider failure + verified auto fallback documented as PARTIAL -- [ ] No key matches found in workspace files, SQLite, or exported ZIP -- [ ] No key stored in git or documentation - -## Notes - -- If no API keys are available, this document remains NOT RUN. Core completion is not blocked. -- Any secret leak blocks all completion claims. -- The `docs/real_api_check_template.md` contains detailed setup and scenario instructions. +**Status: PARTIAL** + +Engineering support for the three providers is implemented and verified offline. Real paid +calls were not run because rotated credentials are not configured. No credential from +conversation history was used. + +| Scenario | Date | Commit | Provider | Model | Status | Command | Evidence | +|---|---|---|---|---|---|---|---| +| DeepSeek Text | 2026-07-13 | working tree based on `135c98c` | DeepSeek | `deepseek-v4-pro` | NOT RUN | `python scripts/run_real_api_check.py --scenario deepseek-text` | Rotated environment variables missing | +| Volcengine Vision | 2026-07-13 | working tree based on `135c98c` | Volcengine Ark | local endpoint ID | NOT RUN | `python scripts/run_real_api_check.py --scenario volcengine-vision` | Rotated environment variables missing | +| SiliconFlow OCR | 2026-07-13 | working tree based on `135c98c` | SiliconFlow | `PaddlePaddle/PaddleOCR-VL-1.5` | NOT RUN | `python scripts/run_real_api_check.py --scenario siliconflow-ocr` | Rotated environment variables missing | +| Auto Fallback | 2026-07-13 | working tree based on `135c98c` | deterministic failure → local fallback | synthetic | PASS | `python scripts/run_real_api_check.py --scenario auto-fallback` | task `task_0001`; selected run `fc7abe5c-535c-4197-b22b-2bae8fc45b0e`; result `tasks/task_0001/derived/run_73ad7d87__model_result_fast.json`; latency 0 ms; token usage unavailable; fallback/review flags present; validation WARN; export PASS | +| Key Safety Audit | 2026-07-13 | working tree based on `135c98c` | all providers | - | PARTIAL | `python scripts/audit_secret_leaks.py --repo .` | scanner PASS; real-value scan remains NOT RUN | + +This file may be changed to `PASS` only after all real-provider scenarios succeed, their +packages validate and export, and the exact-value audit passes. Never record keys, headers, +raw request bodies, base64 images, or full raw responses here. diff --git a/VALIDATION_CHECK.md b/VALIDATION_CHECK.md index 861ac63..ca46c40 100644 --- a/VALIDATION_CHECK.md +++ b/VALIDATION_CHECK.md @@ -1,5 +1,7 @@ # Validation Check +> Historical checkpoint. This report records the state at its original commit/date and is not the current release status. See `CURRENT_RELEASE_STATUS.md` for the latest verified state. + ## Current Status - **Date**: 2026-07-11 diff --git a/docs/real_api_check_template.md b/docs/real_api_check_template.md index f2ee49e..76599f8 100644 --- a/docs/real_api_check_template.md +++ b/docs/real_api_check_template.md @@ -2,130 +2,60 @@ ## Prerequisites -1. A configured `model_profiles.yaml` with valid provider entries. -2. Environment variables set for each profile's API key (e.g., `BEST_MODEL_API_KEY`, `FAST_MODEL_API_KEY`, `VISION_MODEL_API_KEY`, `OCR_MODEL_API_KEY`). -3. A workspace with ingested and processed demo tasks (see `docs/ui_walkthrough.md`). +1. Revoke any credential exposed outside the local machine and create a new credential. +2. Copy `.env.example` to ignored `.env`, or set the documented variables through a local secret manager. +3. Never pass a key as a command argument and never paste it into reports. +4. Load ignored local values into the current shell; the application does not auto-load `.env`. -## Safe Environment Setup - -The application reads environment variables but does **not** auto-load `.env`. You must load them into the current shell. - -### macOS / Linux ```bash -set -a; source .env; set +a -``` - -### Windows (PowerShell) -```powershell -Get-Content .env | ForEach-Object { if ($_ -match '^([^=]+)=(.*)') { [Environment]::SetEnvironmentVariable($matches[1], $matches[2]) } } +set -a +source .env +set +a ``` -## Key Safety Rules - -- Never put a real API key in command arguments. -- `.env` and `model_profiles.yaml` are gitignored — never commit real keys. -- Use `models check` to verify configuration without exposing key values. -- `models check --verbose` shows only `configured/missing`, never the key itself. - -## Provider Guidance - -- **DeepSeek**: Can serve fast/best text roles. Endpoint typically `https://api.deepseek.com/v1`. Check current official documentation for the latest model names. -- **OpenAI-compatible multimodal**: Can serve vision/ocr roles. Examples: OpenAI GPT-4V, Qwen-VL, Gemini via OpenAI-compatible proxy. -- Always use the most current endpoint and model name from the provider's official documentation. +Required variable groups: -## Manual Test Scenarios +- `DEEPSEEK_TEXT_BASE_URL`, `DEEPSEEK_TEXT_API_KEY`, `DEEPSEEK_TEXT_MODEL` +- `VOLCENGINE_VISION_BASE_URL`, `VOLCENGINE_VISION_API_KEY`, `VOLCENGINE_VISION_MODEL` +- `SILICONFLOW_OCR_BASE_URL`, `SILICONFLOW_OCR_API_KEY`, `SILICONFLOW_OCR_MODEL` -Run each scenario against a **separate temporary workspace**. Do not use the demo workspace. +`VOLCENGINE_VISION_MODEL` is the local Ark endpoint ID. Do not commit it in the example. -### Scenario 1: Observation Text with Fast/Cloud +## Safe configuration check ```bash -# Create a fresh workspace -rm -rf /tmp/real-api-text-ws && mkdir -p /tmp/real-api-text-ws - -# Ingest an observation text task -.venv/bin/python -m data_agent ingest \ - --inbox "$DEMO_INBOX" \ - --workspace /tmp/real-api-text-ws - -# Process with cloud mode (text only) -.venv/bin/python -m data_agent process \ - --workspace /tmp/real-api-text-ws \ - --task task_0001 \ - --models cloud +cp model_profiles.yaml.example model_profiles.yaml +.venv/bin/python -m data_agent models check --verbose ``` -### Scenario 2: Chart Image with OCR/Vision Cloud +The check must show only `configured`/`missing`, never values. -```bash -.venv/bin/python -m data_agent process \ - --workspace /tmp/real-api-text-ws \ - --task task_0002 \ - --models cloud -``` - -### Scenario 3: Visual Image with Vision/OCR Auto Fallback +## Synthetic smoke scenarios ```bash -.venv/bin/python -m data_agent process \ - --workspace /tmp/real-api-text-ws \ - --task task_0006 \ - --models auto +.venv/bin/python scripts/run_real_api_check.py --scenario deepseek-text +.venv/bin/python scripts/run_real_api_check.py --scenario volcengine-vision +.venv/bin/python scripts/run_real_api_check.py --scenario siliconflow-ocr +.venv/bin/python scripts/run_real_api_check.py --scenario auto-fallback ``` -## Recording Results +The runner generates synthetic input in a temporary workspace. DeepSeek and SiliconFlow model +IDs are checked with `/models` before paid inference. Missing configuration returns `SKIPPED`; +model mismatch or provider failure returns `FAIL`. The runner never accepts an API-key option. -For each scenario, record **only** the following: -- Date -- Provider and model names (from `model_profiles.yaml`) -- Role (fast/text/vision/ocr) -- Result status (processing run status) -- Task ID -- Model-result file path -- Processing-run ID -- Quality flags generated -- Validation status +## Validation, export, and exact-value audit -**Do NOT record**: request headers, API key fragments, raw request JSON, signed URLs, or full model responses. - -## Key Safety Audit - -After running real API tests, run the following inline audit: +Use the workspace and task ID printed by each successful scenario: ```bash -# Inside the project directory, NOT inside a workspace -.venv/bin/python -c " -import os, json, sqlite3, sys -secrets = {v for k in ('BEST_MODEL_API_KEY','FAST_MODEL_API_KEY','VISION_MODEL_API_KEY','OCR_MODEL_API_KEY') if (v:=os.environ.get(k,''))} -matches = [] -# Scan workspace files -import pathlib -ws = pathlib.Path('/tmp/real-api-text-ws') -for f in ws.rglob('*'): - if f.is_file() and f.suffix in ('.json','.md','.txt','.csv'): - try: - content = f.read_text() - for s in secrets: - if s in content: - matches.append(str(f)) - break - except: pass -# Scan SQLite -try: - db = ws / 'agent.sqlite' - if db.exists(): - conn = sqlite3.connect(str(db)) - for row in conn.execute('SELECT sql FROM sqlite_master').fetchall(): - for s in secrets: - if row[0] and s in row[0]: - matches.append('sqlite_schema') - conn.close() -except: pass -if matches: - print('SECURITY FAILURE - key found in:') - for m in matches: print(f' {m}') - sys.exit(1) -else: - print('PASS - no key found') -" +.venv/bin/python -m data_agent validate --workspace "$SMOKE_WORKSPACE" --task "$TASK_ID" +.venv/bin/python -m data_agent export --workspace "$SMOKE_WORKSPACE" --task "$TASK_ID" +.venv/bin/python scripts/audit_secret_leaks.py \ + --repo . \ + --workspace "$SMOKE_WORKSPACE" \ + --zip "$EXPORTED_ZIP" ``` + +Record only date, commit, provider, model, task ID, run ID, result path, latency, token-usage +availability, flags, validation/export status, and known limitations. Never record keys, key +fragments, request headers, full raw responses, signed URLs, or base64 media. diff --git a/docs/real_model_integration_audit.md b/docs/real_model_integration_audit.md new file mode 100644 index 0000000..89f7fb1 --- /dev/null +++ b/docs/real_model_integration_audit.md @@ -0,0 +1,27 @@ +# Real Model Integration Audit + +## Routing and evidence chain + +| Data type | Role | Provider profile | Input | Request | Persistence | Failure/fallback audit | +|---|---|---|---|---|---|---| +| `descriptive_observation_text` | `fast` | DeepSeek | bounded UTF-8 L1 text | Chat Completions text + JSON mode | `model_result_fast.json`, `model:fast` run, flags and relationships | cloud failure retained as `model_result_fast_cloud_attempt.json`; auto fallback separate | +| `chart_image_input` | `vision` | Volcengine Ark | L1 image | multimodal `image_url` data URL | `model_result_vision.json`, `model:vision` run | cloud does not fallback; auto preserves both attempts | +| `chart_image_input` | `ocr` | SiliconFlow | L1 image | multimodal `image_url` data URL | `model_result_ocr.json`, `model:ocr` run | cloud does not fallback; auto preserves both attempts | +| `visual_image` | `vision` | Volcengine Ark | L1 image | multimodal `image_url` data URL | `model_result_vision.json`, `model:vision` run | manual-review boundary remains mandatory | + +All model results are sanitized before persistence. Runs, quality flags, relationships, +validation, UI previews, and exports consume the same evidence. Local mode makes no network call. + +## Implementation boundaries + +- Provider differences are expressed through `ModelProfile` capabilities. +- Role-level Pydantic schemas reject missing or incorrectly typed core fields. +- `reasoning_content` is never used as final output. +- Failed cloud attempts cannot be presented as successful local fallback. +- `ingest.py`, `reviews.py`, and `db.py` were not changed. + +## Verification state + +- Offline provider, parser, fallback, persistence, UI, validation, export, and security tests: PASS. +- Deterministic offline auto-fallback smoke: PASS. +- DeepSeek, Volcengine, and SiliconFlow real calls: NOT RUN because rotated credentials are not configured. From 060817bb0c52cd3112d96fa76c376cdaaa2e78d2 Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 00:54:21 +0800 Subject: [PATCH 3/6] ci: run offline checks on branch pushes --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ebe879..29e5b2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,6 @@ name: CI on: push: - branches: [main] pull_request: permissions: From b5fd8d56f2c5bcf816c6410f34fb94a7c9904957 Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 00:56:11 +0800 Subject: [PATCH 4/6] fix: make CI tests use active Python runtime --- .github/workflows/ci.yml | 4 ++-- tests/test_real_api_tools.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29e5b2a..97fd224 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.11" cache: pip diff --git a/tests/test_real_api_tools.py b/tests/test_real_api_tools.py index dc4f490..ea42d24 100644 --- a/tests/test_real_api_tools.py +++ b/tests/test_real_api_tools.py @@ -3,11 +3,12 @@ import os import subprocess +import sys from pathlib import Path REPO = Path(__file__).resolve().parents[1] -PYTHON = REPO / ".venv" / "bin" / "python" +PYTHON = Path(sys.executable) def _clean_provider_env() -> dict[str, str]: From 4f00ac5d06733dffe9cdc121b3254fd326e87955 Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 23:23:25 +0800 Subject: [PATCH 5/6] feat: verify real model providers --- .env.example | 6 +- .../model_adapters/openai_compatible.py | 81 ++++++++++++++++--- data_agent/model_adapters/prompts.py | 20 +---- data_agent/model_adapters/redaction.py | 2 +- data_agent/ui/security.py | 2 +- model_profiles.yaml.example | 18 ++--- scripts/audit_secret_leaks.py | 2 +- scripts/run_real_api_check.py | 31 ++++--- tests/test_model_provider_contracts.py | 16 ++-- tests/test_model_provider_mock.py | 23 ++++++ tests/test_real_api_tools.py | 2 +- 11 files changed, 147 insertions(+), 56 deletions(-) diff --git a/.env.example b/.env.example index ba3a70f..592c57d 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,9 @@ DEEPSEEK_TEXT_BASE_URL=https://api.deepseek.com DEEPSEEK_TEXT_API_KEY= DEEPSEEK_TEXT_MODEL=deepseek-v4-pro -VOLCENGINE_VISION_BASE_URL=https://ark.cn-beijing.volces.com/api/v3 -VOLCENGINE_VISION_API_KEY= -VOLCENGINE_VISION_MODEL= +MIMO_VISION_BASE_URL=https://api.xiaomimimo.com/v1 +MIMO_VISION_API_KEY= +MIMO_VISION_MODEL=mimo-v2.5 SILICONFLOW_OCR_BASE_URL=https://api.siliconflow.cn/v1 SILICONFLOW_OCR_API_KEY= diff --git a/data_agent/model_adapters/openai_compatible.py b/data_agent/model_adapters/openai_compatible.py index 70091dd..6a5c050 100644 --- a/data_agent/model_adapters/openai_compatible.py +++ b/data_agent/model_adapters/openai_compatible.py @@ -67,16 +67,18 @@ def build_chat_request( if image_path: if "image" not in modalities or not ctx.has_image: raise ValueError(f"Profile '{profile.name}' does not accept image input") + image_url: dict[str, str] = {"url": _encode_image(image_path)} + # Ark accepts OpenAI-style image_url payloads but its endpoint can stall + # on the optional detail extension for otherwise valid chart images. + if profile.provider not in {"volcengine_ark", "xiaomi_mimo"}: + image_url["detail"] = profile.image_detail messages.append({ "role": "user", "content": [ {"type": "text", "text": user_prompt}, { "type": "image_url", - "image_url": { - "url": _encode_image(image_path), - "detail": profile.image_detail, - }, + "image_url": image_url, }, ], }) @@ -91,8 +93,11 @@ def build_chat_request( "model": model_name, "messages": messages, "temperature": 0.0, - "max_tokens": profile.max_output_tokens, } + if profile.provider == "xiaomi_mimo": + payload["max_completion_tokens"] = profile.max_output_tokens + else: + payload["max_tokens"] = profile.max_output_tokens if profile.effective_json_mode() in {"required", "preferred"}: payload["response_format"] = {"type": "json_object"} if profile.thinking_mode != "provider_default": @@ -177,6 +182,52 @@ def _parse_response(raw: Any) -> tuple[dict[str, Any], str, list[str], bool]: return parsed, content, warnings, requires_review +def _parse_siliconflow_ocr_plaintext(raw: Any) -> tuple[dict[str, Any], str, list[str], bool]: + """Conservatively normalize SiliconFlow OCR's documented plain-text output. + + PaddleOCR-VL may return recognized text instead of the JSON requested by the + generic extraction contract. This preserves only visible text and marks the + result for review; it does not infer missing layout, values, or units. + """ + if not isinstance(raw, dict): + raise ValueError("response_body_not_object") + choices = raw.get("choices") + if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): + raise ValueError("empty_choices") + message = choices[0].get("message") + if not isinstance(message, dict): + raise ValueError("missing_message") + content = _content_to_text(message.get("content")) + if not content.strip(): + raise ValueError("empty_content") + + blocks: list[str] = [] + seen: set[str] = set() + for line in content.splitlines(): + normalized = re.sub(r"^\s*\d+[.)]\s*", "", line).strip() + if normalized and normalized not in seen: + seen.add(normalized) + blocks.append(normalized) + if not blocks: + raise ValueError("empty_content") + + joined = "\n".join(blocks) + units = re.findall(r"(? set[str]: for env_var in ( "BEST_MODEL_API_KEY", "FAST_MODEL_API_KEY", "VISION_MODEL_API_KEY", "OCR_MODEL_API_KEY", - "DEEPSEEK_TEXT_API_KEY", "VOLCENGINE_VISION_API_KEY", + "DEEPSEEK_TEXT_API_KEY", "MIMO_VISION_API_KEY", "SILICONFLOW_OCR_API_KEY", ): val = os.environ.get(env_var, "") diff --git a/data_agent/ui/security.py b/data_agent/ui/security.py index 1f70ac3..c044a6f 100644 --- a/data_agent/ui/security.py +++ b/data_agent/ui/security.py @@ -13,7 +13,7 @@ _KEY_ENV_NAMES = { "BEST_MODEL_API_KEY", "FAST_MODEL_API_KEY", "VISION_MODEL_API_KEY", "OCR_MODEL_API_KEY", - "DEEPSEEK_TEXT_API_KEY", "VOLCENGINE_VISION_API_KEY", + "DEEPSEEK_TEXT_API_KEY", "MIMO_VISION_API_KEY", "SILICONFLOW_OCR_API_KEY", } diff --git a/model_profiles.yaml.example b/model_profiles.yaml.example index 1bbcb1b..cc4f6fb 100644 --- a/model_profiles.yaml.example +++ b/model_profiles.yaml.example @@ -20,23 +20,23 @@ profiles: vision: role: vision - provider: volcengine_ark - base_url_env: VOLCENGINE_VISION_BASE_URL - api_key_env: VOLCENGINE_VISION_API_KEY - model_env: VOLCENGINE_VISION_MODEL + provider: xiaomi_mimo + base_url_env: MIMO_VISION_BASE_URL + api_key_env: MIMO_VISION_API_KEY + model_env: MIMO_VISION_MODEL endpoint_path: /chat/completions input_modalities: [text, image] - json_mode: disabled + json_mode: required image_detail: high - thinking_mode: provider_default - max_output_tokens: 2048 + thinking_mode: disabled + max_output_tokens: 256 enabled: true priority: 10 fallback: [local_stub] timeout_seconds: 90 cost_tier: medium supports_vision: true - supports_json: false + supports_json: true ocr: role: ocr @@ -49,7 +49,7 @@ profiles: json_mode: disabled image_detail: high thinking_mode: provider_default - max_output_tokens: 2048 + max_output_tokens: 256 enabled: true priority: 10 fallback: [local_ocr_stub, local_stub] diff --git a/scripts/audit_secret_leaks.py b/scripts/audit_secret_leaks.py index 8a03f2c..e1e1e8b 100644 --- a/scripts/audit_secret_leaks.py +++ b/scripts/audit_secret_leaks.py @@ -12,7 +12,7 @@ SECRET_NAMES = ( "DEEPSEEK_TEXT_API_KEY", - "VOLCENGINE_VISION_API_KEY", + "MIMO_VISION_API_KEY", "SILICONFLOW_OCR_API_KEY", ) diff --git a/scripts/run_real_api_check.py b/scripts/run_real_api_check.py index 327e3fe..0705fbe 100644 --- a/scripts/run_real_api_check.py +++ b/scripts/run_real_api_check.py @@ -24,23 +24,26 @@ from data_agent.export import export_task from data_agent.ingest import ingest_inbox from data_agent.model_adapters.redaction import redact_string -from data_agent.process import process_single_task +import data_agent.process as process_module from data_agent.validation import validate_task -SCENARIOS = ("deepseek-text", "volcengine-vision", "siliconflow-ocr", "auto-fallback") +SCENARIOS = ("deepseek-text", "mimo-vision", "siliconflow-ocr", "auto-fallback") CONFIG = { "deepseek-text": { "role": "fast", "provider": "deepseek", "prefix": "DEEPSEEK_TEXT", "modalities": ["text"], "json_mode": "required", "thinking_mode": "disabled", + "max_output_tokens": 2048, }, - "volcengine-vision": { - "role": "vision", "provider": "volcengine_ark", "prefix": "VOLCENGINE_VISION", - "modalities": ["text", "image"], "json_mode": "disabled", "thinking_mode": "provider_default", + "mimo-vision": { + "role": "vision", "provider": "xiaomi_mimo", "prefix": "MIMO_VISION", + "modalities": ["text", "image"], "json_mode": "required", "thinking_mode": "disabled", + "max_output_tokens": 512, }, "siliconflow-ocr": { "role": "ocr", "provider": "siliconflow", "prefix": "SILICONFLOW_OCR", "modalities": ["text", "image"], "json_mode": "disabled", "thinking_mode": "provider_default", + "max_output_tokens": 256, }, } @@ -75,7 +78,7 @@ def _write_profile(path: Path, spec: dict[str, object], prefix: str) -> None: "model_env": f"{prefix}_MODEL", "endpoint_path": "/chat/completions", "input_modalities": spec["modalities"], "json_mode": spec["json_mode"], "image_detail": "high", "thinking_mode": spec["thinking_mode"], - "max_output_tokens": 2048, "enabled": True, + "max_output_tokens": int(spec["max_output_tokens"]), "enabled": True, "fallback": ["local_ocr_stub", "local_stub"] if role == "ocr" else ["local_stub"], "timeout_seconds": 90, "supports_vision": "image" in spec["modalities"], "supports_json": spec["json_mode"] != "disabled", @@ -89,7 +92,7 @@ def _make_input(inbox: Path, scenario: str) -> None: "2026-01-01 10:00，样品 SYN-01 表面可见轻微浑浊。可能与温度变化有关。操作员备注：仅为合成测试。", encoding="utf-8", ) - elif scenario == "volcengine-vision": + elif scenario == "mimo-vision": x = [1, 2, 3, 4, 5] fig, ax = plt.subplots(figsize=(6, 4)) ax.plot(x, [1, 3, 2, 4, 3], label="Synthetic A") @@ -138,7 +141,7 @@ def _run_one(scenario: str) -> int: print(f"scenario={scenario} status=SKIPPED reason=missing_environment workspace={workspace}") return 2 requested = values[f"{prefix}_MODEL"] - if scenario in {"deepseek-text", "siliconflow-ocr"}: + if scenario in {"deepseek-text", "mimo-vision", "siliconflow-ocr"}: available = _preflight_models(values, requested) print(f"scenario={scenario} provider={spec['provider']} requested_model={'available' if available else 'unavailable'}") if not available: @@ -157,13 +160,21 @@ def _run_one(scenario: str) -> int: original_post = adapter.requests.post adapter.requests.post = lambda *args, **kwargs: SimpleNamespace(status_code=429, text="synthetic rate limit") try: - process_single_task(workspace, task_id, "auto") + process_module.process_single_task(workspace, task_id, "auto") finally: adapter.requests.post = original_post provider = "local_stub" role = "fast" else: - process_single_task(workspace, task_id, "cloud") + # A provider smoke validates one role only. Chart inputs normally route + # to both OCR and vision, which would otherwise turn an intentionally + # single-provider profile into a false validation failure. + original_router = process_module.route_model_calls + process_module.route_model_calls = lambda _ctx: [str(spec["role"])] + try: + process_module.process_single_task(workspace, task_id, "cloud") + finally: + process_module.route_model_calls = original_router provider = str(spec["provider"]) role = str(spec["role"]) diff --git a/tests/test_model_provider_contracts.py b/tests/test_model_provider_contracts.py index dbb44f1..515e10f 100644 --- a/tests/test_model_provider_contracts.py +++ b/tests/test_model_provider_contracts.py @@ -72,11 +72,11 @@ def test_deepseek_payload_contains_real_text_and_json_mode(): assert "S-01 became cloudy" in payload["messages"][1]["content"] -@pytest.mark.parametrize("provider_name,model", [ - ("volcengine_ark", "ep-test"), - ("siliconflow", "PaddlePaddle/PaddleOCR-VL-1.5"), +@pytest.mark.parametrize("provider_name,model,expect_detail", [ + ("xiaomi_mimo", "mimo-v2.5", False), + ("siliconflow", "PaddlePaddle/PaddleOCR-VL-1.5", True), ]) -def test_multimodal_provider_payload(provider_name, model, tmp_path): +def test_multimodal_provider_payload(provider_name, model, expect_detail, tmp_path): image = tmp_path / "chart.png" image.write_bytes(b"valid-test-bytes") p = profile("ocr", provider_name, image=True, json_mode="disabled") @@ -88,7 +88,13 @@ def test_multimodal_provider_payload(provider_name, model, tmp_path): parts = payload["messages"][1]["content"] image_part = next(part for part in parts if part["type"] == "image_url") assert image_part["image_url"]["url"].startswith("data:image/png;base64,") - assert image_part["image_url"]["detail"] == "high" + if expect_detail: + assert image_part["image_url"]["detail"] == "high" + else: + assert "detail" not in image_part["image_url"] + if provider_name == "xiaomi_mimo": + assert payload["max_completion_tokens"] == 2048 + assert "max_tokens" not in payload @pytest.mark.parametrize("wrapped", [ diff --git a/tests/test_model_provider_mock.py b/tests/test_model_provider_mock.py index 9cb9b4c..1d75561 100644 --- a/tests/test_model_provider_mock.py +++ b/tests/test_model_provider_mock.py @@ -256,6 +256,29 @@ def test_vision_forbidden_keys_removed(self): assert "final_conclusion" not in result.output_json assert "model_output_excluded_from_conclusion" in result.warnings + def test_siliconflow_ocr_plaintext_is_conservatively_normalized(self): + profile = ModelProfile( + name="ocr", role="ocr", provider="siliconflow", supports_vision=True, + supports_json=False, input_modalities=["text", "image"], json_mode="disabled", + ) + ctx = TaskContext(task_id="task_0001", data_type="chart_image_input", model_mode="cloud", has_image=True) + env = _make_env() + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: + f.write(b"fake png content") + image_path = f.name + mock_resp = MagicMock(status_code=200) + mock_resp.json.return_value = { + "choices": [{"message": {"content": "1. Wavelength 550 nm\n2. Wavelength 550 nm"}, "finish_reason": "length"}], + } + with patch("data_agent.model_adapters.openai_compatible.requests.post", return_value=mock_resp): + result = call_openai_compatible(profile, ctx, env, image_path=image_path) + os.unlink(image_path) + assert result.success + assert result.output_json["text_blocks"] == ["Wavelength 550 nm"] + assert result.output_json["detected_units"] == ["nm"] + assert result.requires_review + assert "siliconflow_ocr_plaintext_normalized" in result.warnings + class TestStubProviders: def test_local_stub(self): diff --git a/tests/test_real_api_tools.py b/tests/test_real_api_tools.py index ea42d24..a55f2b2 100644 --- a/tests/test_real_api_tools.py +++ b/tests/test_real_api_tools.py @@ -13,7 +13,7 @@ def _clean_provider_env() -> dict[str, str]: env = os.environ.copy() - for prefix in ("DEEPSEEK_TEXT", "VOLCENGINE_VISION", "SILICONFLOW_OCR"): + for prefix in ("DEEPSEEK_TEXT", "MIMO_VISION", "SILICONFLOW_OCR"): for suffix in ("BASE_URL", "API_KEY", "MODEL"): env.pop(f"{prefix}_{suffix}", None) return env From 599d4dc5bec923cf9c417e993457139bf6f63a39 Mon Sep 17 00:00:00 2001 From: Zander Date: Mon, 13 Jul 2026 23:23:54 +0800 Subject: [PATCH 6/6] docs: record real API release verification --- CURRENT_RELEASE_STATUS.md | 42 +++++++++++++++++---------------------- README.md | 6 +++--- REAL_API_CHECK.md | 35 +++++++++++++++++++------------- 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/CURRENT_RELEASE_STATUS.md b/CURRENT_RELEASE_STATUS.md index d5ca06e..35115e2 100644 --- a/CURRENT_RELEASE_STATUS.md +++ b/CURRENT_RELEASE_STATUS.md @@ -1,42 +1,36 @@ # Current Release Status -**Release judgment: PARTIALLY READY** +**Release judgment: READY FOR PORTFOLIO README REWRITE** ## Verified baseline - Date: 2026-07-13 -- Base commit: `135c98c` - Branch: `codex/real-model-release-gate` +- Provider implementation commit: `4f00ac5` - Python: 3.11.15 -- Default offline tests: `262 passed, 52 skipped` -- Demo tests: skipped because `DATA_AGENT_DEMO_INBOX` is not configured +- Default offline tests: `263 passed, 52 skipped` +- Demo tests: not run because `DATA_AGENT_DEMO_INBOX` is not configured - Compile check: PASS -- Deterministic auto fallback: PASS -- Fallback package validation/export: WARN/PASS +- `git diff --check`: PASS +- Exact-value key safety audit: PASS for the repository and all successful synthetic evidence packages ## Provider status -| Provider | Role | Model | Offline contract | Real API | +| Provider | Role | Model | Real API status | Evidence status | |---|---|---|---|---| -| DeepSeek | observation text | `deepseek-v4-pro` | PASS | NOT RUN — rotated credentials missing | -| Volcengine Ark | chart/surface vision | local endpoint ID | PASS | NOT RUN — rotated credentials missing | -| SiliconFlow | OCR | `PaddlePaddle/PaddleOCR-VL-1.5` | PASS | NOT RUN — rotated credentials missing | +| DeepSeek | observation text | `deepseek-v4-pro` | PASS | real run, validation WARN, export PASS | +| Xiaomi MiMo | chart/surface vision | `mimo-v2.5` | PASS | `/models` preflight, real run, validation WARN, export PASS | +| SiliconFlow | OCR | `PaddlePaddle/PaddleOCR-VL-1.5` | PASS | `/models` preflight, real run, validation WARN, export PASS | +| Deterministic fallback | cloud failure → local result | synthetic | PASS | failed cloud attempt and independent fallback run persisted | ## Mode and security status -- `local`: PASS; zero network requests. -- `cloud`: offline request, response, schema, error, redaction, and persistence tests PASS. -- `auto`: cloud failure and fallback are separate auditable results; synthetic smoke PASS. +- `local`: offline tests confirm zero network calls. +- `cloud`: all three active providers are verified with synthetic real calls. +- `auto`: deterministic HTTP 429 failure is retained separately from the selected local fallback. +- SiliconFlow plain OCR responses are normalized conservatively into the OCR schema and always marked for review; no missing layout or units are invented. - `.env` and real `model_profiles.yaml` remain ignored and untracked. -- Exact-value scanner is implemented and tested without printing secret values. -- Real-key scan is NOT RUN because rotated keys are not configured. +- Secret scans did not find exact configured key values in tracked files, evidence workspaces, SQLite, reports, or exported ZIP packages. -## Remaining release gates - -1. Configure newly rotated provider credentials locally. -2. Run all three real-provider smoke scenarios. -3. Validate and export each real-provider package. -4. Run exact-value audit against each workspace and ZIP. -5. Re-run the full suite and bind documentation to the final commit. - -The project is not yet `READY FOR PORTFOLIO README REWRITE`. +The historical Volcengine Ark timeout is not part of the active provider configuration. See +[REAL_API_CHECK.md](REAL_API_CHECK.md) for the detailed, non-sensitive evidence record. diff --git a/README.md b/README.md index e62774d..781b437 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ task_XXXX/ 当前 provider profile 模板对应： - DeepSeek `deepseek-v4-pro`：观察文本结构化提取 -- 火山引擎方舟视觉 endpoint：图表和表面图像观察 +- Xiaomi MiMo `mimo-v2.5`：图表和表面图像观察 - SiliconFlow `PaddlePaddle/PaddleOCR-VL-1.5`：图片文字提取 真实 provider 验证状态以 [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) 和 [REAL_API_CHECK.md](REAL_API_CHECK.md) 为准。没有真实调用证据时统一标记为 `NOT RUN`。 @@ -188,7 +188,7 @@ set +a ```bash .venv/bin/python scripts/run_real_api_check.py --scenario deepseek-text -.venv/bin/python scripts/run_real_api_check.py --scenario volcengine-vision +.venv/bin/python scripts/run_real_api_check.py --scenario mimo-vision .venv/bin/python scripts/run_real_api_check.py --scenario siliconflow-ocr .venv/bin/python scripts/run_real_api_check.py --scenario auto-fallback ``` @@ -205,7 +205,7 @@ env -u DATA_AGENT_DEMO_INBOX .venv/bin/python -m pytest -q git diff --check ``` -当前离线基线：`262 passed, 52 skipped`。52 个 skip 为未配置 `DATA_AGENT_DEMO_INBOX` 时的 demo 集成测试，不代表真实 demo 流程已经在当前环境执行。 +当前离线基线：`263 passed, 52 skipped`。52 个 skip 为未配置 `DATA_AGENT_DEMO_INBOX` 时的 demo 集成测试，不代表真实 demo 流程已经在当前环境执行。 真实调用后，对仓库、workspace、SQLite 和 ZIP 执行精确密钥扫描： diff --git a/REAL_API_CHECK.md b/REAL_API_CHECK.md index 76fbffa..0bdeb4b 100644 --- a/REAL_API_CHECK.md +++ b/REAL_API_CHECK.md @@ -1,19 +1,26 @@ # Real API Check -**Status: PARTIAL** +**Status: PASS** -Engineering support for the three providers is implemented and verified offline. Real paid -calls were not run because rotated credentials are not configured. No credential from -conversation history was used. +All current-provider calls below used synthetic, temporary inputs only. Result paths are +relative to each scenario's temporary evidence workspace. No API keys, request headers, +base64 images, or full provider responses are recorded in this report. -| Scenario | Date | Commit | Provider | Model | Status | Command | Evidence | -|---|---|---|---|---|---|---|---| -| DeepSeek Text | 2026-07-13 | working tree based on `135c98c` | DeepSeek | `deepseek-v4-pro` | NOT RUN | `python scripts/run_real_api_check.py --scenario deepseek-text` | Rotated environment variables missing | -| Volcengine Vision | 2026-07-13 | working tree based on `135c98c` | Volcengine Ark | local endpoint ID | NOT RUN | `python scripts/run_real_api_check.py --scenario volcengine-vision` | Rotated environment variables missing | -| SiliconFlow OCR | 2026-07-13 | working tree based on `135c98c` | SiliconFlow | `PaddlePaddle/PaddleOCR-VL-1.5` | NOT RUN | `python scripts/run_real_api_check.py --scenario siliconflow-ocr` | Rotated environment variables missing | -| Auto Fallback | 2026-07-13 | working tree based on `135c98c` | deterministic failure → local fallback | synthetic | PASS | `python scripts/run_real_api_check.py --scenario auto-fallback` | task `task_0001`; selected run `fc7abe5c-535c-4197-b22b-2bae8fc45b0e`; result `tasks/task_0001/derived/run_73ad7d87__model_result_fast.json`; latency 0 ms; token usage unavailable; fallback/review flags present; validation WARN; export PASS | -| Key Safety Audit | 2026-07-13 | working tree based on `135c98c` | all providers | - | PARTIAL | `python scripts/audit_secret_leaks.py --repo .` | scanner PASS; real-value scan remains NOT RUN | +| Scenario | Date | Commit | Provider | Model | Status | Evidence | +|---|---|---|---|---|---|---| +| DeepSeek Text | 2026-07-13 | `4f00ac5` | DeepSeek | `deepseek-v4-pro` | PASS | task `task_0001`; run `8e8a5576-f13e-4329-8857-2088ec05eac1`; `tasks/task_0001/derived/run_c39af2b2__model_result_fast.json`; 2,717 ms; token usage available; validation WARN; export PASS | +| MiMo Vision | 2026-07-13 | `4f00ac5` | Xiaomi MiMo | `mimo-v2.5` | PASS | `/models` preflight available; task `task_0001`; run `42528c84-651d-4afb-adc8-77c07affe065`; `tasks/task_0001/derived/run_20b83d14__model_result_vision.json`; 2,449 ms; token usage available; validation WARN; export PASS | +| SiliconFlow OCR | 2026-07-13 | `4f00ac5` | SiliconFlow | `PaddlePaddle/PaddleOCR-VL-1.5` | PASS | `/models` preflight available; task `task_0001`; run `74f0eb77-4ee7-470e-8774-cfd89ed0f756`; `tasks/task_0001/derived/run_84b4cb4b__model_result_ocr.json`; 8,268 ms; token usage available; validation WARN; export PASS; plain OCR output normalized and marked for review | +| Auto Fallback | 2026-07-13 | `4f00ac5` | deterministic HTTP 429 → local fallback | synthetic | PASS | failed cloud attempt and selected fallback run `5ba8c4bf-24cf-40ee-a033-d12c1de42feb` persisted independently; validation WARN; export PASS | +| Key Safety Audit | 2026-07-13 | `4f00ac5` | all configured providers | - | PASS | exact-value scans passed for repository and all successful evidence workspaces, including SQLite/JSON/Markdown contents and exported ZIP packages | -This file may be changed to `PASS` only after all real-provider scenarios succeed, their -packages validate and export, and the exact-value audit passes. Never record keys, headers, -raw request bodies, base64 images, or full raw responses here. +Validation WARN is expected for synthetic image review flags and fallback audit flags; none +of the passing scenarios had validation errors. + +## Historical provider note + +The previously configured Volcengine Ark endpoint was tested on synthetic chart inputs and +timed out twice at 90 seconds. It has been removed from the active vision profile and is +not a current release gate; its failed evidence remains in local temporary workspaces. + +Never record keys, headers, raw request bodies, base64 images, or full raw responses here.