diff --git a/.gitignore b/.gitignore index 120b2b24a..33d5ce36f 100755 --- a/.gitignore +++ b/.gitignore @@ -260,8 +260,16 @@ evaluation/src/adapters/*/prompts/profile/*.json # Locomo source dataset (downloadable, not source code) data/locomo10.json +# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are +# large, redistributable only by their own authors, and not ours to ship. The directory +# itself is kept so a fresh clone has somewhere obvious to put them. +benchmarks/data/* +!benchmarks/data/.gitkeep evaluation/data/locomo/locomo10.json evaluation/locomo_evaluation/data/locomo10.json +# Benchmark inputs live here (benchmarks/configs/*.toml default to it), and they are +# large, redistributable only by their own authors, and not ours to ship. The directory +# itself is kept so a fresh clone has somewhere obvious to put them. # Legacy src kept locally for migration reference; not under version control. src_old/ @@ -280,3 +288,62 @@ benchmarks/.env # Local everos runtime data (memory root, indexes, OME state) .everos*/ + +# --------------------------------------------------------------------------- +# Internal-only benchmark scaffolding — kept in the working tree, never shipped +# --------------------------------------------------------------------------- +# These exist to run ablations on this machine, not to let anyone reproduce a +# published number. Two things make them unshippable rather than merely +# unpolished: they hardcode absolute paths into a private workspace +# (/Evermind/..., /root/fullrun, /root/.everos_bench), and the audit set compares +# against reference harnesses that live outside this repository and are not +# published — so a user who cloned this would get code that cannot run at all. +# +# The shippable harness is what remains: run.py + config.py + adapters/ + +# configs/ + metrics/ + scripts/reproduce.sh + README.md + .env.example. + +# Everything that measures something OTHER than a benchmark's published number: +# extraction-backbone arms, retrieval-policy arms, and the two store helpers that +# only those flows use (rewriting an existing store's episodes with a different +# extractor, and waiting for the index to catch up afterwards). Both sweep drivers +# read checkpoints out of a private path, and one needs a train/test split file +# from another repository, so neither can run anywhere else. +# +# The shipped surface is the reproduction and nothing else: reproduce.sh + run.py + +# config.py + adapters/ + configs/ + metrics/ + README.md + .env.example. +benchmarks/ablations/ + +# Migration audit. Every file here resolves paths under Evaluation/ to diff this +# harness against the reference implementations it was ported from; without that +# checkout none of it executes. MIGRATION*.md are the internal record of that +# port, not user documentation. +benchmarks/audit/ + +# Tests that require the same absent checkout. benchmark_parity_env.py says it +# plainly: 269 of the 395 benchmark tests skip when the references are missing, +# and `make test` still reports success — so shipping them hands users a suite +# that is permanently green and permanently vacuous. +tests/unit/benchmark_parity_env.py +tests/unit/test_benchmark_differential.py +tests/unit/test_benchmark_differential_audit.py +tests/unit/test_benchmark_differential_lme_emb.py +tests/unit/test_benchmark_differential_phases.py +tests/unit/test_benchmark_differential_pool.py +tests/unit/test_benchmark_differential_surface.py +tests/unit/test_benchmark_parity.py +tests/unit/test_benchmark_parity_gate.py +tests/unit/test_benchmark_parity_reported.py +# Also diffs against LoCoMo/EverOS/test_locomo.py (line 27), so it belongs here +# rather than in the shipped suite. +tests/unit/test_benchmark_client_and_gate.py + +# Pins the ablation tooling above, so it goes with it. A test whose subject is not +# shipped would fail at import on a fresh clone, not skip. +tests/unit/test_benchmark_reextract_sampling.py +tests/unit/test_benchmark_metrics_and_reextract.py +tests/unit/test_benchmark_store_tools.py +tests/unit/test_benchmark_ablation_tools.py + +# Hardcodes four store paths on this machine (/root/smoke_*). Marked `slow` so +# it is deselected by default, but the paths would still be published. +tests/unit/test_multiround_per_store.py diff --git a/Makefile b/Makefile index 139f16cec..768c09ad4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci clean +.PHONY: help install install-deps lint docs-check check-commits check-pr-title check-assets check-file-sizes check-deprecated-names check-github-docs check-cjk check-datetime openapi check-openapi format test integration package cov ci verify-parity clean help: @echo "Targets:" @@ -36,8 +36,8 @@ install: install-deps uv run pre-commit install --hook-type commit-msg lint: - uv run ruff check src tests - uv run ruff format --check src tests + uv run ruff check src tests benchmarks + uv run ruff format --check src tests benchmarks uv run lint-imports uv run python scripts/check_repo_assets.py uv run python scripts/check_file_sizes.py @@ -105,12 +105,23 @@ check-openapi: uv run python scripts/dump_openapi.py --check format: - uv run ruff check --fix src tests - uv run ruff format src tests + uv run ruff check --fix src tests benchmarks + uv run ruff format src tests benchmarks test: uv run pytest tests/unit -v +# The differential tests compare against a checkout outside this repository, so all of +# them skip when it is absent -- 269 of the 395 benchmark tests, and `make test` still +# reported success. This target refuses to skip: run it before any evaluation, because a +# number produced by an unverified harness is worse than no number. +verify-parity: + BENCHMARK_PARITY_STRICT=1 uv run pytest tests/unit -q -k benchmark + uv run python benchmarks/audit/check_parity.py + uv run python benchmarks/audit/check_protocol.py + uv run python benchmarks/audit/coverage.py + uv run python benchmarks/audit/partition.py + integration: uv run pytest tests/integration -v diff --git a/benchmarks/.env.example b/benchmarks/.env.example index b8af07bb6..150e816a2 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -4,3 +4,69 @@ ANSWER_API_KEY=sk-... ANSWER_BASE_URL=https://openrouter.ai/api/v1 JUDGE_API_KEY=sk-... JUDGE_BASE_URL=https://openrouter.ai/api/v1 + +# --- Infrastructure the servers need. Model choice does NOT belong here: +# it lives in configs/.toml, which is the single source of the +# experiment's identity. A model name in this file was silently overridden by +# the config at launch while still being what the run record reported. --- +# Required when using --servers: the fleet inherits this process's environment, and a +# server whose LLM is unconfigured aborts at startup ("LLM api_key and base_url is not +# configured"). `everos init` only writes a blank template, so these belong here rather +# than in each generated everos.toml. +EVEROS_LLM__BASE_URL=https://openrouter.ai/api/v1 +EVEROS_LLM__API_KEY= +EVEROS_LLM__TIMEOUT_SECONDS=300 +EVEROS_EMBEDDING__BASE_URL=http://127.0.0.1:9200/v1 +EVEROS_EMBEDDING__MODEL=Qwen3-Embedding-4B +EVEROS_EMBEDDING__API_KEY= +# Engine-wide cap on concurrent extractions. The default of 20 throttles a bulk ingest +# long before the machine does -- each slot spends its time waiting on a remote call. +EVEROS_OME_MAX_CONCURRENT_RUNS=64 + +# --------------------------------------------------------------------------- +# Paths (all optional — every one has a working default) +# --------------------------------------------------------------------------- +# LEFT EMPTY ON PURPOSE. Each of these has a default that works out of the box, and +# an empty value falls through to it. A placeholder like `/path/to/Evaluation` would +# NOT: it is a set variable, so it wins over the default, and the run then writes to +# a directory literally called `/path/to/...`. A value that looks configured but is +# not is worse than no value at all. +# +# Fill one in only when the real path is somewhere else. + +# Datasets. Default: benchmarks/data/. +# locomo -> benchmarks/data/locomo10.json +# longmemeval -> benchmarks/data/longmemeval_s.json +# subtlememory -> benchmarks/data/subtlememory/ (a directory) +# evermembench -> benchmarks/data/evermembench.json +BENCH_DATA_LOCOMO= +BENCH_DATA_LONGMEMEVAL= +BENCH_DATA_SUBTLEMEMORY= +BENCH_DATA_EVERMEMBENCH= + +# EverMemBench only: its raw release, read to recover session names the converted +# file does not carry. Default: benchmarks/data/raw/EverMemBench-Dynamic. +EVERMEMBENCH_RAW_ROOT= + +# Where runs are written. Default: benchmarks/results//. Point this at a +# directory outside the repository when the filesystem holding it is near capacity — +# a full disk surfaces as [Errno 28] on every answer write, which is retried and then +# recorded as a failed question rather than as an outage. +BENCH_EVAL_ROOT= + +# --------------------------------------------------------------------------- +# The multi-round retrieval decider (optional) +# --------------------------------------------------------------------------- +# Leave BOTH empty and the decider runs the model from [llm] — the single-model +# setup, which needs no extra configuration and is what a first run should use. +# +# Set them TOGETHER to give the decider its own model. A name without its endpoint +# returns 404 on every call, and the retrieval loop then falls back to a fixed core +# and still reports a complete result. run.py probes the decider at startup and +# refuses to run rather than let that happen quietly. +# +# Our published numbers used BENCH_DECIDER_MODEL=qwen3.6-27B. +BENCH_DECIDER_MODEL= +BENCH_DECIDER_BASE_URL= +# Any non-empty value for a self-hosted server; a hosted API needs its real key. +BENCH_DECIDER_API_KEY=EMPTY diff --git a/benchmarks/README.md b/benchmarks/README.md index 416f683bf..6d711981e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,338 +1,127 @@ -# Running the LoCoMo Benchmark +# Running the benchmarks -EverOS ships a self-contained runner for the -[LoCoMo](https://github.com/snap-research/locomo) (Long Conversation Memory) -benchmark ([Maharana et al., 2024](https://arxiv.org/abs/2402.17753)). -LoCoMo evaluates how well a memory system retrieves facts from long -multi-session dialogues across four question categories: **single-hop**, -**multi-hop**, **open-domain**, and **temporal**. This guide walks through -reproducing EverOS's LoCoMo retrieval scores locally. +One runner, four benchmarks, four stages: ADD, SEARCH, ANSWER, JUDGE. +Per-benchmark rules live in `benchmarks/adapters/.py`. -## Pipeline at a glance - -Each conversation runs through a four-stage pipeline: - -``` -ADD ──► wait_ready ──► SEARCH ──► ANSWER ──► JUDGE - │ │ │ │ │ - │ ingest msgs & query EverOS generate LLM-as-judge - │ flush per-session per QA pair answers majority vote - │ into EverOS from (judge_runs×) - │ context - ▼ - cascade + OME drain - (per-conv polling) -``` - -- **ADD** — sends LoCoMo sessions to EverOS (`/add` + `/flush`), then polls - cascade and OME queues until the conversation's data is fully indexed. -- **SEARCH** — queries EverOS `/search` for each QA question. -- **ANSWER** — feeds retrieved episodes to an LLM to generate an answer. -- **JUDGE** — an LLM judge scores each answer as CORRECT or WRONG against - the gold answer; runs `judge_runs` times per question and majority-votes. - -Stages are **independently re-runnable** — each reads from and writes to -JSONL files, so you can re-judge with a different model without re-ingesting -or re-searching. - -Multiple conversations run **in parallel** via `conversations_concurrency`. -Within each conversation, search and eval questions run concurrently via -`search_concurrency` and `eval_concurrency`. - -## Contents - -- [Prerequisites](#prerequisites) -- [Configuration](#configuration) -- [1. Prepare the dataset](#1-prepare-the-dataset) -- [2. Start the server](#2-start-the-server) -- [3. Run the benchmark](#3-run-the-benchmark) -- [4. Output](#4-output) -- [CLI reference](#cli-reference) -- [Notes](#notes) +> Also available in Chinese: [README.zh.md](README.zh.md). --- -## Prerequisites - -- A working EverOS installation — complete **all steps** in - [QUICKSTART.md](../QUICKSTART.md) (configure providers, start server, - verify search works — not just `/health`) -- Python 3.12+ with `tqdm` installed (`pip install tqdm`) -- EverOS configured for chat-only extraction — in your `everos.toml`: - - ```toml - [memorize] - mode = "chat" - ``` - - And in `ome.toml`, disable strategies the benchmark does not use: - - ```toml - [strategies.extract_foresight] - enabled = false - - [strategies.extract_user_profile] - enabled = false - ``` - - This keeps episode extraction, `extract_atomic_facts`, and - `trigger_profile_clustering` (agentic search relies on clusters), - while cutting unnecessary LLM calls from foresight and profile - extraction. - -- Copy `benchmarks/.env.example` → `benchmarks/.env` and fill in your API - keys: +## 1. Install dependencies ```bash -cp benchmarks/.env.example benchmarks/.env -# Edit benchmarks/.env: -ANSWER_API_KEY=sk-... # LLM for generating answers -ANSWER_BASE_URL=https://openrouter.ai/api/v1 -JUDGE_API_KEY=sk-... # LLM for judging answers -JUDGE_BASE_URL=https://openrouter.ai/api/v1 +uv sync ``` -Keys are comma-separated for round-robin failover (e.g. -`ANSWER_API_KEY=sk-aaa,sk-bbb`). More keys raise the effective RPM -ceiling, which lets you increase `eval_concurrency` in `config.toml` for -faster answer/judge throughput. - -## Configuration - -The only required configuration is provider credentials — copy -`benchmarks/.env.example` → `benchmarks/.env` and fill in your API keys -(already done in [Prerequisites](#prerequisites)). - -Everything else has sensible defaults in `benchmarks/config.toml` — see -the comments in that file for tunable parameters. +## 2. Get the data -## 1. Prepare the dataset - -LoCoMo 10 contains 10 multi-session conversations (~50 sessions each, -~150 QA pairs per conversation across 4 categories, adversarial -category excluded). - -```bash -mkdir -p data -curl -o data/locomo10.json \ - https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json -``` - -## 2. Start the server - -Raise the file descriptor limit **before** starting — concurrent agentic -searches open many LanceDB segment files simultaneously (EverOS compacts -segments automatically, but burst concurrency during benchmark can exceed -the default macOS limit of 256): - -```bash -ulimit -n 10240 -everos server start [--root ] -``` - -> **Important:** if you use a custom `--root`, pass the same path to the -> benchmark runner via `--everos-root` — the runner polls the cascade and -> OME databases under that root to know when data is ready. A mismatch -> causes silent readiness false-positives. - -## 3. Run the benchmark +| Benchmark | Target path | Source | +|---|---|---| +| LoCoMo | `benchmarks/data/locomo10.json` | [snap-research/locomo](https://github.com/snap-research/locomo) | +| LongMemEval | `benchmarks/data/longmemeval_s.json` | [xiaowu0162/longmemeval](https://huggingface.co/datasets/xiaowu0162/longmemeval) | +| EverMemBench | `benchmarks/data/evermembench.json` | [EverMind-AI/EverMemBench-Dynamic](https://huggingface.co/datasets/EverMind-AI/EverMemBench-Dynamic) | +| SubtleMemory | `benchmarks/data/subtlememory/` | [Yummytanmo/SubtleMemory](https://huggingface.co/datasets/Yummytanmo/SubtleMemory) | -All runs require `--run-name`, which becomes the `project_id` used for data -isolation (see [Run isolation](#run-isolation) below). +> **EverMemBench** — download the snapshot to `benchmarks/data/raw/EverMemBench-Dynamic/`, +> then run `python -m benchmarks.adapters.evermembench` once. +> +> **SubtleMemory** — keep the `persona_0` .. `persona_9` directory layout. -**Smoke test first** — verify end-to-end connectivity before a full run: +## 3. Configure the environment ```bash -python benchmarks/run.py --run-name smoke --smoke [--everos-root ] -``` - -**Full run (all 10 conversations):** - -```bash -python benchmarks/run.py --run-name locomo-agentic [--everos-root ] +cp benchmarks/.env.example benchmarks/.env ``` -**Single conversation:** +| Role | Keys | Notes | +|---|---|---| +| Extraction model | `EVEROS_LLM__MODEL` / `__API_KEY` / `__BASE_URL` | used by ADD | +| Retrieval model | `BENCH_DECIDER_MODEL` / `BENCH_DECIDER_BASE_URL` | the multi-round decider; defaults to `qwen3.6-27B`, and falls back to the extraction model when no endpoint is set | +| Answer model | `ANSWER_API_KEY` / `ANSWER_BASE_URL` | model name in the config | +| Judge model | `JUDGE_API_KEY` / `JUDGE_BASE_URL` | model name in the config | -```bash -python benchmarks/run.py --run-name locomo-agentic --conv 0 [--everos-root ] -``` +Models, `top_k`, retrieval settings and concurrency are set in +`benchmarks/configs/.toml`. `reproduce.sh` passes no model overrides. -**Skip ingest, re-run search + answer + judge:** +## 4. Run ```bash -python benchmarks/run.py --run-name locomo-agentic --stages search answer judge -``` - -**Re-judge only (reuse existing answer JSONL):** +# all conversations, all four stages +DATASET=locomo bash benchmarks/reproduce.sh -```bash -python benchmarks/run.py --run-name locomo-agentic --stages judge +# smoke test: one conversation, 10 sampled questions +DATASET=locomo CONV=0 bash benchmarks/reproduce.sh --smoke ``` -## 4. Output +| Variable | Default | Values | +|---|---|---| +| `DATASET` | `locomo` | `locomo` \| `longmemeval` \| `subtlememory` \| `evermembench` | +| `CONV` | `all` | conversation indices | +| `STAGES` | `add search answer judge` | any subset, in order | +| `RUN` | derived | result directory name | -Output root is `benchmarks/results//`: +`reproduce.sh` starts and stops its own EverOS servers. -``` -benchmarks/results// -├── run_spec.json # reproducibility snapshot (git hash, config, stages) -├── conv0/ -│ ├── search_.jsonl # per-question search results -│ ├── answer_.jsonl # per-question generated answers -│ ├── judge_.jsonl # per-question judge verdicts -│ └── error.log # only on failure — full traceback -├── conv1/ … conv9/ -├── report.json # aggregate accuracy by method + category -└── report.txt # human-readable accuracy table -``` +| Stage | Action | +|---|---| +| ADD | streams conversations into EverOS, which extracts memories | +| SEARCH | one retrieval per question; records the injected episodes | +| ANSWER | answers each question from those episodes | +| JUDGE | grades each answer against the reference | -`report.json` and `report.txt` are written after all conversations finish -(only when the `judge` stage is included). +Each stage appends per-conversation JSONL and skips entries already present, so +an interrupted run resumes. -**Sample `report.txt`:** +## 5. Output ``` -================================================================ - EverOS LoCoMo Benchmark Report -================================================================ - -Run Info - Run name: locomo-agentic - Generated: 2026-06-28T14:30:00+00:00 - Git hash: abc1234 - EverOS version: 1.1.0 - Python: 3.12.11 - Conversations: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - Stages: ['add', 'search', 'answer', 'judge'] - -Configuration - Answer model: gpt-4.1-mini - Judge model: gpt-4o-mini - Judge runs: 3 - Top-k: 10 - Eval owner: speaker_a - ----------------------------------------------------------------- - Method: agentic ----------------------------------------------------------------- - - Max accuracy: 93.4% (best of 3 judge runs / mean / majority) - Majority: 93.3% (1437/1540) - Mean accuracy: 93.3% (avg across 3 judge runs) - - Per category: - 1. single-hop 94.0% (265/282) - 2. multi-hop 91.0% (292/321) - 3. open-domain 80.2% (77/96) - 4. temporal 95.5% (803/841) - - Per conversation: - conv0 93.4% (142/152) - conv1 96.3% (78/81) - ... - - Search: 1540 queries, avg 23.1s, p50 19.4s, max 142.3s - Answer: 1540 questions, avg 4.7s, 7,224,168 tokens - Judge: 1540 questions × 3 runs, 2,335,683 tokens, unanimous 95.2% - - Total tokens: 9,559,851 +benchmarks/results/// +├── report.txt summary +├── report.json machine-readable summary +├── run_spec.json models served, endpoints, package versions, knobs +├── store/ built by ADD; absent when --everos-root points elsewhere +├── conv/ +│ ├── search_.jsonl +│ ├── answer_.jsonl +│ └── judge_.jsonl +└── traces/ per-round retrieval traces ``` -## CLI reference +`run_spec.json` records the models actually served and the `everalgo` package +versions, which determine what a store contains. -| Flag | Default | Description | -|---|---|---| -| `--run-name` | *(required)* | Run name — maps to `project_id` for data isolation | -| `--conv` | `0 1 2 … 9` | Conversation indices to run | -| `--stages` | `add search answer judge` | Pipeline stages to execute | -| `--config` | `config` | TOML config name (without `.toml` extension) | -| `--base-url` | `http://localhost:8000` | EverOS server address | -| `--everos-root` | `~/.everos` | EverOS root path (for cascade/OME queue polling) | -| `--data-path` | `data/locomo10.json` | Path to LoCoMo dataset JSON | -| `--smoke` | off | Smoke mode: 2 convs, first 50 msgs each, 10 QA (stratified), `judge_runs=1` | - -## Notes - -### Evaluation methodology +## 6. Results -The runner uses an **LLM-as-Judge** approach: a judge LLM receives the -question, the gold answer, and the generated answer, then outputs `CORRECT` -or `WRONG`. Each question is judged `judge_runs` times (default 3); the -final verdict is a **majority vote**. Accuracy = correct / total per method -and per category. +Each benchmark's published number, produced by the configuration in +`benchmarks/configs/.toml`. -The four LoCoMo question categories test different retrieval capabilities: - -| Category | Name | Tests | -|---|---|---| -| 1 | single-hop | Direct fact retrieval from one episode | -| 2 | multi-hop | Reasoning across multiple episodes | -| 3 | open-domain | General knowledge grounded in conversation | -| 4 | temporal | Time-sensitive questions requiring date reasoning | +| Benchmark | Questions | Accuracy | Decider | Answer model | +|---|---|---|---|---| +| LoCoMo | 1,540 | **94.42** | `deepseek/deepseek-v4-flash-0731` | `openai/gpt-4.1-mini` | +| LongMemEval | 500 | **94.00** | `qwen3.6-27B` | `google/gemini-3.6-flash` | +| EverMemBench | 2,400 | **66.67** | `qwen3.6-27B` | `google/gemini-3-flash-preview` | +| SubtleMemory | 1,522 | **71.75** | `qwen3.6-27B` | `openai/gpt-5.4` | -Category 5 (adversarial — questions with no answer in the conversation) is -excluded from evaluation. +> **EverMemBench** — scores as the mean of its nine category columns, which is +> how the benchmark reports it. +> +> **SubtleMemory** — routes each question between two answer contracts on whether +> fact extraction found a conflict; the adapter does this automatically. -### Run isolation -Each benchmark run is scoped by three identifiers: - -| Scope | Value | Purpose | -|---|---|---| -| `app_id` | `locomo_benchmark` | Fixed; separates benchmark data from production | -| `project_id` | `--run-name` value | Per-experiment isolation | -| `owner_id` | `_conv` | Per-conversation memory partition | - -Two runs with the **same** `--run-name` share the same memory corpus — -useful when re-running later stages, but problematic if you want a clean -ingest. Use distinct names (e.g. `locomo-agentic`, `locomo-hybrid`) for -independent experiments. - -### Stage independence - -Each stage reads from and writes to JSONL files in `conv/`. This means: - -- `--stages search` reads from the EverOS server (requires prior `add`). -- `--stages answer` reads `search_.jsonl` (requires prior `search`). -- `--stages judge` reads `answer_.jsonl` (requires prior `answer`). - -You can swap the judge model and re-run `--stages judge` without touching -ingest or search. - -### Smoke mode - -`--smoke` is a **pipeline sanity check**, not a scored run. It forces: - -- 2 conversations (conv 0, 1) running in parallel -- First 50 messages each (across however many sessions that covers) -- 10 QA pairs per conversation, stratified-sampled to cover all categories -- `judge_runs=1` (no majority vote) - -Use it to verify end-to-end connectivity before committing to a full run. - -### Runtime estimates - -Rough estimates with default settings (varies by provider latency): - -| Scope | Time | Token cost (approx.) | -|---|---|---| -| Smoke | 2–5 min | ~80k tokens | -| Single conv (full) | 15–30 min | ~1M tokens | -| Full 10-conv run | 2–4 hours | ~10M tokens | - -The `add` + `wait_ready` phase dominates wall-clock time; LLM calls -(answer + judge) dominate token cost. - -### Troubleshooting +--- -| Symptom | Cause | Fix | -|---|---|---| -| `Connection refused` on run | EverOS server not running | `everos server start` | -| `ANSWER_API_KEY not set` | Missing `.env` | Copy `.env.example` → `.env`, fill keys | -| `Timeout after 1800s` in wait_ready | Cascade/OME still processing | Increase `cascade_timeout` in config.toml or check server logs | -| `OME task(s) failed` warning | OME strategy crashed | Check `everos cascade status`; data may be incomplete | -| `Missing search_*.jsonl` | Running `answer` without prior `search` | Add `search` to `--stages` or run it first | -| `Too many open files (os error 24)` | LanceDB FD exhaustion from concurrent searches | Lower `search_concurrency` in config.toml (agentic needs more FDs per query) or raise `ulimit -n` | -| Low accuracy across all categories | Embedding/rerank not configured | Verify `everos.toml` has working embedding + rerank providers | -| `conv/error.log` exists | Unhandled exception in that conversation | Read the traceback; other conversations are unaffected | +## run.py flags + +| Flag | Meaning | +|---|---| +| `--conv` | conversation indices, or `all` | +| `--stages` | any of `add search answer judge` | +| `--everos-root` | store to use; default `//store` | +| `--servers` | EverOS servers to run in parallel | +| `--results-root` | output root; default `benchmarks/results/` | +| `--data-path` | dataset path for one run | +| `--methods` | `llm_multiround` \| `hybrid` \| `agentic` | +| `--answer-model` / `--judge-model` | model override for one run | +| `--decider-model` / `--decider-base-url` | decider; set both | +| `--smoke` | 10 sampled questions per conversation | diff --git a/benchmarks/README.zh.md b/benchmarks/README.zh.md new file mode 100644 index 000000000..1dff46ab8 --- /dev/null +++ b/benchmarks/README.zh.md @@ -0,0 +1,123 @@ +# 评测运行指南 + +一个运行器,四个评测,四个阶段:ADD、SEARCH、ANSWER、JUDGE。 +各评测自己的规则在 `benchmarks/adapters/.py`。 + +> English version: [README.md](README.md). + +--- + +## 1. 安装依赖 + +```bash +uv sync +``` + +## 2. 准备数据 + +| 评测 | 目标路径 | 来源 | +|---|---|---| +| LoCoMo | `benchmarks/data/locomo10.json` | [snap-research/locomo](https://github.com/snap-research/locomo) | +| LongMemEval | `benchmarks/data/longmemeval_s.json` | [xiaowu0162/longmemeval](https://huggingface.co/datasets/xiaowu0162/longmemeval) | +| EverMemBench | `benchmarks/data/evermembench.json` | [EverMind-AI/EverMemBench-Dynamic](https://huggingface.co/datasets/EverMind-AI/EverMemBench-Dynamic) | +| SubtleMemory | `benchmarks/data/subtlememory/` | [Yummytanmo/SubtleMemory](https://huggingface.co/datasets/Yummytanmo/SubtleMemory) | + +> **EverMemBench** —— 将快照下载到 `benchmarks/data/raw/EverMemBench-Dynamic/`, +> 然后执行一次 `python -m benchmarks.adapters.evermembench`。 +> +> **SubtleMemory** —— 保留 `persona_0` .. `persona_9` 目录结构。 + +## 3. 配置环境 + +```bash +cp benchmarks/.env.example benchmarks/.env +``` + +| 角色 | 键 | 说明 | +|---|---|---| +| 抽取模型 | `EVEROS_LLM__MODEL` / `__API_KEY` / `__BASE_URL` | ADD 阶段使用 | +| 检索模型 | `BENCH_DECIDER_MODEL` / `BENCH_DECIDER_BASE_URL` | 多轮 decider;默认 `qwen3.6-27B`,未设端点时改用抽取模型 | +| 答题模型 | `ANSWER_API_KEY` / `ANSWER_BASE_URL` | 模型名在配置文件中 | +| 判分模型 | `JUDGE_API_KEY` / `JUDGE_BASE_URL` | 模型名在配置文件中 | + +模型、`top_k`、检索设置和并发在 `benchmarks/configs/.toml` 中设置。 +`reproduce.sh` 不传任何模型覆盖参数。 + +## 4. 运行 + +```bash +# 全部对话、四个阶段 +DATASET=locomo bash benchmarks/reproduce.sh + +# 冒烟测试:单个对话,抽样 10 题 +DATASET=locomo CONV=0 bash benchmarks/reproduce.sh --smoke +``` + +| 变量 | 默认 | 取值 | +|---|---|---| +| `DATASET` | `locomo` | `locomo` \| `longmemeval` \| `subtlememory` \| `evermembench` | +| `CONV` | `all` | 对话下标 | +| `STAGES` | `add search answer judge` | 任意子集,按顺序 | +| `RUN` | 自动推导 | 结果目录名 | + +`reproduce.sh` 自行启动和关闭 EverOS server。 + +| 阶段 | 动作 | +|---|---| +| ADD | 将对话流式送入 EverOS,由其抽取记忆 | +| SEARCH | 每题一次检索,记录注入的 episode | +| ANSWER | 基于这些 episode 回答每题 | +| JUDGE | 对照参考答案判分 | + +各阶段按对话追加 JSONL 并跳过已存在的条目,中断后可续跑。 + +## 5. 输出 + +``` +benchmarks/results/// +├── report.txt 汇总 +├── report.json 机器可读汇总 +├── run_spec.json 实际服务的模型、端点、包版本、参数 +├── store/ ADD 建立的库;--everos-root 指向别处时不存在 +├── conv/ +│ ├── search_.jsonl +│ ├── answer_.jsonl +│ └── judge_.jsonl +└── traces/ 逐轮检索 trace +``` + +`run_spec.json` 记录实际提供服务的模型,以及决定库内容的 `everalgo` 包版本。 + +## 6. 结果 + +各评测的已发表数字,由 `benchmarks/configs/.toml` 的配置产出。 + +| 评测 | 题量 | 准确率 | decider | 答题模型 | +|---|---|---|---|---| +| LoCoMo | 1,540 | **94.42** | `deepseek/deepseek-v4-flash-0731` | `openai/gpt-4.1-mini` | +| LongMemEval | 500 | **94.00** | `qwen3.6-27B` | `google/gemini-3.6-flash` | +| EverMemBench | 2,400 | **66.67** | `qwen3.6-27B` | `google/gemini-3-flash-preview` | +| SubtleMemory | 1,522 | **71.75** | `qwen3.6-27B` | `openai/gpt-5.4` | + +> **EverMemBench** —— 按其九个类目列的均值计分,这是该评测自身的报告方式。 +> +> **SubtleMemory** —— 按事实抽取是否发现冲突,在两套答题契约之间逐题路由, +> 适配器自动完成。 + + +--- + +## run.py 参数 + +| 参数 | 含义 | +|---|---| +| `--conv` | 对话下标,或 `all` | +| `--stages` | `add search answer judge` 的任意组合 | +| `--everos-root` | 使用的库;默认 `//store` | +| `--servers` | 并行的 EverOS server 数 | +| `--results-root` | 输出根目录;默认 `benchmarks/results/` | +| `--data-path` | 单次运行的数据集路径 | +| `--methods` | `llm_multiround` \| `hybrid` \| `agentic` | +| `--answer-model` / `--judge-model` | 单次运行的模型覆盖 | +| `--decider-model` / `--decider-base-url` | decider;两项须同时设置 | +| `--smoke` | 每个对话抽样 10 题 | diff --git a/benchmarks/adapters/__init__.py b/benchmarks/adapters/__init__.py new file mode 100644 index 000000000..5e374a68e --- /dev/null +++ b/benchmarks/adapters/__init__.py @@ -0,0 +1,36 @@ +"""Dataset adapters, resolved by name. + +``run.py`` never imports a specific adapter; it asks for one by ``--benchmark``. Adding +a benchmark means adding a module here and one line to the registry -- no change to the +pipeline. +""" + +from __future__ import annotations + +from types import ModuleType + +from . import evermembench, locomo, longmemeval, subtlememory + +_REGISTRY: dict[str, ModuleType] = { + "locomo": locomo, + "longmemeval": longmemeval, + "evermembench": evermembench, + "subtlememory": subtlememory, +} + + +def get(name: str) -> ModuleType: + """Return the adapter module for ``name``, or raise with the valid choices.""" + try: + return _REGISTRY[name] + except KeyError: + raise KeyError( + f"unknown benchmark {name!r}; choices: {sorted(_REGISTRY)}" + ) from None + + +def names() -> list[str]: + return sorted(_REGISTRY) + + +__all__ = ["get", "names"] diff --git a/benchmarks/adapters/_profile.py b/benchmarks/adapters/_profile.py new file mode 100644 index 000000000..d0b0d739c --- /dev/null +++ b/benchmarks/adapters/_profile.py @@ -0,0 +1,66 @@ +"""Rendering the owner's profile into an answer prompt. + +Shared because it was not: the renderer lived inside ``evermembench.py`` and every other +benchmark's context builder simply dropped the ``profiles`` argument. That made +``include_profile`` a no-op on three of the four benchmarks -- the server fetched the +profile, the harness threw it away, and two "profile on vs off" runs on LoCoMo and +LongMemEval therefore compared a configuration against itself. Their whole measured +difference (0.91 pp and 1.20 pp, McNemar p=0.10 and p=0.15) was decider nondeterminism. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +PROFILE_HEADING = "## User profile" +MEMORY_HEADING = "## Retrieved memories" + + +def render_profile_lines(profiles: Sequence[dict]) -> list[str]: + """The profile as dash-prefixed lines, or empty when there is nothing to show. + + Reads both shapes the search response uses: a row wrapping ``profile_data``, and the + profile object itself. + """ + out: list[str] = [] + for prof in profiles or (): + data = prof.get("profile_data") or prof + summary = str(data.get("summary") or "").strip() + if summary: + out.append(f"- {summary}") + for item in data.get("explicit_info") or []: + if not isinstance(item, dict): + continue + cat = str(item.get("category") or "").strip() + desc = str(item.get("description") or "").strip() + if desc: + out.append(f"- [{cat}] {desc}" if cat else f"- {desc}") + for item in data.get("implicit_traits") or []: + if not isinstance(item, dict): + continue + trait = str(item.get("trait") or item.get("name") or "").strip() + basis = str(item.get("basis") or "").strip() + if trait and basis: + out.append(f"- [trait] {trait} -- {basis}") + elif basis: + out.append(f"- [trait] {basis}") + return out + + +def with_profile_block(memories: str, profiles: Sequence[dict]) -> str: + """Put the profile ahead of the rendered memories, as its own labelled section. + + Kept out of the memory list rather than prepended to it: a profile is standing + context, with no timestamp to reason about and no session it belongs to. A model + told to weigh recency would otherwise treat it as one more dated memory. + + Returns ``memories`` unchanged when there is no profile, which is what keeps every + benchmark's prompt byte-identical to its reference harness while the flag is off. + """ + lines = render_profile_lines(profiles) + if not lines: + return memories + block = PROFILE_HEADING + "\n" + "\n".join(lines) + if not memories: + return block + return f"{block}\n\n{MEMORY_HEADING}\n{memories}" diff --git a/benchmarks/adapters/base.py b/benchmarks/adapters/base.py new file mode 100644 index 000000000..8c0074a68 --- /dev/null +++ b/benchmarks/adapters/base.py @@ -0,0 +1,160 @@ +"""Dataset adapters: the one place a benchmark's own shape is allowed to live. + +`run.py` drives ADD -> SEARCH -> ANSWER -> JUDGE and knows nothing about any particular +benchmark. Everything that differs between benchmarks answers four questions, and an +adapter is exactly those four answers: + + 1. load_units() -- how to read the conversations and questions off disk + 2. owner_of() -- what memory owner a question's answer lives under. Owner naming is + decided when the store is BUILT, so this must reproduce the + builder's convention rather than invent one. Getting it wrong + returns zero episodes and scores 0% with no error anywhere. + 3. gold_of() -- gold session ids, in the form THE STORE uses. Every benchmark + cites + evidence differently (haystack positions, D: dia + ids, original session names) and none of them matches the store + directly. + 4. judge_spec() -- which judge, and the answer prompt it grades against. The hybrid + judge's leniency clauses are keyed to LongMemEval's categories and + would mis-fire on any other benchmark, so the judge belongs to the + adapter, not to a shared scoring layer. + +Anything that is NOT one of those four belongs in run.py. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class DatasetAdapter(Protocol): + """Four questions, one benchmark.""" + + name: str + + def load_units(self, data_path: str) -> list[dict[str, Any]]: + """Return one entry per conversation/topic, each carrying its own questions. + + The shape run.py consumes is ``{"index": int, "sessions": [...], "qa": [...]}``; + an adapter is free to derive that however its source data is organised. + """ + ... + + def owner_of(self, unit: dict[str, Any], eval_owner: str) -> str: + """Memory owner id to query for this unit. + + ``eval_owner`` carries the config's preference where a benchmark has more than + one candidate (LoCoMo has two speakers); benchmarks with a single owner per unit + ignore it. + """ + ... + + def gold_of(self, unit: dict[str, Any], qa: dict[str, Any]) -> set[str]: + """Gold session ids for one question, already translated into store ids.""" + ... + + def sessions_of(self, unit: dict[str, Any]) -> list[dict[str, Any]]: + """Ingestible sessions for the ADD stage. + + Each session is ``{"session_idx": int, "messages": [...], "timestamp_ms": int}`` + and each message carries ``speaker`` / ``text`` / ``dia_id``. Only the ADD stage + needs this; a benchmark scored against a pre-built store never calls it. + + Splitting this out is what lets ADD run for anything other than LoCoMo: the + loader used to read ``unit["conversation"]`` directly, so every other dataset + died with ``KeyError: 'conversation'`` the moment ingestion started. + """ + raise NotImplementedError + + def judge_spec(self) -> dict[str, Any]: + """``{"judge": , "answer_prompt":