diff --git a/README.md b/README.md index a3882e0..40ac7c4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ lm=1,2 -> max_chunks=keep_chunks=30 lm=3,4 -> max_chunks=keep_chunks=20 ``` -On the final global-cache snapshot, RASST improves terminology accuracy over InfiniSST in all 24 evaluated cells, with positive BLEU deltas in 19/24 cells. +On the final global-cache snapshot, RASST improves terminology accuracy over InfiniSST in all 24 evaluated cells, with positive BLEU deltas in 19/24 cells. A target-term-masked BLEU audit is included in the tracked result directory; on the 24 cells with artifact-backed InfiniSST hypotheses, RASST keeps positive masked-term BLEU deltas in 19/24 cells. | Track | Avg. BLEU delta vs. InfiniSST | Avg. TERM_ACC delta vs. InfiniSST | | --- | ---: | ---: | @@ -26,6 +26,7 @@ On the final global-cache snapshot, RASST improves terminology accuracy over Inf ![Medicine main result](docs/results/main_result_global_cache30_30_20_20/medicine_main_result_global_cache30_30_20_20.png) The tracked result tables and figure sources are in [docs/results/main_result_global_cache30_30_20_20](docs/results/main_result_global_cache30_30_20_20/). +The retriever compute RTF audit is in [docs/results/rag_compute_rtf](docs/results/rag_compute_rtf/). ## Release Assets diff --git a/code/rasst/analysis/main_result/compute_masked_terms_quality.py b/code/rasst/analysis/main_result/compute_masked_terms_quality.py new file mode 100644 index 0000000..055c6e3 --- /dev/null +++ b/code/rasst/analysis/main_result/compute_masked_terms_quality.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Compute target-term-masked BLEU for the release main-result artifacts.""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import os +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +REPO_ROOT = Path(__file__).resolve().parents[4] +DEFAULT_RESULT_DIR = REPO_ROOT / "docs/results/main_result_global_cache30_30_20_20" +DEFAULT_MAIN_RESULT_TSV = DEFAULT_RESULT_DIR / "main_result.tsv" +DEFAULT_OUTPUT_TSV = DEFAULT_RESULT_DIR / "masked_terms_quality.tsv" +DEFAULT_COMPARE_TSV = DEFAULT_RESULT_DIR / "masked_terms_quality_compare_vs_infinisst.tsv" +DEFAULT_ARTIFACT_MAP_TSV = DEFAULT_RESULT_DIR / "masked_terms_artifacts.tsv" +DEFAULT_RELEASE_DATA_ROOT = Path( + "/mnt/taurus/data2/jiaxuanluo/RASST_release_runs/hf_datasets/rasst-main-result-data" +) +DEFAULT_MWERSEGMENTER_ROOT = Path("/mnt/taurus/home/jiaxuanluo/mwerSegmenter") + +OFFLINE_EVAL_PATH = REPO_ROOT / "code/rasst/eval/offline_sst_eval/offline_streamlaal_eval.py" + +LANG_DEFAULTS = { + "zh": {"tokenizer": "zh", "latency_unit": "char", "term_lang": "zh"}, + "ja": {"tokenizer": "ja-mecab", "latency_unit": "char", "term_lang": "ja"}, + "de": {"tokenizer": "13a", "latency_unit": "word", "term_lang": "de"}, +} + + +def _load_offline_eval_module() -> Any: + spec = importlib.util.spec_from_file_location("rasst_offline_streamlaal_eval", OFFLINE_EVAL_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"Cannot import offline eval module: {OFFLINE_EVAL_PATH}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _read_tsv(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def _read_single_tsv_row(path: Path) -> Dict[str, str]: + rows = _read_tsv(path) + if len(rows) != 1: + raise ValueError(f"Expected exactly one TSV row in {path}, found {len(rows)}") + return rows[0] + + +def _write_tsv(path: Path, rows: List[Dict[str, str]], fieldnames: Iterable[str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(fieldnames), delimiter="\t", lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def _load_artifact_map(path: Path) -> Dict[Tuple[str, str, str, str], Dict[str, str]]: + if not path.is_file(): + return {} + return { + (row["dataset"], row["method"], row["lang"], row["lm"]): row + for row in _read_tsv(path) + } + + +def _artifact_inputs( + release_data_root: Path, + dataset: str, + lang: str, +) -> Tuple[Path, Path, Path]: + if dataset == "acl_tagged_raw": + input_dir = release_data_root / f"main_result/inputs/acl_{lang}" + return ( + input_dir / "ref.txt", + input_dir / "audio.yaml", + release_data_root / "glossaries/acl6060_tagged_gt_raw_min_norm2.json", + ) + if dataset == "medicine_hardraw": + input_dir = release_data_root / f"main_result/inputs/medicine_{lang}" + return ( + input_dir / f"medicine.ref.{lang}__medicine5_hardraw.txt", + input_dir / "medicine.audio__medicine5_hardraw.yaml", + release_data_root / "glossaries/hard_medicine_glossary_raw_llm_judge_manual_zh215_unique212.json", + ) + raise ValueError(f"Unsupported dataset for masked terms quality: {dataset}") + + +def _source_eval_row(source_path: str) -> Tuple[Optional[Dict[str, str]], str]: + if not source_path: + return None, "empty_source_path" + path = Path(source_path) + if not path.is_file(): + return None, f"source_path_not_file:{source_path}" + try: + return _read_single_tsv_row(path), "ok" + except Exception as exc: + return None, f"source_tsv_unreadable:{exc}" + + +def _instances_log_from_source(row: Dict[str, str], source_eval: Optional[Dict[str, str]]) -> Tuple[str, str]: + if source_eval is not None: + instances_log = source_eval.get("instances_log", "").strip() + if instances_log and Path(instances_log).is_file(): + return instances_log, "ok" + if instances_log: + return instances_log, f"instances_log_not_file:{instances_log}" + + source_path = row.get("source_path", "").strip() + if not source_path or not Path(source_path).is_file(): + return "", "source_artifact_unavailable" + parent = Path(source_path).parent + for name in ("instances.strip_term.log", "instances.log"): + candidate = parent / name + if candidate.is_file(): + return str(candidate), "ok" + return "", f"no_instances_log_near:{source_path}" + + +def _mapped_source( + artifact_row: Optional[Dict[str, str]], +) -> Tuple[Optional[Dict[str, str]], str, str]: + if artifact_row is None: + return None, "", "missing_artifact_map" + + instances_log = artifact_row.get("instances_log", "").strip() + eval_results = artifact_row.get("eval_results", "").strip() + if not instances_log: + return None, "", "artifact_map_missing_instances_log" + if not Path(instances_log).is_file(): + return None, instances_log, f"artifact_instances_log_not_file:{instances_log}" + + if eval_results: + if not Path(eval_results).is_file(): + return None, instances_log, f"artifact_eval_results_not_file:{eval_results}" + try: + return _read_single_tsv_row(Path(eval_results)), instances_log, "ok" + except Exception as exc: + return None, instances_log, f"artifact_eval_results_unreadable:{exc}" + return None, instances_log, "ok" + + +def _set_mwer_env(mwersegmenter_root: Path) -> None: + os.environ.setdefault("MWERSEGMENTER_ROOT", str(mwersegmenter_root)) + path_items = os.environ.get("PATH", "").split(os.pathsep) + if str(mwersegmenter_root) not in path_items: + os.environ["PATH"] = str(mwersegmenter_root) + os.pathsep + os.environ.get("PATH", "") + + +def _as_float(text: str) -> Optional[float]: + try: + return float(text) + except (TypeError, ValueError): + return None + + +def _format_delta(lhs: str, rhs: str) -> str: + left = _as_float(lhs) + right = _as_float(rhs) + if left is None or right is None: + return "" + return f"{left - right:+.4f}" + + +def _compute_rows(args: argparse.Namespace) -> List[Dict[str, str]]: + offline_eval = _load_offline_eval_module() + rows = _read_tsv(Path(args.input_tsv)) + artifact_map = _load_artifact_map(Path(args.artifact_map)) + include_methods = {item.strip() for item in args.include_methods.split(",") if item.strip()} + + out_rows: List[Dict[str, str]] = [] + for row in rows: + dataset = row.get("dataset", "") + method = row.get("method", "") + lang = row.get("lang", "") + lm = row.get("lm", "") + if method not in include_methods: + continue + if dataset not in {"acl_tagged_raw", "medicine_hardraw"}: + continue + if lang not in LANG_DEFAULTS or not lm or lm == "NA": + continue + + artifact_key = (dataset, method, lang, lm) + mapped_eval, mapped_instances_log, mapped_status = _mapped_source(artifact_map.get(artifact_key)) + if mapped_status == "ok": + source_eval = mapped_eval + source_status = "ok" + instances_log = mapped_instances_log + instances_status = "ok" + else: + source_eval, source_status = _source_eval_row(row.get("source_path", "")) + instances_log, instances_status = _instances_log_from_source(row, source_eval) + base_out = { + "dataset": dataset, + "method": method, + "lang": lang, + "lm": lm, + "BLEU": row.get("BLEU", ""), + "MASKED_TERMS_BLEU": "", + "DELTA_MASKED_MINUS_BLEU": "", + "TERM_ACC": row.get("TERM_ACC", ""), + "TERM_FCR": source_eval.get("TERM_FCR", "") if source_eval else "", + "FALSE_COPY": source_eval.get("FALSE_COPY", "") if source_eval else "", + "NEG_TOTAL": source_eval.get("NEG_TOTAL", "") if source_eval else "", + "FALSE_COPY_TERMS": source_eval.get("FALSE_COPY_TERMS", "") if source_eval else "", + "MASKED_TERMS_HYP_REMOVED": "", + "MASKED_TERMS_REF_REMOVED": "", + "MASKED_TERMS_TYPES": "", + "instances_log": instances_log, + "source_path": row.get("source_path", ""), + "status": "", + "note": "", + } + if source_status != "ok" or instances_status != "ok": + base_out["status"] = "unavailable" + base_out["note"] = ";".join(part for part in (source_status, instances_status) if part != "ok") + out_rows.append(base_out) + continue + + ref_file, audio_yaml, glossary_path = _artifact_inputs(Path(args.release_data_root), dataset, lang) + missing_inputs = [str(path) for path in (ref_file, audio_yaml, glossary_path) if not path.is_file()] + if missing_inputs: + base_out["status"] = "unavailable" + base_out["note"] = "missing_inputs:" + ",".join(missing_inputs) + out_rows.append(base_out) + continue + + defaults = LANG_DEFAULTS[lang] + try: + masked = offline_eval._compute_masked_terms_bleu( + instances_path=Path(instances_log), + ref_file=ref_file, + audio_yaml=audio_yaml, + sacrebleu_tokenizer=defaults["tokenizer"], + latency_unit=defaults["latency_unit"], + target_lang=defaults["term_lang"], + glossary_path=glossary_path, + ) + except Exception as exc: + base_out["status"] = "error" + base_out["note"] = str(exc) + out_rows.append(base_out) + continue + + base_out["MASKED_TERMS_BLEU"] = f"{float(masked.bleu):.4f}" + base_out["DELTA_MASKED_MINUS_BLEU"] = _format_delta(base_out["MASKED_TERMS_BLEU"], base_out["BLEU"]) + base_out["MASKED_TERMS_HYP_REMOVED"] = masked.hyp_terms_removed + base_out["MASKED_TERMS_REF_REMOVED"] = masked.ref_terms_removed + base_out["MASKED_TERMS_TYPES"] = masked.term_types + base_out["status"] = "ok" + out_rows.append(base_out) + return out_rows + + +def _comparison_rows(metric_rows: List[Dict[str, str]]) -> List[Dict[str, str]]: + by_key = { + (row["dataset"], row["lang"], row["lm"], row["method"]): row + for row in metric_rows + } + compare: List[Dict[str, str]] = [] + for row in metric_rows: + if row["method"] != "RASST": + continue + key = (row["dataset"], row["lang"], row["lm"]) + baseline = by_key.get((key[0], key[1], key[2], "InfiniSST")) + out = { + "dataset": key[0], + "lang": key[1], + "lm": key[2], + "RASST_BLEU": row.get("BLEU", ""), + "InfiniSST_BLEU": baseline.get("BLEU", "") if baseline else "", + "delta_BLEU_vs_InfiniSST": "", + "RASST_MASKED_TERMS_BLEU": row.get("MASKED_TERMS_BLEU", ""), + "InfiniSST_MASKED_TERMS_BLEU": baseline.get("MASKED_TERMS_BLEU", "") if baseline else "", + "delta_MASKED_TERMS_BLEU_vs_InfiniSST": "", + "masked_delta_minus_original_delta": "", + "status": "ok", + "note": "", + } + out["delta_BLEU_vs_InfiniSST"] = _format_delta(out["RASST_BLEU"], out["InfiniSST_BLEU"]) + out["delta_MASKED_TERMS_BLEU_vs_InfiniSST"] = _format_delta( + out["RASST_MASKED_TERMS_BLEU"], + out["InfiniSST_MASKED_TERMS_BLEU"], + ) + out["masked_delta_minus_original_delta"] = _format_delta( + out["delta_MASKED_TERMS_BLEU_vs_InfiniSST"], + out["delta_BLEU_vs_InfiniSST"], + ) + if row.get("status") != "ok": + out["status"] = "rasst_unavailable" + out["note"] = row.get("note", "") + elif baseline is None: + out["status"] = "baseline_missing" + elif baseline.get("status") != "ok": + out["status"] = "baseline_unavailable" + out["note"] = baseline.get("note", "") + compare.append(out) + return compare + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input-tsv", default=str(DEFAULT_MAIN_RESULT_TSV)) + parser.add_argument("--output-tsv", default=str(DEFAULT_OUTPUT_TSV)) + parser.add_argument("--output-compare-tsv", default=str(DEFAULT_COMPARE_TSV)) + parser.add_argument("--artifact-map", default=str(DEFAULT_ARTIFACT_MAP_TSV)) + parser.add_argument("--release-data-root", default=str(DEFAULT_RELEASE_DATA_ROOT)) + parser.add_argument("--mwersegmenter-root", default=str(DEFAULT_MWERSEGMENTER_ROOT)) + parser.add_argument("--include-methods", default="RASST,InfiniSST") + args = parser.parse_args() + + _set_mwer_env(Path(args.mwersegmenter_root)) + metric_rows = _compute_rows(args) + metric_fields = [ + "dataset", + "method", + "lang", + "lm", + "BLEU", + "MASKED_TERMS_BLEU", + "DELTA_MASKED_MINUS_BLEU", + "TERM_ACC", + "TERM_FCR", + "FALSE_COPY", + "NEG_TOTAL", + "FALSE_COPY_TERMS", + "MASKED_TERMS_HYP_REMOVED", + "MASKED_TERMS_REF_REMOVED", + "MASKED_TERMS_TYPES", + "instances_log", + "source_path", + "status", + "note", + ] + _write_tsv(Path(args.output_tsv), metric_rows, metric_fields) + + compare_rows = _comparison_rows(metric_rows) + compare_fields = [ + "dataset", + "lang", + "lm", + "RASST_BLEU", + "InfiniSST_BLEU", + "delta_BLEU_vs_InfiniSST", + "RASST_MASKED_TERMS_BLEU", + "InfiniSST_MASKED_TERMS_BLEU", + "delta_MASKED_TERMS_BLEU_vs_InfiniSST", + "masked_delta_minus_original_delta", + "status", + "note", + ] + _write_tsv(Path(args.output_compare_tsv), compare_rows, compare_fields) + + ok_rows = sum(1 for row in metric_rows if row["status"] == "ok") + print(f"Wrote {args.output_tsv} ({ok_rows}/{len(metric_rows)} rows computed)") + print(f"Wrote {args.output_compare_tsv} ({len(compare_rows)} comparison rows)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/code/rasst/analysis/main_result/plot_masked_terms_bleu.py b/code/rasst/analysis/main_result/plot_masked_terms_bleu.py new file mode 100644 index 0000000..1aaf6f9 --- /dev/null +++ b/code/rasst/analysis/main_result/plot_masked_terms_bleu.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Plot target-term-masked BLEU for the final main-result snapshot.""" + +from __future__ import annotations + +import argparse +import csv +import shutil +from pathlib import Path +from typing import Dict, List, Sequence, Tuple + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + + +REPO_ROOT = Path(__file__).resolve().parents[4] +RESULT_DIR = REPO_ROOT / "docs/results/main_result_global_cache30_30_20_20" +FIGURE_DIR = REPO_ROOT / "figures/main_result_global_cache30_30_20_20" +DEFAULT_MASKED_DATA = RESULT_DIR / "masked_terms_quality.tsv" +DEFAULT_MAIN_DATA = RESULT_DIR / "main_result.tsv" +DEFAULT_PREFIX = FIGURE_DIR / "masked_terms_bleu_global_cache30_30_20_20" +DEFAULT_DOCS_PREFIX = RESULT_DIR / "masked_terms_bleu_global_cache30_30_20_20" + +DATASETS: Sequence[Tuple[str, str]] = ( + ("acl_tagged_raw", "ACL6060 tagged"), + ("medicine_hardraw", "Medicine hard/raw"), +) +LANGS: Sequence[Tuple[str, str]] = (("zh", "En-Zh"), ("de", "En-De"), ("ja", "En-Ja")) +METHODS = ("InfiniSST", "RASST") + +METHOD_STYLES = { + "InfiniSST": { + "color": "#2b6cb0", + "marker": "^", + "linestyle": "-", + "linewidth": 2.8, + "markersize": 9.5, + }, + "RASST": { + "color": "#d62728", + "marker": "*", + "linestyle": "-", + "linewidth": 3.0, + "markersize": 13.5, + }, +} + + +def load_rows(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def finite(value: str) -> float | None: + if value in {"", "NA"}: + return None + return float(value) + + +def row_key(row: Dict[str, str]) -> Tuple[str, str, str, str]: + return (row["dataset"], row["method"], row["lang"], row["lm"]) + + +def merge_rows( + masked_rows: Sequence[Dict[str, str]], + main_rows: Sequence[Dict[str, str]], +) -> List[Dict[str, str]]: + main_by_key = {row_key(row): row for row in main_rows if row.get("lm") != "NA"} + merged: List[Dict[str, str]] = [] + for row in masked_rows: + base = main_by_key.get(row_key(row)) + if base is None: + continue + out = dict(row) + out["StreamLAAL"] = base.get("StreamLAAL", "") + merged.append(out) + return merged + + +def plot_masked_bleu(rows: Sequence[Dict[str, str]], output_prefix: Path) -> None: + plt.rcParams.update( + { + "font.family": "serif", + "font.serif": ["DejaVu Serif", "Times New Roman", "Times"], + "font.size": 16, + "axes.titlesize": 18, + "axes.labelsize": 17, + "legend.fontsize": 16, + "xtick.labelsize": 14, + "ytick.labelsize": 14, + "axes.linewidth": 1.2, + "pdf.fonttype": 42, + "ps.fonttype": 42, + } + ) + fig, axes = plt.subplots(2, 3, figsize=(12.0, 6.4)) + handles: List[object] = [] + labels: List[str] = [] + + for row_idx, (dataset, dataset_label) in enumerate(DATASETS): + dataset_rows = [r for r in rows if r["dataset"] == dataset] + for col, (lang, lang_label) in enumerate(LANGS): + ax = axes[row_idx][col] + lang_rows = [r for r in dataset_rows if r["lang"] == lang] + x_values: List[float] = [] + y_values: List[float] = [] + plotted_methods = set() + + for method in METHODS: + method_rows = [ + r + for r in lang_rows + if r["method"] == method and r.get("status") == "ok" + ] + points: List[Tuple[float, float]] = [] + for item in sorted( + method_rows, + key=lambda value: int(value["lm"]) if value["lm"].isdigit() else 99, + ): + x = finite(item.get("StreamLAAL", "")) + y = finite(item.get("MASKED_TERMS_BLEU", "")) + if x is None or y is None: + continue + points.append((x, y)) + if not points: + continue + plotted_methods.add(method) + line = ax.plot( + [point[0] for point in points], + [point[1] for point in points], + label=method, + **METHOD_STYLES[method], + )[0] + x_values.extend(point[0] for point in points) + y_values.extend(point[1] for point in points) + if method not in labels: + handles.append(line) + labels.append(method) + + if dataset == "acl_tagged_raw" and lang == "zh" and "InfiniSST" not in plotted_methods: + ax.text( + 0.03, + 0.96, + "InfiniSST artifact\nunavailable", + transform=ax.transAxes, + va="top", + ha="left", + fontsize=10.5, + color="#555555", + ) + + if x_values: + x_low = min(x_values) + x_high = max(x_values) + pad = max((x_high - x_low) * 0.10, 90.0) + ax.set_xlim(x_low - pad, x_high + pad) + if y_values: + y_low = min(y_values) + y_high = max(y_values) + pad = max((y_high - y_low) * 0.12, 1.1) + ax.set_ylim(y_low - pad, y_high + pad) + + ax.grid(True, linestyle=":", linewidth=0.7, alpha=0.65) + if row_idx == 0: + ax.set_title(lang_label, fontweight="bold") + if row_idx == len(DATASETS) - 1: + ax.set_xlabel("StreamLAAL (ms)") + if col == 0: + ax.set_ylabel(f"{dataset_label}\nMasked-Term BLEU") + + if handles: + fig.legend( + handles, + labels, + loc="lower center", + ncol=len(labels), + frameon=True, + bbox_to_anchor=(0.5, 0.01), + columnspacing=1.8, + handlelength=2.4, + ) + fig.tight_layout(rect=(0.0, 0.09, 1.0, 1.0), w_pad=1.25, h_pad=1.5) + output_prefix.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_prefix.with_suffix(".png"), dpi=300) + fig.savefig(output_prefix.with_suffix(".pdf")) + plt.close(fig) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--masked-data", type=Path, default=DEFAULT_MASKED_DATA) + parser.add_argument("--main-data", type=Path, default=DEFAULT_MAIN_DATA) + parser.add_argument("--out-prefix", type=Path, default=DEFAULT_PREFIX) + parser.add_argument("--docs-prefix", type=Path, default=DEFAULT_DOCS_PREFIX) + args = parser.parse_args() + + rows = merge_rows(load_rows(args.masked_data), load_rows(args.main_data)) + plot_masked_bleu(rows, args.out_prefix) + print(f"wrote {args.out_prefix.with_suffix('.pdf')}") + print(f"wrote {args.out_prefix.with_suffix('.png')}") + + if args.docs_prefix: + args.docs_prefix.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(args.out_prefix.with_suffix(".pdf"), args.docs_prefix.with_suffix(".pdf")) + shutil.copy2(args.out_prefix.with_suffix(".png"), args.docs_prefix.with_suffix(".png")) + print(f"updated {args.docs_prefix.with_suffix('.pdf')}") + print(f"updated {args.docs_prefix.with_suffix('.png')}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/code/rasst/eval/offline_sst_eval/offline_streamlaal_eval.py b/code/rasst/eval/offline_sst_eval/offline_streamlaal_eval.py index fe0e27e..80acffe 100644 --- a/code/rasst/eval/offline_sst_eval/offline_streamlaal_eval.py +++ b/code/rasst/eval/offline_sst_eval/offline_streamlaal_eval.py @@ -61,13 +61,14 @@ import argparse import json import re +import shutil import subprocess import sys import tempfile from collections import Counter from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple import yaml @@ -159,6 +160,14 @@ class ParsedMetrics: neg_total: str = "" +@dataclass(frozen=True) +class MaskedTermsBleu: + bleu: str + hyp_terms_removed: str + ref_terms_removed: str + term_types: str + + _NUMBER_RE = r"-?(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)" _METRIC_TRIPLE_RE = re.compile(rf"^\s*({_NUMBER_RE})\s+({_NUMBER_RE})\s+({_NUMBER_RE})\s*$") _TERM_OUTPUT_TAG_RE = re.compile(r"", flags=re.IGNORECASE) @@ -183,6 +192,10 @@ def _tag_edge_space_regexes(*, include_short_t: bool) -> Tuple[re.Pattern[str], COMPUTE_ADOPTION_SCRIPT_REL = "eval/offline_sst_eval/compute_sentence_term_adoption.py" +def _normalise_space(text: str) -> str: + return re.sub(r"\s+", " ", str(text or "")).strip() + + def _strip_term_output_tags_with_mask( text: str, *, @@ -632,6 +645,270 @@ def _run_stream_laal_term( return p.stdout +_ALNUM_TERM_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+/#&%()-]*$") +_CJK_OR_KANA_RE = re.compile(r"[\u3040-\u30ff\u3400-\u9fff]") + + +def _term_to_mask_regex(term: str) -> re.Pattern[str]: + term_norm = _normalise_space(term) + if not term_norm: + raise ValueError("Cannot build a mask regex for an empty term.") + escaped = re.escape(term_norm).replace(r"\ ", r"\s+") + if _ALNUM_TERM_RE.fullmatch(term_norm): + return re.compile( + r"(? List[re.Pattern[str]]: + return [_term_to_mask_regex(term) for term in target_terms] + + +def _load_target_terms_for_masking(glossary_path: Path, target_lang: str) -> List[str]: + data = json.loads(_read_text(glossary_path)) + if isinstance(data, dict): + raw_entries: Iterable[Any] = data.values() + elif isinstance(data, list): + raw_entries = data + else: + raise ValueError(f"Unsupported glossary format for masked BLEU: {glossary_path}") + + terms: List[str] = [] + seen = set() + for entry in raw_entries: + if not isinstance(entry, dict): + continue + translations = entry.get("target_translations") + translation = "" + if isinstance(translations, dict): + translation = _normalise_space(translations.get(target_lang) or "") + if not translation: + translation = _normalise_space( + entry.get("translation") + or entry.get("target_translation") + or entry.get(target_lang) + or "" + ) + if not translation: + continue + key = translation.casefold() + if key in seen: + continue + seen.add(key) + terms.append(translation) + + # Longer terms must be removed before shorter overlapping terms. + terms.sort(key=lambda text: (len(text), text), reverse=True) + return terms + + +def _mask_target_terms(text: str, term_patterns: Sequence[re.Pattern[str]]) -> Tuple[str, int]: + masked = str(text or "") + removed = 0 + for pattern in term_patterns: + masked, count = pattern.subn(" ", masked) + removed += count + return _normalise_space(masked), removed + + +def _mwer_command() -> str: + cmd = shutil.which("mwerSegmenter") + if cmd: + return cmd + root = os.environ.get("MWERSEGMENTER_ROOT", "").strip() + if root: + candidate = Path(root) / "mwerSegmenter" + if candidate.is_file(): + return str(candidate) + raise RuntimeError("mwerSegmenter not found in PATH and MWERSEGMENTER_ROOT is not set") + + +def _segment_prediction_by_references( + prediction: str, + reference_sentences: List[str], + latency_unit: str, +) -> List[str]: + command = _mwer_command() + character_level = latency_unit == "char" + pred_text = str(prediction or "") + refs = [str(x or "") for x in reference_sentences] + if character_level: + pred_text = " ".join(pred_text) + refs = [" ".join(x) for x in refs] + + tmp_dir = tempfile.mkdtemp() + pred_file = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) + ref_file = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) + segments_path = Path(tmp_dir) / "__segments" + try: + pred_file.write(pred_text) + pred_file.flush() + ref_file.writelines(ref + "\n" for ref in refs) + ref_file.flush() + subprocess.run( + [command, "-mref", ref_file.name, "-hypfile", pred_file.name, "-usecase", "1"], + cwd=tmp_dir, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + segments = segments_path.read_text(encoding="utf-8", errors="replace").splitlines() + if character_level: + segments = [re.sub(r"(.)\s", r"\1", line).strip() for line in segments] + else: + segments = [line.strip() for line in segments] + if len(segments) != len(reference_sentences): + raise RuntimeError( + f"mwerSegmenter returned {len(segments)} segments for " + f"{len(reference_sentences)} references" + ) + return segments + finally: + pred_file.close() + ref_file.close() + for path in (Path(pred_file.name), Path(ref_file.name), segments_path): + path.unlink(missing_ok=True) + Path(tmp_dir).rmdir() + + +def _resegment_instances_for_bleu( + instances_path: Path, + ref_file: Path, + audio_yaml: Path, + latency_unit: str, +) -> List[Tuple[str, List[str], List[str]]]: + refs = _read_text(ref_file).splitlines() + audio = yaml.safe_load(_read_text(audio_yaml)) + if not isinstance(audio, list): + raise ValueError(f"Invalid audio yaml format (expect list): {audio_yaml}") + if len(audio) != len(refs): + raise ValueError( + f"audio yaml entries {len(audio)} != reference lines {len(refs)}. " + f"audio_yaml={audio_yaml} ref_file={ref_file}" + ) + + refs_by_wav: Dict[str, List[str]] = {} + for ref, item in zip(refs, audio): + if not isinstance(item, dict): + raise ValueError(f"Invalid audio yaml row in {audio_yaml}: {item!r}") + wav = _basename(item.get("wav", "")) + if not wav: + raise ValueError(f"Audio yaml row is missing wav in {audio_yaml}: {item!r}") + refs_by_wav.setdefault(wav, []).append(ref) + + predictions_by_wav: Dict[str, str] = {} + for row in _iter_jsonl(instances_path): + src = row.get("source") + if not isinstance(src, list) or not src: + raise ValueError(f"Instance is missing source[0] in {instances_path}: {row!r}") + wav = _basename(src[0]) + predictions_by_wav[wav] = str(row.get("prediction") or "") + + missing = sorted(set(refs_by_wav) - set(predictions_by_wav)) + extra = sorted(set(predictions_by_wav) - set(refs_by_wav)) + if missing or extra: + raise ValueError( + f"Reference/prediction wav mismatch for masked BLEU. " + f"missing_predictions={missing} extra_predictions={extra}" + ) + + groups: List[Tuple[str, List[str], List[str]]] = [] + for wav, wav_refs in refs_by_wav.items(): + wav_hyps = _segment_prediction_by_references( + predictions_by_wav[wav], + wav_refs, + latency_unit=latency_unit, + ) + if len(wav_hyps) != len(wav_refs): + raise ValueError( + f"mwerSegmenter returned {len(wav_hyps)} segments for {len(wav_refs)} refs " + f"for wav={wav}" + ) + groups.append((wav, wav_hyps, wav_refs)) + return groups + + +def _compute_masked_terms_bleu( + *, + instances_path: Path, + ref_file: Path, + audio_yaml: Path, + sacrebleu_tokenizer: str, + latency_unit: str, + target_lang: str, + glossary_path: Optional[Path] = None, + glossary_by_paper: Optional[Dict[str, Path]] = None, +) -> MaskedTermsBleu: + try: + import sacrebleu + except ImportError as exc: + raise RuntimeError( + "sacrebleu is required for MASKED_TERMS_BLEU. " + "Install requirements.txt or run from the release eval environment." + ) from exc + + groups = _resegment_instances_for_bleu( + instances_path=instances_path, + ref_file=ref_file, + audio_yaml=audio_yaml, + latency_unit=latency_unit, + ) + + terms_cache: Dict[Path, Tuple[List[str], List[re.Pattern[str]]]] = {} + + def terms_for_wav(wav: str) -> Tuple[List[str], List[re.Pattern[str]]]: + if glossary_by_paper is not None: + paper_id = _paper_id_from_wav_basename(wav) + if not paper_id or paper_id not in glossary_by_paper: + raise ValueError(f"No glossary found for wav={wav} paper_id={paper_id}") + selected = glossary_by_paper[paper_id] + else: + if glossary_path is None: + raise ValueError("glossary_path is required when glossary_by_paper is not set.") + selected = glossary_path + if selected not in terms_cache: + target_terms = _load_target_terms_for_masking(selected, target_lang) + terms_cache[selected] = (target_terms, _compile_term_mask_patterns(target_terms)) + return terms_cache[selected] + + masked_hyps: List[str] = [] + masked_refs: List[str] = [] + hyp_removed = 0 + ref_removed = 0 + term_types = set() + for wav, wav_hyps, wav_refs in groups: + target_terms, term_patterns = terms_for_wav(wav) + term_types.update(target_terms) + for hyp, ref in zip(wav_hyps, wav_refs): + masked_hyp, hyp_count = _mask_target_terms(hyp, term_patterns) + masked_ref, ref_count = _mask_target_terms(ref, term_patterns) + masked_hyps.append(masked_hyp) + masked_refs.append(masked_ref) + hyp_removed += hyp_count + ref_removed += ref_count + + score = sacrebleu.corpus_bleu( + masked_hyps, + [masked_refs], + tokenize=sacrebleu_tokenizer, + ).score + return MaskedTermsBleu( + bleu=str(score), + hyp_terms_removed=str(hyp_removed), + ref_terms_removed=str(ref_removed), + term_types=str(len(term_types)), + ) + + def _load_acl6060_dev(audio_yaml: Path, ref_file: Path) -> Tuple[List[Dict[str, Any]], List[str]]: data = yaml.safe_load(_read_text(audio_yaml)) refs = _read_text(ref_file).splitlines() @@ -1129,6 +1406,15 @@ def main() -> int: term_mismatch_examples=str(args.term_mismatch_examples), ) m = _parse_stream_laal_output(eval_out) + masked_terms_bleu = _compute_masked_terms_bleu( + instances_path=instances_path, + ref_file=ref_file, + audio_yaml=audio_yaml, + sacrebleu_tokenizer=sacrebleu_tokenizer, + latency_unit=latency_unit, + target_lang=term_lang, + glossary_path=glossary_acl6060, + ) if out_log: _write_text(out_log, eval_out) @@ -1176,6 +1462,10 @@ def main() -> int: "mode", "lang_code", "BLEU", + "MASKED_TERMS_BLEU", + "MASKED_TERMS_HYP_REMOVED", + "MASKED_TERMS_REF_REMOVED", + "MASKED_TERMS_TYPES", "StreamLAAL", "StreamLAAL_CA", "TERM_ACC", @@ -1206,6 +1496,10 @@ def main() -> int: args.mode, args.lang_code, m.bleu, + masked_terms_bleu.bleu, + masked_terms_bleu.hyp_terms_removed, + masked_terms_bleu.ref_terms_removed, + masked_terms_bleu.term_types, m.stream_laal, m.stream_laal_ca, m.term_acc, @@ -1256,6 +1550,15 @@ def main() -> int: term_mismatch_examples=str(args.term_mismatch_examples), ) m_full = _parse_stream_laal_output(eval_out_full) + masked_terms_bleu = _compute_masked_terms_bleu( + instances_path=instances_path, + ref_file=ref_file, + audio_yaml=audio_yaml, + sacrebleu_tokenizer=sacrebleu_tokenizer, + latency_unit=latency_unit, + target_lang=term_lang, + glossary_by_paper=_load_glossary_manifest(extracted_glossary_manifest), + ) # 2) TERM metrics by paper using extracted glossary. total_correct, total_terms, per_paper_counts = _compute_term_sums_extracted_by_paper( @@ -1336,6 +1639,10 @@ def main() -> int: "instances_log": str(instances_path), "extracted_glossary_manifest": str(extracted_glossary_manifest), "bleu": m_full.bleu, + "masked_terms_bleu": masked_terms_bleu.bleu, + "masked_terms_hyp_removed": masked_terms_bleu.hyp_terms_removed, + "masked_terms_ref_removed": masked_terms_bleu.ref_terms_removed, + "masked_terms_types": masked_terms_bleu.term_types, "stream_laal": m_full.stream_laal, "stream_laal_ca": m_full.stream_laal_ca, "term_correct_sum": total_correct, @@ -1370,6 +1677,10 @@ def main() -> int: "mode", "lang_code", "BLEU", + "MASKED_TERMS_BLEU", + "MASKED_TERMS_HYP_REMOVED", + "MASKED_TERMS_REF_REMOVED", + "MASKED_TERMS_TYPES", "StreamLAAL", "StreamLAAL_CA", "TERM_ACC", @@ -1400,6 +1711,10 @@ def main() -> int: args.mode, args.lang_code, m_full.bleu, + masked_terms_bleu.bleu, + masked_terms_bleu.hyp_terms_removed, + masked_terms_bleu.ref_terms_removed, + masked_terms_bleu.term_types, m_full.stream_laal, m_full.stream_laal_ca, term_acc, diff --git a/docs/results/main_result_global_cache30_30_20_20/README.md b/docs/results/main_result_global_cache30_30_20_20/README.md index c27f39b..7fff18c 100644 --- a/docs/results/main_result_global_cache30_30_20_20/README.md +++ b/docs/results/main_result_global_cache30_30_20_20/README.md @@ -18,6 +18,11 @@ directory is the small Git-tracked copy used for release documentation. | `main_result.tsv` | Full merged main-result table with baselines and RASST rows. | | `rasst24.tsv` | The 24 RASST cells used for the release main result. | | `compare_vs_infinisst_and_paper.tsv` | Per-cell BLEU and TERM_ACC deltas versus InfiniSST and the paper-exact RASST table. | +| `masked_terms_quality.tsv` | Per-row target-term-masked BLEU computed from available hypothesis artifacts. | +| `masked_terms_quality_compare_vs_infinisst.tsv` | Per-cell RASST versus InfiniSST target-term-masked BLEU comparison. | +| `masked_terms_artifacts.tsv` | Local artifact map for masked-term hypothesis logs not named in `main_result.tsv`. | +| `masked_terms_bleu_global_cache30_30_20_20.pdf/png` | Combined ACL tagged and Medicine hard/raw target-term-masked BLEU figure. | +| `artifacts/acl_tagged_raw_infinisst_zh/` | Local copies of ACL6060 tagged En-Zh InfiniSST no-RAG validation TSVs and ignored `instances.log` files. | | `new_main_result_tagged_global_cache30_30_20_20.pdf/png` | ACL tagged main-result figure. | | `medicine_main_result_global_cache30_30_20_20.pdf/png` | Medicine hardraw main-result figure. | @@ -43,12 +48,42 @@ The global policy improves BLEU over InfiniSST in 19 of 24 RASST cells. | `medicine_hardraw` | `ja` | 3/4 | +1.3330 | +0.4405 | | `medicine_hardraw` | `zh` | 4/4 | +3.1231 | +0.3540 | +## Masked-Term BLEU + +`MASKED_TERMS_BLEU` removes target-side glossary translations from both the +hypothesis and reference after the same mWER resegmentation used by the regular +BLEU scorer. It is a non-term quality check: if the RASST delta disappears after +masking, the original BLEU gain is mostly terminology-driven. + +Among the 24 cells with artifact-backed InfiniSST hypothesis logs, RASST keeps +positive masked-term BLEU deltas in 19/24 cells. The average delta drops from +`+1.7373` regular BLEU to `+1.1238` masked-term BLEU, so terminology accounts +for part of the BLEU gain but does not fully explain it in the artifact-backed +comparison. + +| Track | Cells | Masked BLEU wins | Avg delta regular BLEU | Avg delta masked-term BLEU | +| --- | ---: | ---: | ---: | ---: | +| ACL6060 tagged | 12 | 10/12 | +1.9110 | +0.9919 | +| Medicine hard/raw | 12 | 9/12 | +1.5637 | +1.2557 | +| Artifact-backed total | 24 | 19/24 | +1.7373 | +1.1238 | + +ACL6060 tagged zh InfiniSST hypothesis logs were backfilled from the old +InfiniSST rank16 baseline no-RAG ACL6060-only v3 run. The copied logs reproduce +the published En-Zh InfiniSST BLEU and StreamLAAL rows; `masked_terms_artifacts.tsv` +maps those local copies into the masked-term scorer without changing +`main_result.tsv` provenance. + ## Checksums ```text 1929cc8d7883c99f136d104adb6efe92199ca529d21a7f6b9bf683ab4f50f95b main_result.tsv 278af3606fe863c2f93d3281dd1714f19a4a3dffe4e6749ff3190b7d4c306406 rasst24.tsv 11221fabdde32e42862014454b81faa56ad9822646b790cc250565d60f1941d0 compare_vs_infinisst_and_paper.tsv +3265c5b3c808c44f5227ebe5626b5a7ff194da6535267a17eb88f7e65604af14 masked_terms_quality.tsv +ac06dec613a35a7c1cc1d04a56dbe7534834178ef229d89cb3357b6f2a727c6b masked_terms_quality_compare_vs_infinisst.tsv +2964438ed82a43a8ba7e394ce9b1ceacc9c0788a6a6726b502b3768401215b7a masked_terms_artifacts.tsv +7c1147cd60c5bfef2007a71405d53af5c8891e5ac3f24e2a8366eae87e30349f masked_terms_bleu_global_cache30_30_20_20.pdf +e55f6c7d2fe309ac1125a47400b28975e259554285af8e9a48300aca41c6c680 masked_terms_bleu_global_cache30_30_20_20.png f2c9d4df06dc0888e347a59418cb7b480ea0d102a34b29142e36c258c09aa31b new_main_result_tagged_global_cache30_30_20_20.pdf 319cc377ba4bda7ddd1311f3e537be51029b6543c3007498df496353eca6de3a new_main_result_tagged_global_cache30_30_20_20.png 845d904908343ea6036fa79fd3aebd06ef96daad6459b8096f9d609c868bcd10 medicine_main_result_global_cache30_30_20_20.pdf diff --git a/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/eval_results.tsv b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/eval_results.tsv new file mode 100644 index 0000000..62ae28f --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/eval_results.tsv @@ -0,0 +1,2 @@ +mode lang_code BLEU MASKED_TERMS_BLEU MASKED_TERMS_HYP_REMOVED MASKED_TERMS_REF_REMOVED MASKED_TERMS_TYPES StreamLAAL StreamLAAL_CA TERM_ACC TERM_CORRECT TERM_TOTAL TERM_ADOPTION TERM_ADOPTED TERM_ADOPTION_TOTAL TERM_ADOPTION_SENTENCES TERM_ADOPTION_MICRO REAL_TERM_ADOPT REAL_TERM_ADOPTED REAL_TERM_ADOPT_TOTAL REAL_TERM_ADOPT_SENTENCES REAL_TERM_ADOPT_MICRO TERM_FCR FALSE_COPY NEG_TOTAL FALSE_COPY_TERMS instances_log TERM_FCR_MODE SOURCE_TERM_SENT_FCR SOURCE_FALSE_COPY SOURCE_NEG_TOTAL SOURCE_FALSE_COPY_TERMS +acl6060 zh 40.66631953556564 35.0391777589255 1331 1389 191 1181.147034771068 1503.3985473053056 0.7191 640 890 N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/instances.log unknown N/A N/A N/A N/A diff --git a/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/eval_results.tsv b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/eval_results.tsv new file mode 100644 index 0000000..7a95b86 --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/eval_results.tsv @@ -0,0 +1,2 @@ +mode lang_code BLEU MASKED_TERMS_BLEU MASKED_TERMS_HYP_REMOVED MASKED_TERMS_REF_REMOVED MASKED_TERMS_TYPES StreamLAAL StreamLAAL_CA TERM_ACC TERM_CORRECT TERM_TOTAL TERM_ADOPTION TERM_ADOPTED TERM_ADOPTION_TOTAL TERM_ADOPTION_SENTENCES TERM_ADOPTION_MICRO REAL_TERM_ADOPT REAL_TERM_ADOPTED REAL_TERM_ADOPT_TOTAL REAL_TERM_ADOPT_SENTENCES REAL_TERM_ADOPT_MICRO TERM_FCR FALSE_COPY NEG_TOTAL FALSE_COPY_TERMS instances_log TERM_FCR_MODE SOURCE_TERM_SENT_FCR SOURCE_FALSE_COPY SOURCE_NEG_TOTAL SOURCE_FALSE_COPY_TERMS +acl6060 zh 45.8267991845899 40.71553626512784 1322 1389 191 1765.719569988509 2124.465084719733 0.7517 669 890 N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/instances.log unknown N/A N/A N/A N/A diff --git a/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/eval_results.tsv b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/eval_results.tsv new file mode 100644 index 0000000..eea1772 --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/eval_results.tsv @@ -0,0 +1,2 @@ +mode lang_code BLEU MASKED_TERMS_BLEU MASKED_TERMS_HYP_REMOVED MASKED_TERMS_REF_REMOVED MASKED_TERMS_TYPES StreamLAAL StreamLAAL_CA TERM_ACC TERM_CORRECT TERM_TOTAL TERM_ADOPTION TERM_ADOPTED TERM_ADOPTION_TOTAL TERM_ADOPTION_SENTENCES TERM_ADOPTION_MICRO REAL_TERM_ADOPT REAL_TERM_ADOPTED REAL_TERM_ADOPT_TOTAL REAL_TERM_ADOPT_SENTENCES REAL_TERM_ADOPT_MICRO TERM_FCR FALSE_COPY NEG_TOTAL FALSE_COPY_TERMS instances_log TERM_FCR_MODE SOURCE_TERM_SENT_FCR SOURCE_FALSE_COPY SOURCE_NEG_TOTAL SOURCE_FALSE_COPY_TERMS +acl6060 zh 46.71194732237352 41.30484254869681 1328 1389 191 2232.673328950615 2679.683434409157 0.7404 659 890 N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/instances.log unknown N/A N/A N/A N/A diff --git a/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/eval_results.tsv b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/eval_results.tsv new file mode 100644 index 0000000..0ed3fc3 --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/eval_results.tsv @@ -0,0 +1,2 @@ +mode lang_code BLEU MASKED_TERMS_BLEU MASKED_TERMS_HYP_REMOVED MASKED_TERMS_REF_REMOVED MASKED_TERMS_TYPES StreamLAAL StreamLAAL_CA TERM_ACC TERM_CORRECT TERM_TOTAL TERM_ADOPTION TERM_ADOPTED TERM_ADOPTION_TOTAL TERM_ADOPTION_SENTENCES TERM_ADOPTION_MICRO REAL_TERM_ADOPT REAL_TERM_ADOPTED REAL_TERM_ADOPT_TOTAL REAL_TERM_ADOPT_SENTENCES REAL_TERM_ADOPT_MICRO TERM_FCR FALSE_COPY NEG_TOTAL FALSE_COPY_TERMS instances_log TERM_FCR_MODE SOURCE_TERM_SENT_FCR SOURCE_FALSE_COPY SOURCE_NEG_TOTAL SOURCE_FALSE_COPY_TERMS +acl6060 zh 47.38972006431048 41.82121421579427 1293 1389 191 2616.3493355732385 3187.8067040039464 0.7551 672 890 N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A N/A /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/instances.log unknown N/A N/A N/A N/A diff --git a/docs/results/main_result_global_cache30_30_20_20/masked_terms_artifacts.tsv b/docs/results/main_result_global_cache30_30_20_20/masked_terms_artifacts.tsv new file mode 100644 index 0000000..6ff5950 --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/masked_terms_artifacts.tsv @@ -0,0 +1,5 @@ +dataset method lang lm eval_results instances_log source_instances_log note +acl_tagged_raw InfiniSST zh 1 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/eval_results.tsv /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/instances.log /mnt/gemini/data2/jiaxuanluo/infinisst_omni_vllm_rag_rank16_baseline_gigaspeech-zh_acl6060_only_v3/zh/gigaspeech-zh-s_origin-bsz4_gglossary_acl6060_cs0.96_hs0.48_lm1_k210_k110_th0p0/instances.log Copied from InfiniSST rank16 baseline no-RAG ACL6060-only v3; BLEU/StreamLAAL match published En-Zh InfiniSST rows. +acl_tagged_raw InfiniSST zh 2 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/eval_results.tsv /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/instances.log /mnt/gemini/data2/jiaxuanluo/infinisst_omni_vllm_rag_rank16_baseline_gigaspeech-zh_acl6060_only_v3/zh/gigaspeech-zh-s_origin-bsz4_gglossary_acl6060_cs1.92_hs0.48_lm2_k210_k110_th0p0/instances.log Copied from InfiniSST rank16 baseline no-RAG ACL6060-only v3; BLEU/StreamLAAL match published En-Zh InfiniSST rows. +acl_tagged_raw InfiniSST zh 3 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/eval_results.tsv /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/instances.log /mnt/gemini/data2/jiaxuanluo/infinisst_omni_vllm_rag_rank16_baseline_gigaspeech-zh_acl6060_only_v3/zh/gigaspeech-zh-s_origin-bsz4_gglossary_acl6060_cs2.88_hs0.48_lm3_k210_k110_th0p0/instances.log Copied from InfiniSST rank16 baseline no-RAG ACL6060-only v3; BLEU/StreamLAAL match published En-Zh InfiniSST rows. +acl_tagged_raw InfiniSST zh 4 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/eval_results.tsv /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/instances.log /mnt/gemini/data2/jiaxuanluo/infinisst_omni_vllm_rag_rank16_baseline_gigaspeech-zh_acl6060_only_v3/zh/gigaspeech-zh-s_origin-bsz4_gglossary_acl6060_cs3.84_hs0.48_lm4_k210_k110_th0p0/instances.log Copied from InfiniSST rank16 baseline no-RAG ACL6060-only v3; BLEU/StreamLAAL match published En-Zh InfiniSST rows. diff --git a/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.pdf b/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.pdf new file mode 100644 index 0000000..780620c Binary files /dev/null and b/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.pdf differ diff --git a/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.png b/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.png new file mode 100644 index 0000000..941777f Binary files /dev/null and b/docs/results/main_result_global_cache30_30_20_20/masked_terms_bleu_global_cache30_30_20_20.png differ diff --git a/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality.tsv b/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality.tsv new file mode 100644 index 0000000..6a1f464 --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality.tsv @@ -0,0 +1,49 @@ +dataset method lang lm BLEU MASKED_TERMS_BLEU DELTA_MASKED_MINUS_BLEU TERM_ACC TERM_FCR FALSE_COPY NEG_TOTAL FALSE_COPY_TERMS MASKED_TERMS_HYP_REMOVED MASKED_TERMS_REF_REMOVED MASKED_TERMS_TYPES instances_log source_path status note +acl_tagged_raw InfiniSST zh 1 40.6663 35.0392 -5.6271 0.7431 N/A N/A N/A N/A 1331 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm1/instances.log user_prompt_2026-05-24 ok +acl_tagged_raw InfiniSST zh 2 45.8268 40.7155 -5.1113 0.7655 N/A N/A N/A N/A 1322 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm2/instances.log user_prompt_2026-05-24 ok +acl_tagged_raw InfiniSST zh 3 46.7119 41.3048 -5.4071 0.7675 N/A N/A N/A N/A 1328 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm3/instances.log user_prompt_2026-05-24 ok +acl_tagged_raw InfiniSST zh 4 47.3897 41.8212 -5.5685 0.7754 N/A N/A N/A N/A 1293 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST/docs/results/main_result_global_cache30_30_20_20/artifacts/acl_tagged_raw_infinisst_zh/lm4/instances.log user_prompt_2026-05-24 ok +acl_tagged_raw InfiniSST ja 1 24.4002 21.7166 -2.6836 0.6245 1.000000 5 5 286 1285 1347 201 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/ja/seg960/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/ja_lm1/eval_results.main.tsv ok +acl_tagged_raw InfiniSST ja 2 27.7202 25.0201 -2.7001 0.6596 1.000000 5 5 288 1268 1347 201 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/ja/seg1920/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/ja_lm2/eval_results.main.tsv ok +acl_tagged_raw InfiniSST ja 3 28.9684 25.5043 -3.4641 0.6617 1.000000 5 5 288 1207 1347 201 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/ja/seg2880/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/ja_lm3/eval_results.main.tsv ok +acl_tagged_raw InfiniSST ja 4 30.7506 27.2414 -3.5092 0.6830 1.000000 5 5 289 1228 1347 201 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/ja/seg3840/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/ja_lm4/eval_results.main.tsv ok +acl_tagged_raw InfiniSST de 1 27.1215 25.6574 -1.4641 0.6075 1.000000 5 5 264 662 957 230 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/de/seg960/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/de_lm1/eval_results.main.tsv ok +acl_tagged_raw InfiniSST de 2 30.2516 28.9758 -1.2758 0.6759 1.000000 5 5 274 683 957 230 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/de/seg1920/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/de_lm2/eval_results.main.tsv ok +acl_tagged_raw InfiniSST de 3 31.4666 30.4183 -1.0483 0.6513 1.000000 5 5 272 679 957 230 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/de/seg2880/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/de_lm3/eval_results.main.tsv ok +acl_tagged_raw InfiniSST de 4 33.1867 31.9432 -1.2435 0.6684 1.000000 5 5 263 685 957 230 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/acl6060_de_ja_results/de/seg3840/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_acl6060_de_ja_baseline_posteval/de_lm4/eval_results.main.tsv ok +acl_tagged_raw RASST de 1 26.4180 24.9360 -1.4820 0.8107 0.148688 51 343 56 927 957 230 /mnt/data1/jiaxuanluo/tagged_acl_de_lm3_serial_promptfix_cache30_audioauto_max40lm_20260526T003437_tagged_acl_de_lm1_serial_promptfix_cache30_audioauto_max40lm_taurus/de/dtagacl_de_lm1_serial_promptfix_cache30_audioauto_max40lm_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/data1/jiaxuanluo/tagged_acl_de_lm3_serial_promptfix_cache30_audioauto_max40lm_20260526T003437_tagged_acl_de_lm1_serial_promptfix_cache30_audioauto_max40lm_taurus/de/dtagacl_de_lm1_serial_promptfix_cache30_audioauto_max40lm_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST de 2 31.5758 29.6852 -1.8906 0.8299 0.119891 44 367 49 911 957 230 /mnt/data1/jiaxuanluo/main_result_rasst_serial_de_ja_cache30_max40lm_20260525T215831_main_result_rasst_serial_de_ja_cache30_max40lm_taurus_retry1/acl_tagged_raw/de_cap16_denoise_ttag_hn1024_tau078_omit_serial_chunks30_max40lm/de/dmainres_acl_serial_de_cap16denoise_ttag_hn1024_tau078_omit_chunks30_max40lm_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/data1/jiaxuanluo/main_result_rasst_serial_de_ja_cache30_max40lm_20260525T215831_main_result_rasst_serial_de_ja_cache30_max40lm_taurus_retry1/acl_tagged_raw/de_cap16_denoise_ttag_hn1024_tau078_omit_serial_chunks30_max40lm/de/dmainres_acl_serial_de_cap16denoise_ttag_hn1024_tau078_omit_chunks30_max40lm_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST de 3 32.5049 30.6110 -1.8939 0.8481 0.105398 41 389 48 910 957 230 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/acl_de_lm3_cache_sweep_20260528T085204Z/cache20_20/cells/acl_tagged_raw__de__lm3/de/drasst_main_result_acl_tagged_raw_de_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/acl_de_lm3_cache_sweep_20260528T085204Z/cache20_20/cells/acl_tagged_raw__de__lm3/de/drasst_main_result_acl_tagged_raw_de_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST de 4 34.2450 32.4669 -1.7781 0.8439 0.133501 53 397 57 924 957 230 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/acl_de_lm4_cache20_probe_20260528T101802Z/cells/acl_tagged_raw__de__lm4/de/drasst_main_result_acl_tagged_raw_de_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/acl_de_lm4_cache20_probe_20260528T101802Z/cells/acl_tagged_raw__de__lm4/de/drasst_main_result_acl_tagged_raw_de_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST ja 1 22.6615 19.4429 -3.2186 0.8170 0.255556 69 270 76 1669 1347 201 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm1/ja/drasst_main_result_acl_tagged_raw_ja_lm1_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm1/ja/drasst_main_result_acl_tagged_raw_ja_lm1_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST ja 2 29.8887 26.0339 -3.8548 0.8457 0.146104 45 308 53 1425 1347 201 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm2/ja/drasst_main_result_acl_tagged_raw_ja_lm2_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm2/ja/drasst_main_result_acl_tagged_raw_ja_lm2_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST ja 3 32.5319 27.8079 -4.7240 0.8755 0.148368 50 337 56 1434 1347 201 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm3/ja/drasst_main_result_acl_tagged_raw_ja_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm3/ja/drasst_main_result_acl_tagged_raw_ja_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST ja 4 33.3522 28.6547 -4.6975 0.8691 0.129477 47 363 54 1378 1347 201 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm4/ja/drasst_main_result_acl_tagged_raw_ja_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/acl_ja/cells/acl_tagged_raw__ja__lm4/ja/drasst_main_result_acl_tagged_raw_ja_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST zh 1 43.8652 37.7378 -6.1274 0.8865 0.158537 39 246 43 1490 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm1/zh/drasst_main_result_acl_tagged_raw_zh_lm1_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm1/zh/drasst_main_result_acl_tagged_raw_zh_lm1_lm1_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST zh 2 48.7921 41.8895 -6.9026 0.8899 0.109929 31 282 33 1408 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm2/zh/drasst_main_result_acl_tagged_raw_zh_lm2_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm2/zh/drasst_main_result_acl_tagged_raw_zh_lm2_lm2_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST zh 3 50.8150 44.0373 -6.7777 0.9079 0.059006 19 322 21 1396 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm3/zh/drasst_main_result_acl_tagged_raw_zh_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm3/zh/drasst_main_result_acl_tagged_raw_zh_lm3_lm3_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +acl_tagged_raw RASST zh 4 50.7421 43.9574 -6.7847 0.8989 0.082621 29 351 29 1401 1389 191 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm4/zh/drasst_main_result_acl_tagged_raw_zh_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8_20260528T114706Z/zh_new_model_2domain_serial_cache30_20_givenchunks_fix_aries_cpus8/cells/acl_tagged_raw__zh__lm4/zh/drasst_main_result_acl_tagged_raw_zh_lm4_lm4_k10_th0.78_gacl6060_tagged_gt_raw_min_norm2/eval_results.tsv ok +medicine_hardraw InfiniSST zh 1 27.9415 26.8063 -1.1352 0.4276 N/A N/A N/A N/A 1210 860 210 /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm1_aries01/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs0.96_hs0.48_lm1_k210_k110_th0p0/instances.log /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm1_aries01/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs0.96_hs0.48_lm1_k210_k110_th0p0/eval_results_streamlaal_term.hard_llm_manual_check.tsv ok +medicine_hardraw InfiniSST zh 2 37.8511 35.9263 -1.9248 0.4628 1.000000 5 5 97 649 860 210 /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260522/zh/gigaspeech-zh-s_origin-bsz4_gmedicine_gt571_abbrev_restored__medicine5_cs1.92_hs0.48_lm2_k210_k110_th0p0/instances.log /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260522/zh/gigaspeech-zh-s_origin-bsz4_gmedicine_gt571_abbrev_restored__medicine5_cs1.92_hs0.48_lm2_k210_k110_th0p0/eval_results_streamlaal_term.hard_llm_manual_check.tsv ok +medicine_hardraw InfiniSST zh 3 40.4082 38.3483 -2.0599 0.4926 N/A N/A N/A N/A 666 860 210 /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm3_aries67/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs2.88_hs0.48_lm3_k210_k110_th0p0/instances.log /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm3_aries67/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs2.88_hs0.48_lm3_k210_k110_th0p0/eval_results_streamlaal_term.hard_llm_manual_check.tsv ok +medicine_hardraw InfiniSST zh 4 41.4557 39.2708 -2.1849 0.5115 1.000000 5 5 116 649 860 210 /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm4_with605000_from_taurus_orig80/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs3.84_hs0.48_lm4_k210_k110_th0p0/instances.log /mnt/gemini/data1/jiaxuanluo/medicine_norag_baseline_abbrev_restored_batched_20260524_zh_lm4_with605000_from_taurus_orig80/zh/gigaspeech-zh-s_origin-bsz4_gstrict_fixed_medicine_glossary_abbrev_restored__medicine5_cs3.84_hs0.48_lm4_k210_k110_th0p0/eval_results_streamlaal_term.hard_llm_manual_check.tsv ok +medicine_hardraw InfiniSST de 1 22.8060 22.8698 +0.0638 0.4297 1.000000 5 5 91 355 655 211 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/de/seg960/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/de_lm1/eval_results.main.tsv ok +medicine_hardraw InfiniSST de 2 26.0778 26.1085 +0.0307 0.5116 1.000000 5 5 101 433 655 211 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/de/seg1920/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/de_lm2/eval_results.main.tsv ok +medicine_hardraw InfiniSST de 3 26.7200 26.7145 -0.0055 0.5240 1.000000 5 5 102 462 655 211 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/de/seg2880/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/de_lm3/eval_results.main.tsv ok +medicine_hardraw InfiniSST de 4 28.1365 28.1049 -0.0316 0.5641 1.000000 5 5 107 434 655 211 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/de/seg3840/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/de_lm4/eval_results.main.tsv ok +medicine_hardraw InfiniSST ja 1 19.8868 18.9874 -0.8994 0.3102 1.000000 5 5 82 397 869 210 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/ja/seg960/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/ja_lm1/eval_results.main.tsv ok +medicine_hardraw InfiniSST ja 2 24.6641 23.5100 -1.1541 0.3472 1.000000 5 5 82 435 869 210 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/ja/seg1920/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/ja_lm2/eval_results.main.tsv ok +medicine_hardraw InfiniSST ja 3 25.1361 23.8125 -1.3236 0.4198 1.000000 5 5 89 486 869 210 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/ja/seg2880/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/ja_lm3/eval_results.main.tsv ok +medicine_hardraw InfiniSST ja 4 26.6835 25.3950 -1.2885 0.4028 1.000000 5 5 88 459 869 210 /home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/figures/eso_de_ja_results/ja/seg3840/instances.log /home/jiaxuanluo/InfiniSST/documents/code/simuleval/reports/20260525_eso_de_ja_baseline_posteval/ja_lm4/eval_results.main.tsv ok +medicine_hardraw RASST zh 1 32.1943 30.5931 -1.6012 0.8024 0.303438 203 669 227 1303 860 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm1/zh/drasst_main_result_medicine_hardraw_zh_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm1/zh/drasst_main_result_medicine_hardraw_zh_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST zh 2 40.7642 39.0263 -1.7379 0.8262 0.214900 150 698 165 1108 860 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm2/zh/drasst_main_result_medicine_hardraw_zh_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm2/zh/drasst_main_result_medicine_hardraw_zh_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST zh 3 42.4881 40.6300 -1.8581 0.8380 0.192513 144 748 159 1085 860 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm3/zh/drasst_main_result_medicine_hardraw_zh_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm3/zh/drasst_main_result_medicine_hardraw_zh_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST zh 4 44.7022 41.8591 -2.8431 0.8440 0.176980 143 808 154 1071 860 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm4/zh/drasst_main_result_medicine_hardraw_zh_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/zh_cap16_denoise_medicine_serial_cache30_20_givenchunks_fix_aries_retry_20260528T130625Z/zh_new_model_medicine_serial_cache30_20_givenchunks_fix_aries_retry/cells/medicine_hardraw__zh__lm4/zh/drasst_main_result_medicine_hardraw_zh_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST de 1 22.6187 22.3654 -0.2533 0.7264 0.115031 75 652 77 661 655 211 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm1/de/drasst_main_result_medicine_hardraw_de_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm1/de/drasst_main_result_medicine_hardraw_de_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST de 2 26.7696 26.3115 -0.4581 0.7821 0.094891 65 685 65 739 655 211 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm2/de/drasst_main_result_medicine_hardraw_de_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm2/de/drasst_main_result_medicine_hardraw_de_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST de 3 27.3676 26.8689 -0.4987 0.8161 0.093496 69 738 72 729 655 211 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm3/de/drasst_main_result_medicine_hardraw_de_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm3/de/drasst_main_result_medicine_hardraw_de_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST de 4 27.9246 27.4331 -0.4915 0.8207 0.086035 69 802 72 747 655 211 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm4/de/drasst_main_result_medicine_hardraw_de_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_de/cells/medicine_hardraw__de__lm4/de/drasst_main_result_medicine_hardraw_de_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST ja 1 19.3213 18.1652 -1.1561 0.7515 0.260125 167 642 195 1209 869 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm1/ja/drasst_main_result_medicine_hardraw_ja_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm1/ja/drasst_main_result_medicine_hardraw_ja_lm1_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST ja 2 25.1412 23.6664 -1.4748 0.8086 0.244444 165 675 177 1154 869 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm2/ja/drasst_main_result_medicine_hardraw_ja_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm2/ja/drasst_main_result_medicine_hardraw_ja_lm2_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST ja 3 27.3388 25.7439 -1.5949 0.8318 0.194787 142 729 153 1102 869 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm3/ja/drasst_main_result_medicine_hardraw_ja_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm3/ja/drasst_main_result_medicine_hardraw_ja_lm3_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok +medicine_hardraw RASST ja 4 29.9012 28.2598 -1.6414 0.8503 0.180556 143 792 154 1119 869 210 /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm4/ja/drasst_main_result_medicine_hardraw_ja_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/instances.strip_term.log /mnt/taurus/data2/jiaxuanluo/RASST_release_runs/cache30_30_20_20_followup_after_acl_de_20260528T1142Z/medicine_ja/cells/medicine_hardraw__ja__lm4/ja/drasst_main_result_medicine_hardraw_ja_lm4_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212/eval_results.tsv ok diff --git a/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality_compare_vs_infinisst.tsv b/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality_compare_vs_infinisst.tsv new file mode 100644 index 0000000..f882f4b --- /dev/null +++ b/docs/results/main_result_global_cache30_30_20_20/masked_terms_quality_compare_vs_infinisst.tsv @@ -0,0 +1,25 @@ +dataset lang lm RASST_BLEU InfiniSST_BLEU delta_BLEU_vs_InfiniSST RASST_MASKED_TERMS_BLEU InfiniSST_MASKED_TERMS_BLEU delta_MASKED_TERMS_BLEU_vs_InfiniSST masked_delta_minus_original_delta status note +acl_tagged_raw de 1 26.4180 27.1215 -0.7035 24.9360 25.6574 -0.7214 -0.0179 ok +acl_tagged_raw de 2 31.5758 30.2516 +1.3242 29.6852 28.9758 +0.7094 -0.6148 ok +acl_tagged_raw de 3 32.5049 31.4666 +1.0383 30.6110 30.4183 +0.1927 -0.8456 ok +acl_tagged_raw de 4 34.2450 33.1867 +1.0583 32.4669 31.9432 +0.5237 -0.5346 ok +acl_tagged_raw ja 1 22.6615 24.4002 -1.7387 19.4429 21.7166 -2.2737 -0.5350 ok +acl_tagged_raw ja 2 29.8887 27.7202 +2.1685 26.0339 25.0201 +1.0138 -1.1547 ok +acl_tagged_raw ja 3 32.5319 28.9684 +3.5635 27.8079 25.5043 +2.3036 -1.2599 ok +acl_tagged_raw ja 4 33.3522 30.7506 +2.6016 28.6547 27.2414 +1.4133 -1.1883 ok +acl_tagged_raw zh 1 43.8652 40.6663 +3.1989 37.7378 35.0392 +2.6986 -0.5003 ok +acl_tagged_raw zh 2 48.7921 45.8268 +2.9653 41.8895 40.7155 +1.1740 -1.7913 ok +acl_tagged_raw zh 3 50.8150 46.7119 +4.1031 44.0373 41.3048 +2.7325 -1.3706 ok +acl_tagged_raw zh 4 50.7421 47.3897 +3.3524 43.9574 41.8212 +2.1362 -1.2162 ok +medicine_hardraw zh 1 32.1943 27.9415 +4.2528 30.5931 26.8063 +3.7868 -0.4660 ok +medicine_hardraw zh 2 40.7642 37.8511 +2.9131 39.0263 35.9263 +3.1000 +0.1869 ok +medicine_hardraw zh 3 42.4881 40.4082 +2.0799 40.6300 38.3483 +2.2817 +0.2018 ok +medicine_hardraw zh 4 44.7022 41.4557 +3.2465 41.8591 39.2708 +2.5883 -0.6582 ok +medicine_hardraw de 1 22.6187 22.8060 -0.1873 22.3654 22.8698 -0.5044 -0.3171 ok +medicine_hardraw de 2 26.7696 26.0778 +0.6918 26.3115 26.1085 +0.2030 -0.4888 ok +medicine_hardraw de 3 27.3676 26.7200 +0.6476 26.8689 26.7145 +0.1544 -0.4932 ok +medicine_hardraw de 4 27.9246 28.1365 -0.2119 27.4331 28.1049 -0.6718 -0.4599 ok +medicine_hardraw ja 1 19.3213 19.8868 -0.5655 18.1652 18.9874 -0.8222 -0.2567 ok +medicine_hardraw ja 2 25.1412 24.6641 +0.4771 23.6664 23.5100 +0.1564 -0.3207 ok +medicine_hardraw ja 3 27.3388 25.1361 +2.2027 25.7439 23.8125 +1.9314 -0.2713 ok +medicine_hardraw ja 4 29.9012 26.6835 +3.2177 28.2598 25.3950 +2.8648 -0.3529 ok diff --git a/docs/results/rag_compute_rtf/README.md b/docs/results/rag_compute_rtf/README.md new file mode 100644 index 0000000..aba0b74 --- /dev/null +++ b/docs/results/rag_compute_rtf/README.md @@ -0,0 +1,62 @@ +# RAG Compute RTF + +This directory tracks the RAG retriever compute real-time-factor figure used to +audit retrieval overhead as the streaming chunk/cadence changes. + +## Files + +| File | Contents | +| --- | --- | +| `data.tsv` | Frozen plotted summary for Medicine En-Zh hard/raw, `lm=1..4`. | +| `rag_compute_rtf.pdf/png` | RAG compute RTF figure. | +| `plot_rag_compute_rtf.py` | Paper-local plotting script that regenerates the local PDF/PNG from `data.tsv`. | + +## Metric + +The reported RTF is: + +```text +RAG compute RTF = retriever call time / (0.96s * LM) +``` + +The retriever encodes the current vLLM generation span plus a fixed 1.92s +look-back. The plotted input spans are therefore 2.88s, 3.84s, 4.80s, and 5.76s +for `lm=1,2,3,4`. + +## Summary + +| LM | vLLM cadence | Retriever input span | Median retrieve time | Median RAG RTF | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 0.96s | 2.88s | 36.957 ms | 3.8497% | +| 2 | 1.92s | 3.84s | 42.345 ms | 2.2055% | +| 3 | 2.88s | 4.80s | 42.560 ms | 1.4778% | +| 4 | 3.84s | 5.76s | 43.645 ms | 1.1366% | + +## Provenance + +This package was copied from the frozen InfiniSST paper-local figure package: + +```text +/home/jiaxuanluo/InfiniSST/documents/code/train/term_train/reports/EMNLP_26_InfiniSST_RAG_src/plot/figure_03_rag_compute_rtf/ +``` + +Original analysis manifest: + +```text +/home/jiaxuanluo/InfiniSST/documents/code/train/term_train/manifests/2026/05/20260525T0149__analysis__rag_compute_rtf_figure.json +``` + +The manifest records the source runtime traces under: + +```text +/mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/ +``` + +## Checksums + +```text +4cc153356f41f1b0ebf105565432129ed927b794937d6a73dc0635b6f54f6b0b data.tsv +ab6f2151195c5130bd3744515c93f5ecbf6ff5c1d1e13ef9e6fc908f5099aff9 rag_compute_rtf.pdf +da445178665f7122f8b7eda63193906f6665f29ef240c0cf2b93745114ac5356 rag_compute_rtf.png +3abd389cffe938016ba6cb1bd352048590f24f2624870ac53f36fad1def3e245 plot_rag_compute_rtf.py +``` diff --git a/docs/results/rag_compute_rtf/data.tsv b/docs/results/rag_compute_rtf/data.tsv new file mode 100644 index 0000000..ba393e3 --- /dev/null +++ b/docs/results/rag_compute_rtf/data.tsv @@ -0,0 +1,5 @@ +dataset lang runtime_bank lm vllm_cadence_sec retriever_input_span_sec lookback_sec streamlaal_sec rag_calls rag_mean_ms rag_median_ms rag_p90_ms rag_p95_ms rag_mean_rtf_pct rag_median_rtf_pct rag_p90_rtf_pct runtime_jsonl eval_results_tsv +Medicine zh raw 1 0.96 2.88 1.92 1.1401 14315 53.644 36.957 63.903 156.114 5.5879 3.8497 6.6566 /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/runtime_omni_vllm_maxsim_rag_1779590694_pid345153.jsonl /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm1_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/eval_results.tsv +Medicine zh raw 2 1.92 3.84 1.92 1.7724 7159 63.977 42.345 90.663 244.947 3.3322 2.2055 4.7220 /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/runtime_omni_vllm_maxsim_rag_1779590693_pid345260.jsonl /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm2_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/eval_results.tsv +Medicine zh raw 3 2.88 4.80 1.92 2.3205 4774 68.892 42.560 150.117 268.103 2.3921 1.4778 5.2124 /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/runtime_omni_vllm_maxsim_rag_1779590691_pid343474.jsonl /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm3_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/eval_results.tsv +Medicine zh raw 4 3.84 5.76 1.92 2.8217 3581 58.286 43.645 65.358 148.633 1.5179 1.1366 1.7020 /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/runtime_omni_vllm_maxsim_rag_1779591220_pid636415.jsonl /mnt/gemini/data1/jiaxuanluo/medicine_hardraw_hn1024_tau078_new_v9_batch_20260524T0242/zh/dmedhard5_new_v9_termtag_delay_oldnewv3_r32a64_hn1024_tau0p78_raw_lm4_k10_th0.78_ghard_medicine_glossary_raw_llm_judge_manual_zh215_unique212_ppmedicine5_hardraw/eval_results.tsv diff --git a/docs/results/rag_compute_rtf/plot_rag_compute_rtf.py b/docs/results/rag_compute_rtf/plot_rag_compute_rtf.py new file mode 100755 index 0000000..df89f62 --- /dev/null +++ b/docs/results/rag_compute_rtf/plot_rag_compute_rtf.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Plot paper Figure 3 from a frozen paper-local TSV snapshot.""" + +from __future__ import annotations + +import argparse +import csv +import shutil +from pathlib import Path +from typing import Dict, List, Sequence + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + + +SCRIPT_DIR = Path(__file__).resolve().parent +PAPER_DIR = SCRIPT_DIR.parents[1] +DEFAULT_DATA = SCRIPT_DIR / "data.tsv" +DEFAULT_OUT = SCRIPT_DIR / "rag_compute_rtf.pdf" + +LMS: Sequence[int] = (1, 2, 3, 4) + + +def read_rows(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8", errors="replace", newline="") as f: + return list(csv.DictReader(f, delimiter="\t")) + + +def select_rows(rows: List[Dict[str, str]]) -> List[Dict[str, str]]: + by_lm = {int(row["lm"]): row for row in rows} + missing = [lm for lm in LMS if lm not in by_lm] + if missing: + raise ValueError(f"missing lm rows: {missing}") + return [by_lm[lm] for lm in LMS] + + +def plot(rows: List[Dict[str, str]], out_pdf: Path) -> None: + out_pdf.parent.mkdir(parents=True, exist_ok=True) + + xs = [float(row["streamlaal_sec"]) * 1000.0 for row in rows] + mean_rtf = [float(row["rag_mean_rtf_pct"]) for row in rows] + median_ms = [float(row["rag_median_ms"]) for row in rows] + + metrics = [ + ("Retriever RTF (%)", mean_rtf, "#D6604D", "o"), + ("Retriever Time\nper Call (ms)", median_ms, "#4393C3", "D"), + ] + + plt.rcParams.update( + { + "font.family": "serif", + "font.size": 14, + "axes.titlesize": 15, + "axes.titleweight": "bold", + "axes.labelsize": 14, + "legend.fontsize": 13, + "xtick.labelsize": 13, + "ytick.labelsize": 13, + "pdf.fonttype": 42, + "ps.fonttype": 42, + } + ) + fig, axes = plt.subplots(len(metrics), 1, figsize=(4.2, 5.2), sharex=True) + for ax, (ylabel, ys, color, marker) in zip(axes, metrics): + ax.plot( + xs, + ys, + marker=marker, + linewidth=2.0, + markersize=7.0, + color=color, + ) + ax.grid(True, which="major", linestyle=":", linewidth=0.6, alpha=0.5) + ax.set_ylabel(ylabel) + + axes[-1].set_xlabel("StreamLAAL (ms)") + fig.tight_layout(h_pad=1.0) + fig.savefig(out_pdf) + fig.savefig(out_pdf.with_suffix(".png"), dpi=220) + plt.close(fig) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--data", type=Path, default=DEFAULT_DATA) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument( + "--update-paper", + action="store_true", + help="Also copy regenerated PDF/PNG into latex/figures.", + ) + args = parser.parse_args() + + rows = select_rows(read_rows(args.data)) + plot(rows, args.out) + print(f"[OK] wrote {args.out}") + print(f"[OK] wrote {args.out.with_suffix('.png')}") + if args.update_paper: + figure_dir = PAPER_DIR / "latex/figures" + figure_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(args.out, figure_dir / "rag_compute_rtf.pdf") + shutil.copy2( + args.out.with_suffix(".png"), + figure_dir / "rag_compute_rtf.png", + ) + print(f"[OK] updated {figure_dir / 'rag_compute_rtf.pdf'}") + print(f"[OK] updated {figure_dir / 'rag_compute_rtf.png'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/results/rag_compute_rtf/rag_compute_rtf.pdf b/docs/results/rag_compute_rtf/rag_compute_rtf.pdf new file mode 100644 index 0000000..2296e6e Binary files /dev/null and b/docs/results/rag_compute_rtf/rag_compute_rtf.pdf differ diff --git a/docs/results/rag_compute_rtf/rag_compute_rtf.png b/docs/results/rag_compute_rtf/rag_compute_rtf.png new file mode 100644 index 0000000..c206970 Binary files /dev/null and b/docs/results/rag_compute_rtf/rag_compute_rtf.png differ